diff --git a/server/config/base.js b/server/config/base.js index b6707a8..4c5d66c 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -26,6 +26,7 @@ module.exports = { cacheCleanInterval: 60,//minutes inpxCheckInterval: 60,//minutes lowMemoryMode: false, + fullOptimization: false, webConfigParams: ['name', 'version', 'branch', 'bookReadLink', 'dbVersion', 'extendedSearch'], diff --git a/server/config/index.js b/server/config/index.js index 5b34533..5f1332d 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -15,6 +15,7 @@ const propsToSave = [ 'cacheCleanInterval', 'inpxCheckInterval', 'lowMemoryMode', + 'fullOptimization', 'allowRemoteLib', 'remoteLib', 'server', diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 6118fa2..873dc92 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -138,7 +138,7 @@ class DbCreator { callback({progress: (readState.current || 0)/totalFiles}); }; - const parseField = (fieldValue, fieldMap, fieldArr, bookId, fillBookIds = true) => { + const parseField = (fieldValue, fieldMap, fieldArr, bookId, rec, fillBookIds = true) => { let value = fieldValue; if (typeof(fieldValue) == 'string') { @@ -154,12 +154,24 @@ class DbCreator { fieldRec = fieldArr[fieldId]; } else { fieldRec = {id: fieldArr.length, value, bookIds: new Set()}; + if (rec !== undefined) { + fieldRec.name = fieldValue; + fieldRec.bookCount = 0; + fieldRec.bookDelCount = 0; + } fieldArr.push(fieldRec); fieldMap.set(value, fieldRec.id); } if (fieldValue !== emptyFieldValue || fillBookIds) fieldRec.bookIds.add(bookId); + + if (rec !== undefined) { + if (!rec.del) + fieldRec.bookCount++; + else + fieldRec.bookDelCount++; + } }; const parseBookRec = (rec) => { @@ -173,14 +185,14 @@ class DbCreator { if (!authorMap.has(a.toLowerCase()) && (author.length == 1 || i < author.length - 1)) //без соавторов authorCount++; - parseField(a, authorMap, authorArr, rec.id); + parseField(a, authorMap, authorArr, rec.id, rec); } //серии - parseField(rec.series, seriesMap, seriesArr, rec.id, false); + parseField(rec.series, seriesMap, seriesArr, rec.id, rec, false); //названия - parseField(rec.title, titleMap, titleArr, rec.id); + parseField(rec.title, titleMap, titleArr, rec.id, rec); //жанры let genre = rec.genre || emptyFieldValue; @@ -393,10 +405,12 @@ class DbCreator { await db.create({table: 'file_hash'}); //-- завершающие шаги -------------------------------- - await db.open({ - table: 'book', - cacheSize: (config.lowMemoryMode ? 5 : 500), - }); + if (config.fullOptimization) { + await db.open({ + table: 'book', + cacheSize: (config.lowMemoryMode ? 5 : 500), + }); + } callback({job: 'optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0}); await this.optimizeTable('author', db, (p) => { @@ -419,7 +433,8 @@ class DbCreator { await this.countStats(db, callback, stats); //чистка памяти, ибо жрет как не в себя - await db.drop({table: 'book'});//больше не понадобится + if (config.fullOptimization) + await db.close({table: 'book'}); await db.freeMemory(); utils.freeMemory(); @@ -440,17 +455,13 @@ class DbCreator { } async optimizeTable(from, db, callback) { + const config = this.config; + const to = `${from}_book`; const toId = `${from}_id`; - const restoreProp = from; - //оптимизация таблицы from, превращаем массив bookId в books, кладем все в таблицу to await db.open({table: from}); - - await db.create({ - table: to, - flag: {name: 'toDel', check: 'r => r.toDel'}, - }); + await db.create({table: to}); const bookId2RecId = new Map(); @@ -469,46 +480,35 @@ class DbCreator { } } - ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb + if (config.fullOptimization) { + ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb - const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`}); + const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`}); - const bookArr = new Map(); - for (const row of rows) - bookArr.set(row.id, row); + const bookArr = new Map(); + for (const row of rows) + bookArr.set(row.id, row); - for (const rec of chunk) { - rec.books = []; - rec.bookCount = 0; - rec.bookDelCount = 0; + for (const rec of chunk) { + rec.books = []; - for (const id of rec.bookIds) { - const book = bookArr.get(id); - if (rec) {//на всякий случай - rec.books.push(book); - if (!book.del) - rec.bookCount++; - else - rec.bookDelCount++; + for (const id of rec.bookIds) { + const book = bookArr.get(id); + if (book) {//на всякий случай + rec.books.push(book); + } } + + delete rec.name; + delete rec.value; + delete rec.bookIds; } - if (rec.books.length) { - rec[restoreProp] = rec.value;//rec.books[0][restoreProp]; - if (!rec[restoreProp]) - rec[restoreProp] = emptyFieldValue; - } else { - rec.toDel = 1; - } - - delete rec.value; - delete rec.bookIds; + await db.insert({ + table: to, + rows: chunk, + }); } - - await db.insert({ - table: to, - rows: chunk, - }); }; const rows = await db.select({table: from, count: true}); @@ -558,7 +558,6 @@ class DbCreator { } } - await db.delete({table: to, where: `@@flag('toDel')`}); await db.close({table: to}); await db.close({table: from}); diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 6d35073..8520139 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -500,6 +500,142 @@ class DbSearcher { return tableIds; } + async restoreBooks(from, ids) { + const db = this.db; + const bookTable = `${from}_book`; + + const rows = await db.select({ + table: bookTable, + where: `@@id(${db.esc(ids)})` + }); + + if (rows.length == ids.length) + return rows; + + const idsSet = new Set(rows.map(r => r.id)); + + for (const id of ids) { + if (!idsSet.has(id)) { + const bookIds = await db.select({ + table: from, + where: `@@id(${db.esc(id)})` + }); + + if (!bookIds.length) + continue; + + let books = await db.select({ + table: 'book', + where: `@@id(${db.esc(bookIds[0].bookIds)})` + }); + + if (!books.length) + continue; + + rows.push({id, name: bookIds[0].name, books}); + + await db.insert({table: bookTable, ignore: true, rows}); + } + } + + return rows; + } + + async search(from, query) { + if (this.closed) + throw new Error('DbSearcher closed'); + + if (!['author', 'series', 'title'].includes(from)) + throw new Error(`Unknown value for param 'from'`); + + this.searchFlag++; + + try { + const db = this.db; + + const ids = await this.selectTableIds(from, query); + + const totalFound = ids.length; + let limit = (query.limit ? query.limit : 100); + limit = (limit > maxLimit ? maxLimit : limit); + const offset = (query.offset ? query.offset : 0); + + //выборка найденных значений + const found = await db.select({ + table: from, + map: `(r) => ({id: r.id, ${from}: r.name, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, + where: `@@id(${db.esc(ids.slice(offset, offset + limit))})` + }); + + return {found, totalFound}; + } finally { + this.searchFlag--; + } + } + + async getAuthorBookList(authorId) { + if (this.closed) + throw new Error('DbSearcher closed'); + + if (!authorId) + return {author: '', books: ''}; + + this.searchFlag++; + + try { + //выборка книг автора по authorId + const rows = await this.restoreBooks('author', [authorId]) + + let author = ''; + let books = ''; + + if (rows.length) { + author = rows[0].name; + books = rows[0].books; + } + + return {author, books: (books && books.length ? JSON.stringify(books) : '')}; + } finally { + this.searchFlag--; + } + } + + async getSeriesBookList(series) { + if (this.closed) + throw new Error('DbSearcher closed'); + + if (!series) + return {books: ''}; + + this.searchFlag++; + + try { + const db = this.db; + + series = series.toLowerCase(); + + //выборка серии по названию серии + let rows = await db.select({ + table: 'series', + rawResult: true, + where: `return Array.from(@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)}))` + }); + + let books; + if (rows.length && rows[0].rawResult.length) { + //выборка книг серии + const rows = await this.restoreBooks('series', [rows[0].rawResult[0]]) + + if (rows.length) + books = rows[0].books; + } + + return {books: (books && books.length ? JSON.stringify(books) : '')}; + } finally { + this.searchFlag--; + } + } + async getCached(key) { if (!this.config.queryCacheEnabled) return null; @@ -572,109 +708,6 @@ class DbSearcher { }); } - async search(from, query) { - if (this.closed) - throw new Error('DbSearcher closed'); - - if (!['author', 'series', 'title'].includes(from)) - throw new Error(`Unknown value for param 'from'`); - - this.searchFlag++; - - try { - const db = this.db; - - const ids = await this.selectTableIds(from, query); - - const totalFound = ids.length; - let limit = (query.limit ? query.limit : 100); - limit = (limit > maxLimit ? maxLimit : limit); - const offset = (query.offset ? query.offset : 0); - - //выборка найденных авторов - const found = await db.select({ - table: `${from}_book`, - map: `(r) => ({id: r.id, ${from}: r.${from}, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, - where: `@@id(${db.esc(ids.slice(offset, offset + limit))})` - }); - - return {found, totalFound}; - } finally { - this.searchFlag--; - } - } - - async getAuthorBookList(authorId) { - if (this.closed) - throw new Error('DbSearcher closed'); - - if (!authorId) - return {author: '', books: ''}; - - this.searchFlag++; - - try { - const db = this.db; - - //выборка книг автора по authorId - const rows = await db.select({ - table: 'author_book', - where: `@@id(${db.esc(authorId)})` - }); - - let author = ''; - let books = ''; - - if (rows.length) { - author = rows[0].author; - books = rows[0].books; - } - - return {author, books: (books && books.length ? JSON.stringify(books) : '')}; - } finally { - this.searchFlag--; - } - } - - async getSeriesBookList(series) { - if (this.closed) - throw new Error('DbSearcher closed'); - - if (!series) - return {books: ''}; - - this.searchFlag++; - - try { - const db = this.db; - - series = series.toLowerCase(); - - //выборка серии по названию серии - let rows = await db.select({ - table: 'series', - rawResult: true, - where: `return Array.from(@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)}))` - }); - - let books; - if (rows.length && rows[0].rawResult.length) { - //выборка книг серии - rows = await db.select({ - table: 'series_book', - where: `@@id(${rows[0].rawResult[0]})` - }); - - if (rows.length) - books = rows[0].books; - } - - return {books: (books && books.length ? JSON.stringify(books) : '')}; - } finally { - this.searchFlag--; - } - } - async periodicCleanCache() { this.timer = null; const cleanInterval = this.config.cacheCleanInterval*60*1000;