Работа над новым поиском

server/config/base.js

@@ -26,6 +26,7 @@ module.exports = {
     cacheCleanInterval: 60,//minutes
     inpxCheckInterval: 60,//minutes
     lowMemoryMode: false,
+    fullOptimization: false,
 
     webConfigParams: ['name', 'version', 'branch', 'bookReadLink', 'dbVersion', 'extendedSearch'],
 

server/config/index.js

@@ -15,6 +15,7 @@ const propsToSave = [
     'cacheCleanInterval',
     'inpxCheckInterval',
     'lowMemoryMode',
+    'fullOptimization',
     'allowRemoteLib',
     'remoteLib',
     'server',

server/core/DbCreator.js

@@ -138,7 +138,7 @@ class DbCreator {
                 callback({progress: (readState.current || 0)/totalFiles});
         };
 
-        const parseField = (fieldValue, fieldMap, fieldArr, bookId, fillBookIds = true) => {
+        const parseField = (fieldValue, fieldMap, fieldArr, bookId, rec, fillBookIds = true) => {
             let value = fieldValue;
 
             if (typeof(fieldValue) == 'string') {
@@ -154,12 +154,24 @@ class DbCreator {
                 fieldRec = fieldArr[fieldId];
             } else {
                 fieldRec = {id: fieldArr.length, value, bookIds: new Set()};                
+                if (rec !== undefined) {
+                    fieldRec.name = fieldValue;
+                    fieldRec.bookCount = 0;
+                    fieldRec.bookDelCount = 0;
+                }
                 fieldArr.push(fieldRec);
                 fieldMap.set(value, fieldRec.id);
             }
 
             if (fieldValue !== emptyFieldValue || fillBookIds)
                 fieldRec.bookIds.add(bookId);
+
+            if (rec !== undefined) {
+                if (!rec.del)
+                    fieldRec.bookCount++;
+                else
+                    fieldRec.bookDelCount++;
+            }
         };        
 
         const parseBookRec = (rec) => {
@@ -173,14 +185,14 @@ class DbCreator {
                 if (!authorMap.has(a.toLowerCase()) && (author.length == 1 || i < author.length - 1)) //без соавторов
                     authorCount++;
                 
-                parseField(a, authorMap, authorArr, rec.id);                
+                parseField(a, authorMap, authorArr, rec.id, rec);                
             }
 
             //серии
-            parseField(rec.series, seriesMap, seriesArr, rec.id, false);
+            parseField(rec.series, seriesMap, seriesArr, rec.id, rec, false);
 
             //названия
-            parseField(rec.title, titleMap, titleArr, rec.id);
+            parseField(rec.title, titleMap, titleArr, rec.id, rec);
 
             //жанры
             let genre = rec.genre || emptyFieldValue;
@@ -393,10 +405,12 @@ class DbCreator {
         await db.create({table: 'file_hash'});
 
         //-- завершающие шаги --------------------------------
-        await db.open({
+        if (config.fullOptimization) {
-            table: 'book',
+            await db.open({
-            cacheSize: (config.lowMemoryMode ? 5 : 500),
+                table: 'book',
-        });
+                cacheSize: (config.lowMemoryMode ? 5 : 500),
+            });
+        }
 
         callback({job: 'optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0});
         await this.optimizeTable('author', db, (p) => {
@@ -419,7 +433,8 @@ class DbCreator {
         await this.countStats(db, callback, stats);
 
         //чистка памяти, ибо жрет как не в себя
-        await db.drop({table: 'book'});//больше не понадобится
+        if (config.fullOptimization)
+            await db.close({table: 'book'});
         await db.freeMemory();
         utils.freeMemory();
 
@@ -440,17 +455,13 @@ class DbCreator {
     }
 
     async optimizeTable(from, db, callback) {
+        const config = this.config;
+
         const to = `${from}_book`;
         const toId = `${from}_id`;
-        const restoreProp = from;
 
-        //оптимизация таблицы from, превращаем массив bookId в books, кладем все в таблицу to
         await db.open({table: from});
-
+        await db.create({table: to});
-        await db.create({
-            table: to,
-            flag: {name: 'toDel', check: 'r => r.toDel'},
-        });
 
         const bookId2RecId = new Map();
 
@@ -469,46 +480,35 @@ class DbCreator {
                 }
             }
 
-            ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb
+            if (config.fullOptimization) {
+                ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb
 
-            const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
+                const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
 
-            const bookArr = new Map();
+                const bookArr = new Map();
-            for (const row of rows)
+                for (const row of rows)
-                bookArr.set(row.id, row);
+                    bookArr.set(row.id, row);
 
-            for (const rec of chunk) {
+                for (const rec of chunk) {
-                rec.books = [];
+                    rec.books = [];
-                rec.bookCount = 0;
-                rec.bookDelCount = 0;
 
-                for (const id of rec.bookIds) {
+                    for (const id of rec.bookIds) {
-                    const book = bookArr.get(id);
+                        const book = bookArr.get(id);
-                    if (rec) {//на всякий случай
+                        if (book) {//на всякий случай
-                        rec.books.push(book);
+                            rec.books.push(book);
-                        if (!book.del)
+                        }
-                            rec.bookCount++;
-                        else
-                            rec.bookDelCount++;
                     }
+
+                    delete rec.name;
+                    delete rec.value;
+                    delete rec.bookIds;
                 }
 
-                if (rec.books.length) {
+                await db.insert({
-                    rec[restoreProp] = rec.value;//rec.books[0][restoreProp];
+                    table: to,
-                    if (!rec[restoreProp])
+                    rows: chunk,
-                        rec[restoreProp] = emptyFieldValue;
+                });
-                } else {
-                    rec.toDel = 1;
-                }
-
-                delete rec.value;
-                delete rec.bookIds;
             }
-
-            await db.insert({
-                table: to,
-                rows: chunk,
-            });
         };
 
         const rows = await db.select({table: from, count: true});
@@ -558,7 +558,6 @@ class DbCreator {
             }
         }
 
-        await db.delete({table: to, where: `@@flag('toDel')`});
         await db.close({table: to});
         await db.close({table: from});
 

server/core/DbSearcher.js

@@ -500,6 +500,142 @@ class DbSearcher {
         return tableIds;
     }
 
+    async restoreBooks(from, ids) {
+        const db = this.db;
+        const bookTable = `${from}_book`;
+
+        const rows = await db.select({
+            table: bookTable,
+            where: `@@id(${db.esc(ids)})`
+        });
+
+        if (rows.length == ids.length)
+            return rows;
+
+        const idsSet = new Set(rows.map(r => r.id));
+
+        for (const id of ids) {
+            if (!idsSet.has(id)) {
+                const bookIds = await db.select({
+                    table: from,
+                    where: `@@id(${db.esc(id)})`
+                });
+
+                if (!bookIds.length)
+                    continue;
+
+                let books = await db.select({
+                    table: 'book',
+                    where: `@@id(${db.esc(bookIds[0].bookIds)})`
+                });
+
+                if (!books.length)
+                    continue;
+
+                rows.push({id, name: bookIds[0].name, books});
+
+                await db.insert({table: bookTable, ignore: true, rows});
+            }
+        }
+
+        return rows;
+    }
+
+    async search(from, query) {
+        if (this.closed)
+            throw new Error('DbSearcher closed');
+
+        if (!['author', 'series', 'title'].includes(from))
+            throw new Error(`Unknown value for param 'from'`);
+
+        this.searchFlag++;
+
+        try {
+            const db = this.db;
+
+            const ids = await this.selectTableIds(from, query);
+
+            const totalFound = ids.length;            
+            let limit = (query.limit ? query.limit : 100);
+            limit = (limit > maxLimit ? maxLimit : limit);
+            const offset = (query.offset ? query.offset : 0);
+
+            //выборка найденных значений
+            const found = await db.select({
+                table: from,
+                map: `(r) => ({id: r.id, ${from}: r.name, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
+                where: `@@id(${db.esc(ids.slice(offset, offset + limit))})`
+            });
+
+            return {found, totalFound};
+        } finally {
+            this.searchFlag--;
+        }
+    }
+
+    async getAuthorBookList(authorId) {
+        if (this.closed)
+            throw new Error('DbSearcher closed');
+
+        if (!authorId)
+            return {author: '', books: ''};
+
+        this.searchFlag++;
+
+        try {
+            //выборка книг автора по authorId
+            const rows = await this.restoreBooks('author', [authorId])
+
+            let author = '';
+            let books = '';
+
+            if (rows.length) {
+                author = rows[0].name;
+                books = rows[0].books;
+            }
+
+            return {author, books: (books && books.length ? JSON.stringify(books) : '')};
+        } finally {
+            this.searchFlag--;
+        }
+    }
+
+    async getSeriesBookList(series) {
+        if (this.closed)
+            throw new Error('DbSearcher closed');
+
+        if (!series)
+            return {books: ''};
+
+        this.searchFlag++;
+
+        try {
+            const db = this.db;
+
+            series = series.toLowerCase();
+
+            //выборка серии по названию серии
+            let rows = await db.select({
+                table: 'series',
+                rawResult: true,
+                where: `return Array.from(@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)}))`
+            });
+
+            let books;
+            if (rows.length && rows[0].rawResult.length) {
+                //выборка книг серии
+                const rows = await this.restoreBooks('series', [rows[0].rawResult[0]])
+
+                if (rows.length)
+                    books = rows[0].books;
+            }
+
+            return {books: (books && books.length ? JSON.stringify(books) : '')};
+        } finally {
+            this.searchFlag--;
+        }
+    }
+
     async getCached(key) {
         if (!this.config.queryCacheEnabled)
             return null;
@@ -572,109 +708,6 @@ class DbSearcher {
         });
     }
 
-    async search(from, query) {
-        if (this.closed)
-            throw new Error('DbSearcher closed');
-
-        if (!['author', 'series', 'title'].includes(from))
-            throw new Error(`Unknown value for param 'from'`);
-
-        this.searchFlag++;
-
-        try {
-            const db = this.db;
-
-            const ids = await this.selectTableIds(from, query);
-
-            const totalFound = ids.length;            
-            let limit = (query.limit ? query.limit : 100);
-            limit = (limit > maxLimit ? maxLimit : limit);
-            const offset = (query.offset ? query.offset : 0);
-
-            //выборка найденных авторов
-            const found = await db.select({
-                table: `${from}_book`,
-                map: `(r) => ({id: r.id, ${from}: r.${from}, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
-                where: `@@id(${db.esc(ids.slice(offset, offset + limit))})`
-            });
-
-            return {found, totalFound};
-        } finally {
-            this.searchFlag--;
-        }
-    }
-
-    async getAuthorBookList(authorId) {
-        if (this.closed)
-            throw new Error('DbSearcher closed');
-
-        if (!authorId)
-            return {author: '', books: ''};
-
-        this.searchFlag++;
-
-        try {
-            const db = this.db;
-
-            //выборка книг автора по authorId
-            const rows = await db.select({
-                table: 'author_book',
-                where: `@@id(${db.esc(authorId)})`
-            });
-
-            let author = '';
-            let books = '';
-
-            if (rows.length) {
-                author = rows[0].author;
-                books = rows[0].books;
-            }
-
-            return {author, books: (books && books.length ? JSON.stringify(books) : '')};
-        } finally {
-            this.searchFlag--;
-        }
-    }
-
-    async getSeriesBookList(series) {
-        if (this.closed)
-            throw new Error('DbSearcher closed');
-
-        if (!series)
-            return {books: ''};
-
-        this.searchFlag++;
-
-        try {
-            const db = this.db;
-
-            series = series.toLowerCase();
-
-            //выборка серии по названию серии
-            let rows = await db.select({
-                table: 'series',
-                rawResult: true,
-                where: `return Array.from(@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)}))`
-            });
-
-            let books;
-            if (rows.length && rows[0].rawResult.length) {
-                //выборка книг серии
-                rows = await db.select({
-                    table: 'series_book',
-                    where: `@@id(${rows[0].rawResult[0]})`
-                });
-
-                if (rows.length)
-                    books = rows[0].books;
-            }
-
-            return {books: (books && books.length ? JSON.stringify(books) : '')};
-        } finally {
-            this.searchFlag--;
-        }
-    }
-
     async periodicCleanCache() {
         this.timer = null;
         const cleanInterval = this.config.cacheCleanInterval*60*1000;