Работа над новым поиском

2022-10-30 15:03:29 +07:00
parent 70de9e2ab4
commit 1210e5bd8a
4 changed files with 186 additions and 152 deletions
--- a/server/config/base.js
+++ b/server/config/base.js
@@ -26,6 +26,7 @@ module.exports = {
    cacheCleanInterval: 60,//minutes
    inpxCheckInterval: 60,//minutes
    lowMemoryMode: false,
+    fullOptimization: false,

    webConfigParams: ['name', 'version', 'branch', 'bookReadLink', 'dbVersion', 'extendedSearch'],

--- a/server/config/index.js
+++ b/server/config/index.js
@@ -15,6 +15,7 @@ const propsToSave = [
    'cacheCleanInterval',
    'inpxCheckInterval',
    'lowMemoryMode',
+    'fullOptimization',
    'allowRemoteLib',
    'remoteLib',
    'server',
--- a/server/core/DbCreator.js
+++ b/server/core/DbCreator.js
@@ -138,7 +138,7 @@ class DbCreator {
                callback({progress: (readState.current || 0)/totalFiles});
        };

-        const parseField = (fieldValue, fieldMap, fieldArr, bookId, fillBookIds = true) => {
+        const parseField = (fieldValue, fieldMap, fieldArr, bookId, rec, fillBookIds = true) => {
            let value = fieldValue;

            if (typeof(fieldValue) == 'string') {
@@ -154,12 +154,24 @@ class DbCreator {
                fieldRec = fieldArr[fieldId];
            } else {
                fieldRec = {id: fieldArr.length, value, bookIds: new Set()};                
+                if (rec !== undefined) {
+                    fieldRec.name = fieldValue;
+                    fieldRec.bookCount = 0;
+                    fieldRec.bookDelCount = 0;
+                }
                fieldArr.push(fieldRec);
                fieldMap.set(value, fieldRec.id);
            }

            if (fieldValue !== emptyFieldValue || fillBookIds)
                fieldRec.bookIds.add(bookId);
+
+            if (rec !== undefined) {
+                if (!rec.del)
+                    fieldRec.bookCount++;
+                else
+                    fieldRec.bookDelCount++;
+            }
        };        

        const parseBookRec = (rec) => {
@@ -173,14 +185,14 @@ class DbCreator {
                if (!authorMap.has(a.toLowerCase()) && (author.length == 1 || i < author.length - 1)) //без соавторов
                    authorCount++;
                
-                parseField(a, authorMap, authorArr, rec.id);                
+                parseField(a, authorMap, authorArr, rec.id, rec);                
            }

            //серии
-            parseField(rec.series, seriesMap, seriesArr, rec.id, false);
+            parseField(rec.series, seriesMap, seriesArr, rec.id, rec, false);

            //названия
-            parseField(rec.title, titleMap, titleArr, rec.id);
+            parseField(rec.title, titleMap, titleArr, rec.id, rec);

            //жанры
            let genre = rec.genre || emptyFieldValue;
@@ -393,10 +405,12 @@ class DbCreator {
        await db.create({table: 'file_hash'});

        //-- завершающие шаги --------------------------------
-        await db.open({
-            table: 'book',
-            cacheSize: (config.lowMemoryMode ? 5 : 500),
-        });
+        if (config.fullOptimization) {
+            await db.open({
+                table: 'book',
+                cacheSize: (config.lowMemoryMode ? 5 : 500),
+            });
+        }

        callback({job: 'optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0});
        await this.optimizeTable('author', db, (p) => {
@@ -419,7 +433,8 @@ class DbCreator {
        await this.countStats(db, callback, stats);

        //чистка памяти, ибо жрет как не в себя
-        await db.drop({table: 'book'});//больше не понадобится
+        if (config.fullOptimization)
+            await db.close({table: 'book'});
        await db.freeMemory();
        utils.freeMemory();

@@ -440,17 +455,13 @@ class DbCreator {
    }

    async optimizeTable(from, db, callback) {
+        const config = this.config;
+
        const to = `${from}_book`;
        const toId = `${from}_id`;
-        const restoreProp = from;

-        //оптимизация таблицы from, превращаем массив bookId в books, кладем все в таблицу to
        await db.open({table: from});
-
-        await db.create({
-            table: to,
-            flag: {name: 'toDel', check: 'r => r.toDel'},
-        });
+        await db.create({table: to});

        const bookId2RecId = new Map();

@@ -469,46 +480,35 @@ class DbCreator {
                }
            }

-            ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb
+            if (config.fullOptimization) {
+                ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb

-            const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
+                const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});

-            const bookArr = new Map();
-            for (const row of rows)
-                bookArr.set(row.id, row);
+                const bookArr = new Map();
+                for (const row of rows)
+                    bookArr.set(row.id, row);

-            for (const rec of chunk) {
-                rec.books = [];
-                rec.bookCount = 0;
-                rec.bookDelCount = 0;
+                for (const rec of chunk) {
+                    rec.books = [];

-                for (const id of rec.bookIds) {
-                    const book = bookArr.get(id);
-                    if (rec) {//на всякий случай
-                        rec.books.push(book);
-                        if (!book.del)
-                            rec.bookCount++;
-                        else
-                            rec.bookDelCount++;
+                    for (const id of rec.bookIds) {
+                        const book = bookArr.get(id);
+                        if (book) {//на всякий случай
+                            rec.books.push(book);
+                        }
                    }
+
+                    delete rec.name;
+                    delete rec.value;
+                    delete rec.bookIds;
                }

-                if (rec.books.length) {
-                    rec[restoreProp] = rec.value;//rec.books[0][restoreProp];
-                    if (!rec[restoreProp])
-                        rec[restoreProp] = emptyFieldValue;
-                } else {
-                    rec.toDel = 1;
-                }
-
-                delete rec.value;
-                delete rec.bookIds;
+                await db.insert({
+                    table: to,
+                    rows: chunk,
+                });
            }
-
-            await db.insert({
-                table: to,
-                rows: chunk,
-            });
        };

        const rows = await db.select({table: from, count: true});
@@ -558,7 +558,6 @@ class DbCreator {
            }
        }

-        await db.delete({table: to, where: `@@flag('toDel')`});
        await db.close({table: to});
        await db.close({table: from});

--- a/server/core/DbSearcher.js
+++ b/server/core/DbSearcher.js
@@ -500,6 +500,142 @@ class DbSearcher {
        return tableIds;
    }

+    async restoreBooks(from, ids) {
+        const db = this.db;
+        const bookTable = `${from}_book`;
+
+        const rows = await db.select({
+            table: bookTable,
+            where: `@@id(${db.esc(ids)})`
+        });
+
+        if (rows.length == ids.length)
+            return rows;
+
+        const idsSet = new Set(rows.map(r => r.id));
+
+        for (const id of ids) {
+            if (!idsSet.has(id)) {
+                const bookIds = await db.select({
+                    table: from,
+                    where: `@@id(${db.esc(id)})`
+                });
+
+                if (!bookIds.length)
+                    continue;
+
+                let books = await db.select({
+                    table: 'book',
+                    where: `@@id(${db.esc(bookIds[0].bookIds)})`
+                });
+
+                if (!books.length)
+                    continue;
+
+                rows.push({id, name: bookIds[0].name, books});
+
+                await db.insert({table: bookTable, ignore: true, rows});
+            }
+        }
+
+        return rows;
+    }
+
+    async search(from, query) {
+        if (this.closed)
+            throw new Error('DbSearcher closed');
+
+        if (!['author', 'series', 'title'].includes(from))
+            throw new Error(`Unknown value for param 'from'`);
+
+        this.searchFlag++;
+
+        try {
+            const db = this.db;
+
+            const ids = await this.selectTableIds(from, query);
+
+            const totalFound = ids.length;            
+            let limit = (query.limit ? query.limit : 100);
+            limit = (limit > maxLimit ? maxLimit : limit);
+            const offset = (query.offset ? query.offset : 0);
+
+            //выборка найденных значений
+            const found = await db.select({
+                table: from,
+                map: `(r) => ({id: r.id, ${from}: r.name, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
+                where: `@@id(${db.esc(ids.slice(offset, offset + limit))})`
+            });
+
+            return {found, totalFound};
+        } finally {
+            this.searchFlag--;
+        }
+    }
+
+    async getAuthorBookList(authorId) {
+        if (this.closed)
+            throw new Error('DbSearcher closed');
+
+        if (!authorId)
+            return {author: '', books: ''};
+
+        this.searchFlag++;
+
+        try {
+            //выборка книг автора по authorId
+            const rows = await this.restoreBooks('author', [authorId])
+
+            let author = '';
+            let books = '';
+
+            if (rows.length) {
+                author = rows[0].name;
+                books = rows[0].books;
+            }
+
+            return {author, books: (books && books.length ? JSON.stringify(books) : '')};
+        } finally {
+            this.searchFlag--;
+        }
+    }
+
+    async getSeriesBookList(series) {
+        if (this.closed)
+            throw new Error('DbSearcher closed');
+
+        if (!series)
+            return {books: ''};
+
+        this.searchFlag++;
+
+        try {
+            const db = this.db;
+
+            series = series.toLowerCase();
+
+            //выборка серии по названию серии
+            let rows = await db.select({
+                table: 'series',
+                rawResult: true,
+                where: `return Array.from(@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)}))`
+            });
+
+            let books;
+            if (rows.length && rows[0].rawResult.length) {
+                //выборка книг серии
+                const rows = await this.restoreBooks('series', [rows[0].rawResult[0]])
+
+                if (rows.length)
+                    books = rows[0].books;
+            }
+
+            return {books: (books && books.length ? JSON.stringify(books) : '')};
+        } finally {
+            this.searchFlag--;
+        }
+    }
+
    async getCached(key) {
        if (!this.config.queryCacheEnabled)
            return null;
@@ -572,109 +708,6 @@ class DbSearcher {
        });
    }

-    async search(from, query) {
-        if (this.closed)
-            throw new Error('DbSearcher closed');
-
-        if (!['author', 'series', 'title'].includes(from))
-            throw new Error(`Unknown value for param 'from'`);
-
-        this.searchFlag++;
-
-        try {
-            const db = this.db;
-
-            const ids = await this.selectTableIds(from, query);
-
-            const totalFound = ids.length;            
-            let limit = (query.limit ? query.limit : 100);
-            limit = (limit > maxLimit ? maxLimit : limit);
-            const offset = (query.offset ? query.offset : 0);
-
-            //выборка найденных авторов
-            const found = await db.select({
-                table: `${from}_book`,
-                map: `(r) => ({id: r.id, ${from}: r.${from}, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
-                where: `@@id(${db.esc(ids.slice(offset, offset + limit))})`
-            });
-
-            return {found, totalFound};
-        } finally {
-            this.searchFlag--;
-        }
-    }
-
-    async getAuthorBookList(authorId) {
-        if (this.closed)
-            throw new Error('DbSearcher closed');
-
-        if (!authorId)
-            return {author: '', books: ''};
-
-        this.searchFlag++;
-
-        try {
-            const db = this.db;
-
-            //выборка книг автора по authorId
-            const rows = await db.select({
-                table: 'author_book',
-                where: `@@id(${db.esc(authorId)})`
-            });
-
-            let author = '';
-            let books = '';
-
-            if (rows.length) {
-                author = rows[0].author;
-                books = rows[0].books;
-            }
-
-            return {author, books: (books && books.length ? JSON.stringify(books) : '')};
-        } finally {
-            this.searchFlag--;
-        }
-    }
-
-    async getSeriesBookList(series) {
-        if (this.closed)
-            throw new Error('DbSearcher closed');
-
-        if (!series)
-            return {books: ''};
-
-        this.searchFlag++;
-
-        try {
-            const db = this.db;
-
-            series = series.toLowerCase();
-
-            //выборка серии по названию серии
-            let rows = await db.select({
-                table: 'series',
-                rawResult: true,
-                where: `return Array.from(@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)}))`
-            });
-
-            let books;
-            if (rows.length && rows[0].rawResult.length) {
-                //выборка книг серии
-                rows = await db.select({
-                    table: 'series_book',
-                    where: `@@id(${rows[0].rawResult[0]})`
-                });
-
-                if (rows.length)
-                    books = rows[0].books;
-            }
-
-            return {books: (books && books.length ? JSON.stringify(books) : '')};
-        } finally {
-            this.searchFlag--;
-        }
-    }
-
    async periodicCleanCache() {
        this.timer = null;
        const cleanInterval = this.config.cacheCleanInterval*60*1000;