From af3bac6a09d771ee388a479b4c9af49803459856 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 19 Oct 2022 13:30:00 +0700 Subject: [PATCH 001/133] =?UTF-8?q?=D0=92=20=D0=BA=D0=BE=D0=BD=D1=84=D0=B8?= =?UTF-8?q?=D0=B3=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=20?= =?UTF-8?q?=D0=BF=D0=B0=D1=80=D0=B0=D0=BC=D0=B5=D1=82=D1=80=20dbCacheSize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++++ server/config/base.js | 1 + server/config/index.js | 1 + server/core/WebWorker.js | 9 ++++++--- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e50e1f3..2deb4bc 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,10 @@ Options: // включить(true)/выключить(false) журналирование "loggingEnabled": true, + // максимальный размер кеша каждой таблицы в БД, в блоках (требуется примерно 1-10Мб памяти на один блок) + // если надо кешировать всю БД, можно поставить значение от 1000 и больше + "dbCacheSize": 5, + // максимальный размер в байтах директории закешированных файлов в <раб.дир>/public/files // чистка каждый час "maxFilesDirSize": 1073741824, diff --git a/server/config/base.js b/server/config/base.js index df298e8..77c0d8a 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -14,6 +14,7 @@ module.exports = { bookReadLink: '', loggingEnabled: true, + dbCacheSize: 5, maxPayloadSize: 500,//in MB maxFilesDirSize: 1024*1024*1024,//1Gb queryCacheEnabled: true, diff --git a/server/config/index.js b/server/config/index.js index 857496f..b0b73ca 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -8,6 +8,7 @@ const propsToSave = [ 'accessPassword', 'bookReadLink', 'loggingEnabled', + 'dbCacheSize', 'maxFilesDirSize', 'queryCacheEnabled', 'cacheCleanInterval', diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index 5f01f06..e3edafe 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -166,15 +166,18 @@ class WebWorker { softLock: true, tableDefaults: { - cacheSize: 5, + cacheSize: config.dbCacheSize, }, }); //открываем все таблицы await db.openAll(); + //переоткроем таблицу 'author' с бОльшим размером кеша блоков, для ускорения выборки - await db.close({table: 'author'}); - await db.open({table: 'author', cacheSize: 100}); + if (config.dbCacheSize < 100) { + await db.close({table: 'author'}); + await db.open({table: 'author', cacheSize: 100}); + } this.dbSearcher = new DbSearcher(config, db); From e0e26df5cb7096c66a02dc18925f87db38384af6 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 19 Oct 2022 13:48:01 +0700 Subject: [PATCH 002/133] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=BE=20=D0=BF=D0=BE=D1=8F=D1=81=D0=BD=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=B4=D0=BB=D1=8F=20=D0=BF=D1=83=D1=81=D1=82?= =?UTF-8?q?=D0=BE=D0=B3=D0=BE=20=D1=81=D0=BF=D0=B8=D1=81=D0=BA=D0=B0=20?= =?UTF-8?q?=D0=B2=20=D1=80=D1=83=D0=B7=D1=83=D0=BB=D1=8C=D1=82=D0=B0=D1=82?= =?UTF-8?q?=D0=B5=20=D0=BF=D0=BE=D0=B8=D1=81=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 905a8ce..28500ac 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -214,6 +214,11 @@ + +
+ + По каждому из заданных критериев у этого автора были найдены разные книги, но нет полного совпадения +
From f8416a4aec3112fb35d8f58dd77cdaa0dc016a79 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 19 Oct 2022 14:30:57 +0700 Subject: [PATCH 003/133] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=B1=D0=B0=D0=B3=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 28500ac..be49bec 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -182,7 +182,7 @@
1) + this.prevPage = this.search.page; + this.makeTitle(); this.refresh(); }, @@ -761,10 +765,8 @@ class Search { this.search.page = this.prevPage; } - if (this.search.page > this.pageCount) { - this.prevPage = this.search.page; + if (this.search.page > this.pageCount) this.search.page = 1; - } } getBookCount(item) { From a5a734a333622d5640e93fe88374a1aeed27e077 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 19 Oct 2022 15:15:57 +0700 Subject: [PATCH 004/133] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3=20=D0=B8=20=D1=83=D0=BB=D1=83?= =?UTF-8?q?=D1=87=D1=88=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BE=D1=82=D0=BE=D0=B1?= =?UTF-8?q?=D1=80=D0=B0=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B2=D1=81=D0=B5?= =?UTF-8?q?=D1=85=20=D0=BA=D0=BD=D0=B8=D0=B3=20=D1=81=D0=B5=D1=80=D0=B8?= =?UTF-8?q?=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 109 ++++++++++++++++++---------- 1 file changed, 69 insertions(+), 40 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index be49bec..fd41915 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -179,32 +179,44 @@
-
+
- +
-
+
+ + + Показать еще {{ showMoreCount }} + + + Показать все + +
+ +
Только найденные книги
-
+
Все книги серии
@@ -774,11 +786,11 @@ class Search { if (!this.showCounts || item.count === undefined) return result; - if (item.loadedBooks) { + if (item.booksLoaded) { let count = 0; - for (const book of item.loadedBooks) { + for (const book of item.booksLoaded) { if (book.type == 'series') - count += book.books.length; + count += book.seriesBooks.length; else count++; } @@ -897,12 +909,12 @@ class Search { return this.expandedSeries.indexOf(seriesItem.key) >= 0; } - isFoundSeriesBook(book, subbook) { - if (!book.booksSet) { - book.booksSet = new Set(book.books.map(b => b.id)); + isFoundSeriesBook(seriesItem, seriesBook) { + if (!seriesItem.booksSet) { + seriesItem.booksSet = new Set(seriesItem.seriesBooks.map(b => b.id)); } - return book.booksSet.has(subbook.id); + return seriesItem.booksSet.has(seriesBook.id); } setSetting(name, newValue) { @@ -1062,22 +1074,23 @@ class Search { async getSeriesBooks(seriesItem) { //асинхронно подгружаем все книги серии, блокируем повторный вызов - if (seriesItem.allBooks === null) { - seriesItem.allBooks = undefined; + if (seriesItem.allBooksLoaded === null) { + seriesItem.allBooksLoaded = undefined; (async() => { - seriesItem.allBooks = await this.loadSeriesBooks(seriesItem.series); + seriesItem.allBooksLoaded = await this.loadSeriesBooks(seriesItem.series); - if (seriesItem.allBooks) { - seriesItem.allBooks = seriesItem.allBooks.filter(book => (this.showDeleted || !book.del)); - this.sortSeriesBooks(seriesItem.allBooks); + if (seriesItem.allBooksLoaded) { + seriesItem.allBooksLoaded = seriesItem.allBooksLoaded.filter(book => (this.showDeleted || !book.del)); + this.sortSeriesBooks(seriesItem.allBooksLoaded); + this.showMoreSeries(seriesItem); } else { - seriesItem.allBooks = null; + seriesItem.allBooksLoaded = null; } })(); } } - filterBooks(loadedBooks) { + filterBooks(books) { const s = this.search; const emptyFieldValue = '?'; @@ -1125,7 +1138,7 @@ class Search { } }; - return loadedBooks.filter((book) => { + return books.filter((book) => { //author let authorFound = false; const authors = splitAuthor(book.author); @@ -1171,22 +1184,37 @@ class Search { } showMore(item, all = false) { - if (item.loadedBooks) { + if (item.booksLoaded) { const currentLen = (item.books ? item.books.length : 0); let books; if (all) { - books = item.loadedBooks; + books = item.booksLoaded; } else { - books = item.loadedBooks.slice(0, currentLen + showMoreCount); + books = item.booksLoaded.slice(0, currentLen + showMoreCount); } - item.showMore = (books.length < item.loadedBooks.length); + item.showMore = (books.length < item.booksLoaded.length); item.books = books; } } - sortSeriesBooks(books) { - books.sort((a, b) => { + showMoreSeries(seriesItem, all = false) { + if (seriesItem.allBooksLoaded) { + const currentLen = (seriesItem.allBooks ? seriesItem.allBooks.length : 0); + let books; + if (all) { + books = seriesItem.allBooksLoaded; + } else { + books = seriesItem.allBooksLoaded.slice(0, currentLen + showMoreCount); + } + + seriesItem.showMore = (books.length < seriesItem.allBooksLoaded.length); + seriesItem.allBooks = books; + } + } + + sortSeriesBooks(seriesBooks) { + seriesBooks.sort((a, b) => { const dserno = (a.serno || Number.MAX_VALUE) - (b.serno || Number.MAX_VALUE); const dtitle = a.title.localeCompare(b.title); const dext = a.ext.localeCompare(b.ext); @@ -1220,9 +1248,8 @@ class Search { })(); } - const loadedBooks = await this.loadBooks(item.key); - - const filtered = this.filterBooks(loadedBooks); + const booksToFilter = await this.loadBooks(item.key); + const filtered = this.filterBooks(booksToFilter); const prepareBook = (book) => { return Object.assign( @@ -1246,16 +1273,18 @@ class Search { key: `${item.author}-${book.series}`, type: 'series', series: book.series, + allBooksLoaded: null, allBooks: null, showAllBooks: false, + showMore: false, - books: [], + seriesBooks: [], })); seriesIndex[book.series] = index; } - books[index].books.push(prepareBook(book)); + books[index].seriesBooks.push(prepareBook(book)); } else { books.push(prepareBook(book)); } @@ -1273,7 +1302,7 @@ class Search { //сортировка внутри серий for (const book of books) { if (book.type == 'series') { - this.sortSeriesBooks(book.books); + this.sortSeriesBooks(book.seriesBooks); //асинхронно подгрузим все книги серии, если она раскрыта if (this.isExpandedSeries(book)) { @@ -1286,7 +1315,7 @@ class Search { this.expandSeries(books[0]); } - item.loadedBooks = books; + item.booksLoaded = books; this.showMore(item); await this.$nextTick(); @@ -1351,7 +1380,7 @@ class Search { author: rec.author, name: rec.author.replace(/,/g, ', '), count, - loadedBooks: false, + booksLoaded: false, books: false, bookLoading: false, showMore: false, From fba127f7cac4175a457712c454c6c2a3b90ac447 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 19 Oct 2022 16:04:41 +0700 Subject: [PATCH 005/133] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D1=81=D1=82=D0=B0=D1=82=D0=B8=D1=81=D1=82?= =?UTF-8?q?=D0=B8=D0=BA=D0=B0=20=D0=BF=D0=BE=20=D0=BA=D0=BE=D0=BB=D0=B8?= =?UTF-8?q?=D1=87=D0=B5=D1=81=D1=82=D0=B2=D1=83=20=D1=84=D0=B0=D0=B9=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=D0=BA=D0=BD=D0=B8=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 2 ++ server/core/DbCreator.js | 9 +++++++++ server/core/InpxHashCreator.js | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index fd41915..8cdd35b 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -679,6 +679,8 @@ class Search { info += `
`; info += ` +
Всего файлов книг:
${stat.filesCount}
+
Обработано ссылок на книги:
${stat.bookCountAll}
Из них актуальных:
${stat.bookCount}
Помеченных как удаленные:
${stat.bookDelCount}
diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 9b74a11..b4f41da 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -60,11 +60,14 @@ class DbCreator { let langArr = []; //stats + let filesCount = 0; let authorCount = 0; let bookCount = 0; let noAuthorBookCount = 0; let bookDelCount = 0; + let filesSet = new Set(); + //stuff let recsLoaded = 0; callback({recsLoaded}); @@ -145,6 +148,8 @@ class DbCreator { rec.id = ++id; + filesSet.add(`${rec.folder}/${rec.file}.${rec.ext}`); + if (!rec.del) { bookCount++; if (!rec.author) @@ -209,6 +214,9 @@ class DbCreator { const parser = new InpxParser(); await parser.parse(config.inpxFile, readFileCallback, parsedCallback); + filesCount = filesSet.size; + filesSet = null; + utils.freeMemory(); //отсортируем авторов и выдадим им правильные id @@ -447,6 +455,7 @@ class DbCreator { }); const stats = { + filesCount, recsLoaded, authorCount, authorCountAll: authorArr.length, diff --git a/server/core/InpxHashCreator.js b/server/core/InpxHashCreator.js index 35460da..0248615 100644 --- a/server/core/InpxHashCreator.js +++ b/server/core/InpxHashCreator.js @@ -4,7 +4,7 @@ const utils = require('./utils'); //поправить в случае, если были критические изменения в DbCreator //иначе будет рассинхронизация между сервером и клиентом на уровне БД -const dbCreatorVersion = '2'; +const dbCreatorVersion = '3'; class InpxHashCreator { constructor(config) { From e70247e4733820d950aadac10cb4ba4d20de612e Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 19 Oct 2022 16:08:25 +0700 Subject: [PATCH 006/133] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B0=20=D0=BD=D0=B0=D1=81=D1=82=D1=80=D0=BE?= =?UTF-8?q?=D0=B9=D0=BA=D0=B0=20cacheSize=20=D0=BF=D1=80=D0=B8=20=D1=81?= =?UTF-8?q?=D0=BE=D0=B7=D0=B4=D0=B0=D0=BD=D0=B8=D0=B8=20=D0=91=D0=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/WebWorker.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index e3edafe..e3e94fa 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -108,7 +108,7 @@ class WebWorker { softLock: true, tableDefaults: { - cacheSize: 5, + cacheSize: config.dbCacheSize, }, }); @@ -172,7 +172,7 @@ class WebWorker { //открываем все таблицы await db.openAll(); - + //переоткроем таблицу 'author' с бОльшим размером кеша блоков, для ускорения выборки if (config.dbCacheSize < 100) { await db.close({table: 'author'}); From 90be2a144794379f69cd7efa57d2ff40eb9bf0b9 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 19 Oct 2022 16:30:33 +0700 Subject: [PATCH 007/133] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B0=20=D1=84=D0=BE=D1=80=D0=BC=D0=B8=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=B7=D0=B0=D0=B3=D0=BE=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 8cdd35b..a18b6ca 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -588,12 +588,13 @@ class Search { const author = (as.length ? as[0] : '') + (as.length > 1 ? ' и др.' : ''); const a = correctValue(author); - const sc = correctValue(search.series); - const s = (sc ? `(${sc})` : ''); - const t = correctValue(search.title); + let s = correctValue(search.series); + s = (s ? `(${s})` : ''); + let t = correctValue(search.title); + t = (t ? `"${t}"` : ''); result = [s, t].filter(v => v).join(' '); - result = [a, result].filter(v => v).join(' - '); + result = [a, result].filter(v => v).join(' '); } this.$root.setAppTitle(result); From a556a88ee53952e5cf8c2287a6e153c2c06dbae6 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 19 Oct 2022 18:09:58 +0700 Subject: [PATCH 008/133] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20=D0=BF=D0=BE=D0=B4=D1=81=D1=87=D0=B5=D1=82=20?= =?UTF-8?q?=D1=81=D1=82=D0=B0=D1=82=D0=B8=D1=81=D1=82=D0=B8=D0=BA=D0=B8,?= =?UTF-8?q?=20=D1=81=D0=BE=D1=80=D1=82=D0=B8=D1=80=D0=BE=D0=B2=D0=BA=D0=B0?= =?UTF-8?q?=20=D1=81=D0=B5=D1=80=D0=B8=D0=B9=20=D0=B8=20=D0=BD=D0=B0=D0=B7?= =?UTF-8?q?=D0=B2=D0=B0=D0=BD=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Api/Api.vue | 3 +- server/core/DbCreator.js | 72 +++++++++++++++++++++++++++++------ 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/client/components/Api/Api.vue b/client/components/Api/Api.vue index b0b7df0..38edf02 100644 --- a/client/components/Api/Api.vue +++ b/client/components/Api/Api.vue @@ -52,7 +52,8 @@ const stepBound = [ 79,// jobStep = 9 79,// jobStep = 10 80,// jobStep = 11 - 100,// jobStep = 12 + 95,// jobStep = 12 + 100,// jobStep = 13 ]; const componentOptions = { diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index b4f41da..6391843 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -60,14 +60,11 @@ class DbCreator { let langArr = []; //stats - let filesCount = 0; let authorCount = 0; let bookCount = 0; let noAuthorBookCount = 0; let bookDelCount = 0; - let filesSet = new Set(); - //stuff let recsLoaded = 0; callback({recsLoaded}); @@ -148,8 +145,6 @@ class DbCreator { rec.id = ++id; - filesSet.add(`${rec.folder}/${rec.file}.${rec.ext}`); - if (!rec.del) { bookCount++; if (!rec.author) @@ -214,9 +209,6 @@ class DbCreator { const parser = new InpxParser(); await parser.parse(config.inpxFile, readFileCallback, parsedCallback); - filesCount = filesSet.size; - filesSet = null; - utils.freeMemory(); //отсортируем авторов и выдадим им правильные id @@ -448,14 +440,35 @@ class DbCreator { utils.freeMemory(); + //сортировка серий + callback({job: 'series sort', jobMessage: 'Сортировка серий', jobStep: 5, progress: 0}); + await utils.sleep(100); + seriesArr.sort((a, b) => a.value.localeCompare(b.value)); + await utils.sleep(100); + callback({progress: 0.3}); + id = 0; + for (const seriesRec of seriesArr) { + seriesRec.id = ++id; + } + + await utils.sleep(100); + callback({progress: 0.5}); + //заодно и названия + titleArr.sort((a, b) => a.value.localeCompare(b.value)); + await utils.sleep(100); + callback({progress: 0.7}); + id = 0; + for (const titleRec of titleArr) { + titleRec.id = ++id; + } + //config - callback({job: 'config save', jobMessage: 'Сохранение конфигурации', jobStep: 5, progress: 0}); await db.create({ table: 'config' }); const stats = { - filesCount, + filesCount: 0, recsLoaded, authorCount, authorCountAll: authorArr.length, @@ -625,10 +638,47 @@ class DbCreator { aChunk = null; } + //статистика по количеству файлов + callback({job: 'files count', jobMessage: 'Подсчет статистики', jobStep: 12, progress: 0}); + + //эмуляция прогресса + let countDone = false; + (async() => { + let i = 0; + while (!countDone) { + callback({progress: i/100}); + i = (i < 100 ? i + 5 : 100); + await utils.sleep(1000); + } + })(); + + //подчсет + const countRes = await db.select({table: 'book', count: true, where: ` + const filesSet = new Set(); + + @@iter(@all(), (r) => { + const file = ${"`${r.folder}/${r.file}.${r.ext}`"}; + if (filesSet.has(file)) { + return false; + } else { + filesSet.add(file); + return true; + } + }); + `}); + + if (countRes.length) { + stats.filesCount = countRes[0].count; + await db.insert({table: 'config', replace: true, rows: [ + {id: 'stats', value: stats}, + ]}); + } + countDone = true; + //чистка памяти, ибо жрет как не в себя - await db.drop({table: 'book'});//таблица больше не понадобится await db.drop({table: 'series_temporary'});//таблица больше не понадобится + await db.close({table: 'book'}); await db.close({table: 'series'}); await db.freeMemory(); utils.freeMemory(); From 4ca56db14253dbde02dd988bcf6dd99acbe06210 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 14:03:49 +0700 Subject: [PATCH 009/133] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbCreator.js | 88 +++++++++++++---------- server/core/DbSearcher.js | 142 ++++++++++++++++++++++++++------------ 2 files changed, 146 insertions(+), 84 deletions(-) diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 6391843..0b5686e 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -539,7 +539,7 @@ class DbCreator { //series callback({job: 'series save', jobMessage: 'Сохранение индекса серий', jobStep: 7, progress: 0}); - await saveTable('series_temporary', seriesArr, () => {seriesArr = null}, true, true); + await saveTable('series', seriesArr, () => {seriesArr = null}, true, true); //title callback({job: 'title save', jobMessage: 'Сохранение индекса названий', jobStep: 8, progress: 0}); @@ -561,21 +561,33 @@ class DbCreator { await db.create({table: 'file_hash'}); //-- завершающие шаги -------------------------------- - //оптимизация series, превращаем массив bookId в books - callback({job: 'series optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0}); - await db.open({ table: 'book', cacheSize: (config.lowMemoryMode ? 5 : 500), }); - await db.open({table: 'series_temporary'}); - await db.create({ - table: 'series', - index: {field: 'value', unique: true, depth: 1000000}, - }); - const count = await db.select({table: 'series_temporary', count: true}); - const seriesCount = (count.length ? count[0].count : 0); + callback({job: 'series optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0}); + await this.optimizeSeries(db, callback); + + callback({job: 'files count', jobMessage: 'Подсчет статистики', jobStep: 12, progress: 0}); + await this.countStats(db, callback, stats); + + //чистка памяти, ибо жрет как не в себя + await db.close({table: 'book'}); + await db.freeMemory(); + utils.freeMemory(); + + callback({job: 'done', jobMessage: ''}); + } + + async optimizeSeries(db, callback) { + //оптимизация series, превращаем массив bookId в books, кладем все в series_book + await db.open({table: 'series'}); + + await db.create({ + table: 'series_book', + flag: {name: 'toDel', check: 'r => r.toDel'}, + }); const saveSeriesChunk = async(seriesChunk) => { const ids = []; @@ -594,52 +606,62 @@ class DbCreator { bookArr.set(row.id, row); for (const s of seriesChunk) { - const sBooks = []; + s.books = []; for (const id of s.bookId) { const rec = bookArr.get(id); - sBooks.push(rec); + s.books.push(rec); + } + + if (s.books.length) { + s.series = s.books[0].value; + } else { + s.toDel = 1; } - s.books = JSON.stringify(sBooks); delete s.bookId; } await db.insert({ - table: 'series', + table: 'series_book', rows: seriesChunk, }); }; - const rows = await db.select({table: 'series_temporary'}); + const rows = await db.select({table: 'series'}); - idsLen = 0; - aChunk = []; - proc = 0; + let idsLen = 0; + let chunk = []; + let processed = 0; for (const row of rows) {// eslint-disable-line - aChunk.push(row); + chunk.push(row); idsLen += row.bookId.length; - proc++; + processed++; if (idsLen > 20000) {//константа выяснена эмпирическим путем "память/скорость" - await saveSeriesChunk(aChunk); + await saveSeriesChunk(chunk); idsLen = 0; - aChunk = []; + chunk = []; - callback({progress: proc/seriesCount}); + callback({progress: processed/rows.length}); await utils.sleep(100); utils.freeMemory(); await db.freeMemory(); } } - if (aChunk.length) { - await saveSeriesChunk(aChunk); - aChunk = null; + if (chunk.length) { + await saveSeriesChunk(chunk); + chunk = null; } + await db.delete({table: 'series_book', where: `@@flag('toDel')`}); + await db.close({table: 'series_book'}); + await db.close({table: 'series'}); + } + + async countStats(db, callback, stats) { //статистика по количеству файлов - callback({job: 'files count', jobMessage: 'Подсчет статистики', jobStep: 12, progress: 0}); //эмуляция прогресса let countDone = false; @@ -674,16 +696,6 @@ class DbCreator { ]}); } countDone = true; - - //чистка памяти, ибо жрет как не в себя - await db.drop({table: 'series_temporary'});//таблица больше не понадобится - - await db.close({table: 'book'}); - await db.close({table: 'series'}); - await db.freeMemory(); - utils.freeMemory(); - - callback({job: 'done', jobMessage: ''}); } } diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 58f685e..b034bfd 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -2,6 +2,8 @@ const utils = require('./utils'); +const maxMemCacheSize = 100; + const maxUtf8Char = String.fromCodePoint(0xFFFFF); const ruAlphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'; const enAlphabet = 'abcdefghijklmnopqrstuvwxyz'; @@ -16,6 +18,11 @@ class DbSearcher { this.timer = null; this.closed = false; + db.searchCache = { + memCache: new Map(), + authorIdsAll: false, + }; + this.periodicCleanCache();//no await } @@ -180,60 +187,82 @@ class DbSearcher { return authorIds; } - async getAuthorIds(query) { + queryKey(q) { + return JSON.stringify([q.author, q.series, q.title, q.genre, q.lang]); + } + + async getCached(key) { + if (!this.config.queryCacheEnabled) + return null; + + let result = null; + const db = this.db; + const memCache = db.searchCache.memCache; - if (!db.searchCache) - db.searchCache = {}; + if (memCache.has(key)) {//есть в недавних + result = memCache.get(key); - let result; + //изменим порядок ключей, для последующей правильной чистки старых + memCache.delete(key); + memCache.set(key, result); + } else {//смотрим в таблице + const rows = await db.select({table: 'query_cache', where: `@@id(${db.esc(key)})`}); - //сначала попробуем найти в кеше - const q = query; - const keyArr = [q.author, q.series, q.title, q.genre, q.lang]; - const keyStr = `query-${keyArr.join('')}`; - - if (!keyStr) {//пустой запрос - if (db.searchCache.authorIdsAll) - result = db.searchCache.authorIdsAll; - else - result = await this.selectAuthorIds(query); + if (rows.length) {//нашли в кеше + await db.insert({ + table: 'query_time', + replace: true, + rows: [{id: key, time: Date.now()}], + }); - } else {//непустой запрос - if (this.config.queryCacheEnabled) { - const key = JSON.stringify(keyArr); - const rows = await db.select({table: 'query_cache', where: `@@id(${db.esc(key)})`}); + result = rows[0].value; + memCache.set(key, result); - if (rows.length) {//нашли в кеше - await db.insert({ - table: 'query_time', - replace: true, - rows: [{id: key, time: Date.now()}], - }); - - result = rows[0].value; - } else {//не нашли в кеше, ищем в поисковых таблицах - result = await this.selectAuthorIds(query); - - await db.insert({ - table: 'query_cache', - replace: true, - rows: [{id: key, value: result}], - }); - await db.insert({ - table: 'query_time', - replace: true, - rows: [{id: key, time: Date.now()}], - }); + if (memCache.size > maxMemCacheSize) { + //удаляем самый старый ключ-значение + for (const k of memCache.keys()) { + memCache.delete(k); + break; + } } - } else { - result = await this.selectAuthorIds(query); } } return result; } + async putCached(key, value) { + if (!this.config.queryCacheEnabled) + return; + + const db = this.db; + + const memCache = db.searchCache.memCache; + memCache.set(key, value); + + if (memCache.size > maxMemCacheSize) { + //удаляем самый старый ключ-значение + for (const k of memCache.keys()) { + memCache.delete(k); + break; + } + } + + //кладем в таблицу + await db.insert({ + table: 'query_cache', + replace: true, + rows: [{id: key, value}], + }); + + await db.insert({ + table: 'query_time', + replace: true, + rows: [{id: key, time: Date.now()}], + }); + } + async search(query) { if (this.closed) throw new Error('DbSearcher closed'); @@ -243,7 +272,15 @@ class DbSearcher { try { const db = this.db; - const authorIds = await this.getAuthorIds(query); + const key = `author-ids-${this.queryKey(query)}`; + + //сначала попробуем найти в кеше + let authorIds = await this.getCached(key); + if (authorIds === null) {//не нашли в кеше, ищем в поисковых таблицах + authorIds = await this.selectAuthorIds(query); + + await this.putCached(key, authorIds); + } const totalFound = authorIds.length; let limit = (query.limit ? query.limit : 100); @@ -251,7 +288,7 @@ class DbSearcher { const offset = (query.offset ? query.offset : 0); //выборка найденных авторов - let result = await db.select({ + const result = await db.select({ table: 'author', map: `(r) => ({id: r.id, author: r.author, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, where: `@@id(${db.esc(authorIds.slice(offset, offset + limit))})` @@ -272,7 +309,7 @@ class DbSearcher { try { const db = this.db; - //выборка автора по authorId + //выборка книг автора по authorId const rows = await db.select({ table: 'author_book', where: `@@id(${db.esc(authorId)})` @@ -302,13 +339,26 @@ class DbSearcher { const db = this.db; series = series.toLowerCase(); + //выборка серии по названию серии - const rows = await db.select({ + let rows = await db.select({ table: 'series', where: `@@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)})` }); - return {books: (rows.length ? rows[0].books : '')}; + let books = []; + if (rows.length) { + //выборка книг серии + rows = await db.select({ + table: 'series_book', + where: `@@id(${rows[0].id})` + }); + + if (rows.length) + books = rows[0].books; + } + + return {books: (books && books.length ? JSON.stringify(books) : '')}; } finally { this.searchFlag--; } From 526683b7ef3994011e44772e7cff8bc0898c288f Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 14:05:56 +0700 Subject: [PATCH 010/133] =?UTF-8?q?=D0=9A=20=D0=BF=D1=80=D0=B5=D0=B4=D1=8B?= =?UTF-8?q?=D0=B4=D1=83=D1=89=D0=B5=D0=BC=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index b034bfd..da47dd4 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -346,7 +346,7 @@ class DbSearcher { where: `@@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)})` }); - let books = []; + let books; if (rows.length) { //выборка книг серии rows = await db.select({ From 88d49852e0af697879000a1c57183362127877ac Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 14:16:44 +0700 Subject: [PATCH 011/133] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index da47dd4..43b9aa6 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -55,7 +55,7 @@ class DbSearcher { async selectAuthorIds(query) { const db = this.db; - let authorIds = new Set(); + let authorIds = []; //сначала выберем все id авторов по фильтру //порядок id соответсвует ASC-сортировке по author @@ -69,7 +69,7 @@ class DbSearcher { }); for (const row of authorRows) - authorIds.add(row.id); + authorIds.push(row.id); } else {//все авторы if (!db.searchCache.authorIdsAll) { const authorRows = await db.select({ @@ -77,18 +77,17 @@ class DbSearcher { dirtyIdsOnly: true, }); - db.searchCache.authorIdsAll = []; for (const row of authorRows) { - authorIds.add(row.id); - db.searchCache.authorIdsAll.push(row.id); + authorIds.push(row.id); } + + db.searchCache.authorIdsAll = authorIds; } else {//оптимизация - authorIds = new Set(db.searchCache.authorIdsAll); + authorIds = db.searchCache.authorIdsAll; } } const idsArr = []; - idsArr.push(authorIds); //серии if (query.series && query.series !== '*') { @@ -176,12 +175,13 @@ class DbSearcher { idsArr.push(ids); } - if (idsArr.length > 1) - authorIds = utils.intersectSet(idsArr); + if (idsArr.length) { + //ищем пересечение множеств + idsArr.push(new Set(authorIds)); + authorIds = Array.from(utils.intersectSet(idsArr)); + } //сортировка - authorIds = Array.from(authorIds); - authorIds.sort((a, b) => a - b); return authorIds; From 78dc1c87c87639b54df865c39690a04a7bc7f191 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 14:27:58 +0700 Subject: [PATCH 012/133] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 43b9aa6..589323b 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -18,7 +18,7 @@ class DbSearcher { this.timer = null; this.closed = false; - db.searchCache = { + this.searchCache = { memCache: new Map(), authorIdsAll: false, }; @@ -71,7 +71,7 @@ class DbSearcher { for (const row of authorRows) authorIds.push(row.id); } else {//все авторы - if (!db.searchCache.authorIdsAll) { + if (!this.searchCache.authorIdsAll) { const authorRows = await db.select({ table: 'author', dirtyIdsOnly: true, @@ -81,9 +81,9 @@ class DbSearcher { authorIds.push(row.id); } - db.searchCache.authorIdsAll = authorIds; + this.searchCache.authorIdsAll = authorIds; } else {//оптимизация - authorIds = db.searchCache.authorIdsAll; + authorIds = this.searchCache.authorIdsAll; } } @@ -198,7 +198,7 @@ class DbSearcher { let result = null; const db = this.db; - const memCache = db.searchCache.memCache; + const memCache = this.searchCache.memCache; if (memCache.has(key)) {//есть в недавних result = memCache.get(key); @@ -238,7 +238,7 @@ class DbSearcher { const db = this.db; - const memCache = db.searchCache.memCache; + const memCache = this.searchCache.memCache; memCache.set(key, value); if (memCache.size > maxMemCacheSize) { @@ -406,6 +406,8 @@ class DbSearcher { await utils.sleep(50); } + this.searchCache = null; + if (this.timer) { clearTimeout(this.timer); this.timer = null; From 31d5fc6df38830b23614a987970ac1c90101ffeb Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 15:09:57 +0700 Subject: [PATCH 013/133] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BE=D1=82=D0=BE=D0=B1=D1=80=D0=B0=D0=B6?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20=D1=81=D0=BF=D0=B8=D1=81=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index a18b6ca..abf8fbe 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -205,10 +205,10 @@
- Показать еще {{ showMoreCount }} + Показать еще ({{ showMoreCount }}) - Показать все + Показать все ({{ (book.allBooksLoaded && book.allBooksLoaded.length) || '?' }})
@@ -236,10 +236,10 @@
- Показать еще {{ showMoreCount }} + Показать еще ({{ showMoreCount }}) - Показать все + Показать все ({{ (item.booksLoaded && item.booksLoaded.length) || '?' }})
From 1aaa27e68b92d522c7d7169a755f487179450fe6 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 15:51:22 +0700 Subject: [PATCH 014/133] jembadb update --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index fd08b94..96525d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "compression": "^1.7.4", "express": "^4.18.1", "fs-extra": "^10.1.0", - "jembadb": "^4.2.0", + "jembadb": "^4.3.0", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", @@ -5077,9 +5077,9 @@ } }, "node_modules/jembadb": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-4.2.0.tgz", - "integrity": "sha512-wnqUwaZSWU99hJYHPBhXJVRYHA1aQVjpt5fDHMuXaz7VWZqK9DhLgNDIKD9z8czICz56ECTR2xlVBpDgBnuQVA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-4.3.0.tgz", + "integrity": "sha512-DJ1w+sbZ1lk7jd39fkn6sz86d6QR/y+rcGYZ/XTdoiG/bWsyr8/mbSDmecbI5+MVca2KOFA1BjKYJ51cPp2LJg==", "engines": { "node": ">=16.16.0" } @@ -12578,9 +12578,9 @@ "dev": true }, "jembadb": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-4.2.0.tgz", - "integrity": "sha512-wnqUwaZSWU99hJYHPBhXJVRYHA1aQVjpt5fDHMuXaz7VWZqK9DhLgNDIKD9z8czICz56ECTR2xlVBpDgBnuQVA==" + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-4.3.0.tgz", + "integrity": "sha512-DJ1w+sbZ1lk7jd39fkn6sz86d6QR/y+rcGYZ/XTdoiG/bWsyr8/mbSDmecbI5+MVca2KOFA1BjKYJ51cPp2LJg==" }, "jest-worker": { "version": "27.5.1", diff --git a/package.json b/package.json index 56f02aa..bfe2684 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "compression": "^1.7.4", "express": "^4.18.1", "fs-extra": "^10.1.0", - "jembadb": "^4.2.0", + "jembadb": "^4.3.0", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", From 3204946fc8cc3616251edfe81d3dffd2a2df11ee Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 16:05:09 +0700 Subject: [PATCH 015/133] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/WebWorker.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index e3e94fa..a34a500 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -170,14 +170,11 @@ class WebWorker { }, }); - //открываем все таблицы - await db.openAll(); + //открываем почти все таблицы + await db.openAll({exclude: ['author']}); - //переоткроем таблицу 'author' с бОльшим размером кеша блоков, для ускорения выборки - if (config.dbCacheSize < 100) { - await db.close({table: 'author'}); - await db.open({table: 'author', cacheSize: 100}); - } + //откроем таблицу 'author' с бОльшим размером кеша блоков, для ускорения выборки + await db.open({table: 'author', cacheSize: (config.dbCacheSize > 100 ? config.dbCacheSize : 100)}); this.dbSearcher = new DbSearcher(config, db); From 2d8dc8c04b0017928e29202c423730a67552b03f Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 16:47:21 +0700 Subject: [PATCH 016/133] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BE=D1=82=D0=BE=D0=B1=D1=80=D0=B0=D0=B6?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20BookView?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/Search/BookView/BookView.vue | 23 ++++++++++++------- client/components/Search/Search.vue | 9 +++++++- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/client/components/Search/BookView/BookView.vue b/client/components/Search/BookView/BookView.vue index 89e4a32..0492a1d 100644 --- a/client/components/Search/BookView/BookView.vue +++ b/client/components/Search/BookView/BookView.vue @@ -31,9 +31,14 @@
-
+
{{ book.serno ? `${book.serno}. ` : '' }} - {{ bookTitle }} +
+ {{ bookAuthor }} +  -  + {{ book.title }} +
+ {{ book.title }}
@@ -106,15 +111,13 @@ class BookView { return this.$store.state.settings; } - get bookTitle() { + get bookAuthor() { if (this.showAuthor && this.book.author) { let a = this.book.author.split(','); - const author = a.slice(0, 2).join(', ') + (a.length > 2 ? ' и др.' : ''); - - return `${author} - ${this.book.title}`; - } else { - return this.book.title; + return a.slice(0, 2).join(', ') + (a.length > 2 ? ' и др.' : ''); } + + return ''; } get bookSize() { @@ -148,6 +151,10 @@ class BookView { return `(${result.join(' / ')})`; } + selectAuthor() { + this.$emit('bookEvent', {action: 'authorClick', book: this.book}); + } + selectTitle() { this.$emit('bookEvent', {action: 'titleClick', book: this.book}); } diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index abf8fbe..ea3ebe0 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -814,6 +814,10 @@ class Search { this.search.series = `=${series}`; } + selectTitle(title) { + this.search.title = `=${title}`; + } + async download(book, action) { if (this.downloadFlag) return; @@ -893,8 +897,11 @@ class Search { bookEvent(event) { switch (event.action) { + case 'authorClick': + this.selectAuthor(event.book.author); + break; case 'titleClick': - this.search.title = `=${event.book.title}`; + this.selectTitle(event.book.title); break; case 'download': case 'copyLink': From 11a6c0699709f83afc5158ece82492356116b2c1 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 16:50:16 +0700 Subject: [PATCH 017/133] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbCreator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 0b5686e..3ffdcff 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -573,7 +573,7 @@ class DbCreator { await this.countStats(db, callback, stats); //чистка памяти, ибо жрет как не в себя - await db.close({table: 'book'}); + await db.drop({table: 'book'});//больше не понадобится await db.freeMemory(); utils.freeMemory(); From ba3d6adac690a6711f292585538474cdb68a5efc Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 18:03:47 +0700 Subject: [PATCH 018/133] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BF=D0=BE=D0=B4=D1=85=D0=B2=D0=B0=D1=82?= =?UTF-8?q?=D0=B0=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD=D0=B8=D0=B9?= =?UTF-8?q?=20inpx-=D1=84=D0=B0=D0=B9=D0=BB=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/WebWorker.js | 34 ++++++++++++++++++++++++++++------ server/index.js | 2 +- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index a34a500..58dac7a 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -5,7 +5,7 @@ const _ = require('lodash'); const ZipReader = require('./ZipReader'); const WorkerState = require('./WorkerState');//singleton -const { JembaDbThread } = require('jembadb'); +const { JembaDb, JembaDbThread } = require('jembadb'); const DbCreator = require('./DbCreator'); const DbSearcher = require('./DbSearcher'); const InpxHashCreator = require('./InpxHashCreator'); @@ -141,10 +141,33 @@ class WebWorker { this.inpxFileHash = await this.inpxHashCreator.getInpxFileHash(); - //пересоздаем БД из INPX если нужно + //проверим полный InxpHash (включая фильтр и версию БД) + //для этого заглянем в конфиг внутри БД, если он есть + if (!(config.recreateDb || recreate) && await fs.pathExists(dbPath)) { + const newInpxHash = await this.inpxHashCreator.getHash(); + + const tmpDb = new JembaDb(); + await tmpDb.lock({dbPath, softLock: true}); + + try { + await tmpDb.open({table: 'config'}); + const rows = await tmpDb.select({table: 'config', where: `@@id('inpxHash')`}); + + if (!rows.length || newInpxHash !== rows[0].value) + throw new Error('inpx file: changes found on start, recreating DB'); + } catch (e) { + log(LM_WARN, e.message); + recreate = true; + } finally { + await tmpDb.unlock(); + } + } + + //удалим БД если нужно if (config.recreateDb || recreate) await fs.remove(dbPath); + //пересоздаем БД из INPX если нужно if (!await fs.pathExists(dbPath)) { try { await this.createDb(dbPath); @@ -160,7 +183,7 @@ class WebWorker { this.setMyState(ssDbLoading); log('Searcher DB loading'); - const db = new JembaDbThread(); + const db = new JembaDbThread();//в отдельном потоке await db.lock({ dbPath, softLock: true, @@ -181,14 +204,13 @@ class WebWorker { db.wwCache = {}; this.db = db; - log('Searcher DB ready'); + this.setMyState(ssNormal); + log('Searcher DB ready'); this.logServerStats(); } catch (e) { log(LM_FATAL, e.message); ayncExit.exit(1); - } finally { - this.setMyState(ssNormal); } } diff --git a/server/index.js b/server/index.js index bcf3442..4d26301 100644 --- a/server/index.js +++ b/server/index.js @@ -114,7 +114,7 @@ async function init() { } } } else { - config.inpxFile = `${config.tempDir}/${utils.randomHexString(20)}`; + config.inpxFile = `${config.dataDir}/remote.inpx`; const RemoteLib = require('./core/RemoteLib');//singleton const remoteLib = new RemoteLib(config); await remoteLib.downloadInpxFile(); From 8e7fda9fbf69918b59028d3de4a8a7f267d4b1cc Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 20:31:07 +0700 Subject: [PATCH 019/133] =?UTF-8?q?=D0=A3=D0=B4=D0=B0=D0=BB=D0=B8=D0=BB=20?= =?UTF-8?q?=D0=BD=D0=B5=D0=BD=D1=83=D0=B6=D0=BD=D1=8B=D0=B9=20=D0=BC=D0=BE?= =?UTF-8?q?=D0=B4=D1=83=D0=BB=D1=8C=20compression?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 115 ---------------------------------------------- package.json | 1 - server/index.js | 3 -- 3 files changed, 119 deletions(-) diff --git a/package-lock.json b/package-lock.json index 96525d4..23a401c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "dependencies": { "@quasar/extras": "^1.15.0", "axios": "^0.27.2", - "compression": "^1.7.4", "express": "^4.18.1", "fs-extra": "^10.1.0", "jembadb": "^4.3.0", @@ -2736,14 +2735,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, - "node_modules/bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -2954,52 +2945,6 @@ "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", "dev": true }, - "node_modules/compressible": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", - "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", - "dependencies": { - "mime-db": ">= 1.43.0 < 2" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", - "dependencies": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", - "debug": "2.6.9", - "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/compression/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/compression/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" - }, - "node_modules/compression/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -5811,14 +5756,6 @@ "node": ">= 0.8" } }, - "node_modules/on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -10832,11 +10769,6 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, - "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==" - }, "call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -11006,48 +10938,6 @@ "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", "dev": true }, - "compressible": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", - "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", - "requires": { - "mime-db": ">= 1.43.0 < 2" - } - }, - "compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", - "requires": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", - "debug": "2.6.9", - "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", - "vary": "~1.1.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - } - } - }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -13118,11 +13008,6 @@ "ee-first": "1.1.1" } }, - "on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==" - }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", diff --git a/package.json b/package.json index bfe2684..c38702a 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,6 @@ "dependencies": { "@quasar/extras": "^1.15.0", "axios": "^0.27.2", - "compression": "^1.7.4", "express": "^4.18.1", "fs-extra": "^10.1.0", "jembadb": "^4.3.0", diff --git a/server/index.js b/server/index.js index 4d26301..b7909d3 100644 --- a/server/index.js +++ b/server/index.js @@ -2,7 +2,6 @@ const fs = require('fs-extra'); const path = require('path'); const express = require('express'); -const compression = require('compression'); const http = require('http'); const WebSocket = require ('ws'); @@ -147,8 +146,6 @@ async function main() { devModule.webpackDevMiddleware(app); } - app.use(compression({ level: 1 })); - //app.use(express.json({limit: `${config.maxPayloadSize}mb`})); if (devModule) devModule.logQueries(app); From f7297ac573114409dcfa79dea1b4a65117994830 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 20:34:21 +0700 Subject: [PATCH 020/133] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B8=D0=B9=20?= =?UTF-8?q?=D1=80=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE=D1=80=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/index.js b/server/index.js index b7909d3..d94217e 100644 --- a/server/index.js +++ b/server/index.js @@ -223,10 +223,11 @@ function initStatic(app, config) { await init(); await main(); } catch (e) { + const mes = (branch == 'development' ? e.stack : e.message); if (log) - log(LM_FATAL, (branch == 'development' ? e.stack : e.message)); + log(LM_FATAL, mes); else - console.error(branch == 'development' ? e.stack : e.message); + console.error(mes); ayncExit.exit(1); } From 8c1ec1dc93f755bbd7dcc75a59623220729fa2c9 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 21:46:05 +0700 Subject: [PATCH 021/133] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BE=D1=82=D0=B4=D0=B0=D1=87=D0=B8=20?= =?UTF-8?q?=D1=84=D0=B0=D0=B9=D0=BB=D0=BE=D0=B2=20=D0=BA=D0=BD=D0=B8=D0=B3?= =?UTF-8?q?=20=D0=B8=20=D1=81=D1=82=D0=B0=D1=82=D0=B8=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/WebWorker.js | 30 +++++++++++++--------- server/createWebApp.js | 13 ---------- server/index.js | 54 +++++++++++++++++++++++++--------------- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index 58dac7a..8591b7f 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -58,8 +58,8 @@ class WebWorker { const dirConfig = [ { - dir: `${this.config.publicDir}/files`, - maxSize: this.config.maxFilesDirSize, + dir: config.filesDir, + maxSize: config.maxFilesDirSize, }, ]; @@ -358,20 +358,25 @@ class WebWorker { hash = await this.remoteLib.downloadBook(bookPath, downFileName); } - const link = `/files/${hash}`; - const publicPath = `${this.config.publicDir}${link}`; + const link = `${this.config.filesPathStatic}/${hash}`; + const bookFile = `${this.config.filesDir}/${hash}`; + const bookFileDesc = `${bookFile}.json`; - if (!await fs.pathExists(publicPath)) { - await fs.ensureDir(path.dirname(publicPath)); + if (!await fs.pathExists(bookFile) || !await fs.pathExists(bookFileDesc)) { + await fs.ensureDir(path.dirname(bookFile)); const tmpFile = `${this.config.tempDir}/${utils.randomHexString(30)}`; await utils.gzipFile(extractedFile, tmpFile, 4); await fs.remove(extractedFile); - await fs.move(tmpFile, publicPath, {overwrite: true}); + await fs.move(tmpFile, bookFile, {overwrite: true}); + + await fs.writeFile(bookFileDesc, JSON.stringify({bookPath, downFileName})); } else { if (extractedFile) await fs.remove(extractedFile); - await utils.touchFile(publicPath); + + await utils.touchFile(bookFile); + await utils.touchFile(bookFileDesc); } await db.insert({ @@ -399,11 +404,10 @@ class WebWorker { const rows = await db.select({table: 'file_hash', where: `@@id(${db.esc(bookPath)})`}); if (rows.length) {//хеш найден по bookPath const hash = rows[0].hash; - link = `/files/${hash}`; - const publicPath = `${this.config.publicDir}${link}`; + const bookFileDesc = `${this.config.filesDir}/${hash}.json`; - if (!await fs.pathExists(publicPath)) { - link = ''; + if (await fs.pathExists(bookFileDesc)) { + link = `${this.config.filesPathStatic}/${hash}`; } } @@ -423,6 +427,7 @@ class WebWorker { } } + /* async restoreBookFile(publicPath) { this.checkMyState(); @@ -462,6 +467,7 @@ class WebWorker { throw new Error('404 Файл не найден'); } } + */ async getInpxFile(params) { let data = null; diff --git a/server/createWebApp.js b/server/createWebApp.js index 705f18c..cc3c85b 100644 --- a/server/createWebApp.js +++ b/server/createWebApp.js @@ -13,15 +13,6 @@ module.exports = async(config) => { return; } - //сохраним files - const filesDir = `${config.publicDir}/files`; - let tmpFilesDir = ''; - if (await fs.pathExists(filesDir)) { - tmpFilesDir = `${config.dataDir}/files`; - if (!await fs.pathExists(tmpFilesDir)) - await fs.move(filesDir, tmpFilesDir); - } - await fs.remove(config.publicDir); //извлекаем новый webApp @@ -35,10 +26,6 @@ module.exports = async(config) => { await zipReader.close(); } - //восстановим files - if (tmpFilesDir) - await fs.move(tmpFilesDir, filesDir); - await fs.writeFile(verFile, config.version); await fs.remove(zipFile); }; \ No newline at end of file diff --git a/server/index.js b/server/index.js index d94217e..99383c1 100644 --- a/server/index.js +++ b/server/index.js @@ -49,9 +49,14 @@ async function init() { config.tempDir = `${config.dataDir}/tmp`; config.logDir = `${config.dataDir}/log`; config.publicDir = `${config.dataDir}/public`; + config.publicFilesDir = `${config.dataDir}/public-files`; + config.filesPathStatic = `/book`; + config.filesDir = `${config.publicFilesDir}${config.filesPathStatic}`; + configManager.config = config; await fs.ensureDir(config.dataDir); + await fs.ensureDir(config.filesDir); await fs.ensureDir(config.tempDir); await fs.emptyDir(config.tempDir); @@ -170,29 +175,42 @@ async function main() { } function initStatic(app, config) { - const WebWorker = require('./core/WebWorker');//singleton - const webWorker = new WebWorker(config); - + /* + publicFilesDir = `${config.dataDir}/public-files`; + filesPathStatic = `/book`; + filesDir = `${config.publicFilesDir}${config.filesPathStatic}`; + */ + const filesPath = `${config.filesPathStatic}/`; //загрузка или восстановление файлов в /files, при необходимости app.use(async(req, res, next) => { if ((req.method !== 'GET' && req.method !== 'HEAD') || - !(req.path.indexOf('/files/') === 0) + !(req.path.indexOf(filesPath) === 0) ) { return next(); } - const publicPath = `${config.publicDir}${req.path}`; + if (path.extname(req.path) == '.json') + return next(); + + const bookFile = `${config.publicFilesDir}${req.path}`; + const bookFileDesc = `${bookFile}.json`; let downFileName = ''; - //восстановим + //восстановим из json-файла описания try { - if (!await fs.pathExists(publicPath)) { - downFileName = await webWorker.restoreBookFile(publicPath); + if (await fs.pathExists(bookFile) && await fs.pathExists(bookFileDesc)) { + await utils.touchFile(bookFile); + await utils.touchFile(bookFileDesc); + + let desc = await fs.readFile(bookFileDesc, 'utf8'); + desc = JSON.parse(desc); + downFileName = desc.downFileName; } else { - downFileName = await webWorker.getDownFileName(publicPath); + await fs.remove(bookFile); + await fs.remove(bookFileDesc); } } catch(e) { - //quiet + log(LM_ERR, e.message); } if (downFileName) @@ -202,20 +220,16 @@ function initStatic(app, config) { }); //заголовки при отдаче - const filesDir = utils.toUnixPath(`${config.publicDir}/files`); - app.use(express.static(config.publicDir, { - setHeaders: (res, filePath) => { - //res.set('Cache-Control', 'no-cache'); - //res.set('Expires', '-1'); - - if (utils.toUnixPath(path.dirname(filePath)) == filesDir) { + app.use(config.filesPathStatic, express.static(config.filesDir, { + setHeaders: (res) => { + if (res.downFileName) { res.set('Content-Encoding', 'gzip'); - - if (res.downFileName) - res.set('Content-Disposition', `inline; filename*=UTF-8''${encodeURIComponent(res.downFileName)}`); + res.set('Content-Disposition', `inline; filename*=UTF-8''${encodeURIComponent(res.downFileName)}`); } }, })); + + app.use(express.static(config.publicDir)); } (async() => { From 7a58b5a3ba1bac77af4ed6c7ec5131734f7a62ca Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 21:57:10 +0700 Subject: [PATCH 022/133] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B8=D0=B9=20?= =?UTF-8?q?=D1=80=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE=D1=80=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbCreator.js | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 3ffdcff..998bc61 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -462,11 +462,7 @@ class DbCreator { titleRec.id = ++id; } - //config - await db.create({ - table: 'config' - }); - + //stats const stats = { filesCount: 0, recsLoaded, @@ -483,14 +479,6 @@ class DbCreator { }; //console.log(stats); - const inpxHashCreator = new InpxHashCreator(config); - - await db.insert({table: 'config', rows: [ - {id: 'inpxInfo', value: (inpxFilter && inpxFilter.info ? inpxFilter.info : parser.info)}, - {id: 'stats', value: stats}, - {id: 'inpxHash', value: await inpxHashCreator.getHash()}, - ]}); - //сохраним поисковые таблицы const chunkSize = 10000; @@ -577,6 +565,19 @@ class DbCreator { await db.freeMemory(); utils.freeMemory(); + //config сохраняем в самом конце, нет конфига - с базой что-то не так + const inpxHashCreator = new InpxHashCreator(config); + + await db.create({ + table: 'config' + }); + + await db.insert({table: 'config', rows: [ + {id: 'inpxInfo', value: (inpxFilter && inpxFilter.info ? inpxFilter.info : parser.info)}, + {id: 'stats', value: stats}, + {id: 'inpxHash', value: await inpxHashCreator.getHash()}, + ]}); + callback({job: 'done', jobMessage: ''}); } @@ -689,12 +690,9 @@ class DbCreator { }); `}); - if (countRes.length) { + if (countRes.length) stats.filesCount = countRes[0].count; - await db.insert({table: 'config', replace: true, rows: [ - {id: 'stats', value: stats}, - ]}); - } + countDone = true; } } From 48ebff923ef005ffe9fb9396849cae4458fe50b2 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 20 Oct 2022 22:02:37 +0700 Subject: [PATCH 023/133] 1.0.7 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 23a401c..a22f7be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "inpx-web", - "version": "1.0.6", + "version": "1.0.7", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "inpx-web", - "version": "1.0.6", + "version": "1.0.7", "hasInstallScript": true, "license": "CC0-1.0", "dependencies": { diff --git a/package.json b/package.json index c38702a..4df3615 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "inpx-web", - "version": "1.0.6", + "version": "1.0.7", "author": "Book Pauk ", "license": "CC0-1.0", "repository": "bookpauk/inpx-web", From 8b67cd09adb594e2d2e76ea201e27b7821d45e39 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 12:41:35 +0700 Subject: [PATCH 024/133] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BE=D1=82=D0=BE=D0=B1=D1=80=D0=B0=D0=B6?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20=D1=81=D0=BF=D0=B8=D1=81=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index ea3ebe0..32cf1b8 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -205,7 +205,7 @@
- Показать еще ({{ showMoreCount }}) + Показать еще (~{{ showMoreCount }}) Показать все ({{ (book.allBooksLoaded && book.allBooksLoaded.length) || '?' }}) @@ -236,7 +236,7 @@
- Показать еще ({{ showMoreCount }}) + Показать еще (~{{ showMoreCount }}) Показать все ({{ (item.booksLoaded && item.booksLoaded.length) || '?' }}) @@ -1197,7 +1197,7 @@ class Search { if (item.booksLoaded) { const currentLen = (item.books ? item.books.length : 0); let books; - if (all) { + if (all || currentLen + showMoreCount*1.5 > item.booksLoaded.length) { books = item.booksLoaded; } else { books = item.booksLoaded.slice(0, currentLen + showMoreCount); @@ -1212,7 +1212,7 @@ class Search { if (seriesItem.allBooksLoaded) { const currentLen = (seriesItem.allBooks ? seriesItem.allBooks.length : 0); let books; - if (all) { + if (all || currentLen + showMoreCount*1.5 > seriesItem.allBooksLoaded.length) { books = seriesItem.allBooksLoaded; } else { books = seriesItem.allBooksLoaded.slice(0, currentLen + showMoreCount); From 1490fc854ad3c2472741cc6b1ea3fd2a90b6e58d Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 13:27:28 +0700 Subject: [PATCH 025/133] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=BE=20=D0=BF=D0=BE=D0=B4=D1=81=D0=B2=D0=B5=D1=87?= =?UTF-8?q?=D0=B8=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20=D0=BD=D0=BE=D0=BC=D0=B5?= =?UTF-8?q?=D1=80=D0=B0=20=D1=81=D1=82=D1=80=D0=B0=D0=BD=D0=B8=D1=86=D1=8B?= =?UTF-8?q?,=20=D0=B5=D1=81=D0=BB=D0=B8=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=BB=D0=B8=D1=81=D1=8C=20=D0=BA=D1=80=D0=B8=D1=82=D0=B5?= =?UTF-8?q?=D1=80=D0=B8=D0=B8=20=D0=BF=D0=BE=D0=B8=D1=81=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Search/PageScroller/PageScroller.vue | 20 ++++++++++++++++++- client/components/Search/Search.vue | 19 +++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/client/components/Search/PageScroller/PageScroller.vue b/client/components/Search/PageScroller/PageScroller.vue index b873800..566abdd 100644 --- a/client/components/Search/PageScroller/PageScroller.vue +++ b/client/components/Search/PageScroller/PageScroller.vue @@ -3,7 +3,7 @@
Страница
-
+
\ No newline at end of file diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 32cf1b8..196fb8a 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -125,7 +125,7 @@
- +
@@ -941,6 +941,22 @@ class Search { }); } + highlightPageScroller(query) { + const q = _.cloneDeep(query); + delete q.limit; + delete q.offset; + delete q.page; + + try { + if (this.search.page < 2 || !this._prevQuery || _.isEqual(this._prevQuery, q)) + return; + + this.$refs.pageScroller1.highlightScroller(); + } finally { + this._prevQuery = q; + } + } + async updateSearchFromRouteQuery(to) { if (this.liberamaReady) this.sendCurrentUrl(); @@ -1477,6 +1493,7 @@ class Search { await this.updateGenreTreeIfNeeded(); await this.updateTableData(); this.scrollToTop(); + this.highlightPageScroller(query); } } catch (e) { this.$root.stdDialog.alert(e.message, 'Ошибка'); From 294fb35f4d6483d07a7d61fb8d20927aeb41ed08 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 13:47:10 +0700 Subject: [PATCH 026/133] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BA?= =?UTF-8?q?=D0=B8=20=D0=BA=D0=BE=D0=BD=D1=84=D0=B8=D0=B3=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/config/index.js | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/server/config/index.js b/server/config/index.js index b0b73ca..ee0aef5 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -85,13 +85,22 @@ class ConfigManager { async load() { if (!this.inited) throw new Error('not inited'); - if (!await fs.pathExists(this.userConfigFile)) { - await this.save(); - return; - } - const data = await fs.readFile(this.userConfigFile, 'utf8'); - this.config = JSON.parse(data); + if (await fs.pathExists(this.userConfigFile)) { + const data = JSON.parse(await fs.readFile(this.userConfigFile, 'utf8')); + const config = _.pick(data, propsToSave); + + this.config = config; + + //сохраним конфиг, если не все атрибуты присутствуют в файле конфига + for (const prop of propsToSave) + if (!Object.prototype.hasOwnProperty.call(config, prop)) { + await this.save(); + break; + } + } else { + await this.save(); + } } async save() { From e865070f233ab736e998a04c4c1dff894fe2d4c1 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 13:50:55 +0700 Subject: [PATCH 027/133] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BE=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/config/index.js | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/server/config/index.js b/server/config/index.js index ee0aef5..cf98140 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -83,23 +83,27 @@ class ConfigManager { } async load() { - if (!this.inited) - throw new Error('not inited'); + try { + if (!this.inited) + throw new Error('not inited'); - if (await fs.pathExists(this.userConfigFile)) { - const data = JSON.parse(await fs.readFile(this.userConfigFile, 'utf8')); - const config = _.pick(data, propsToSave); - - this.config = config; + if (await fs.pathExists(this.userConfigFile)) { + const data = JSON.parse(await fs.readFile(this.userConfigFile, 'utf8')); + const config = _.pick(data, propsToSave); - //сохраним конфиг, если не все атрибуты присутствуют в файле конфига - for (const prop of propsToSave) - if (!Object.prototype.hasOwnProperty.call(config, prop)) { - await this.save(); - break; - } - } else { - await this.save(); + this.config = config; + + //сохраним конфиг, если не все атрибуты присутствуют в файле конфига + for (const prop of propsToSave) + if (!Object.prototype.hasOwnProperty.call(config, prop)) { + await this.save(); + break; + } + } else { + await this.save(); + } + } catch(e) { + throw new Error(`Error while loading "${this.userConfigFile}": ${e.message}`); } } From d5a9b77334ea6cadc5f4505ad1f9376c1c6274b1 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 17:30:36 +0700 Subject: [PATCH 028/133] "jembadb": "^5.0.1" --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index a22f7be..9428d45 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "axios": "^0.27.2", "express": "^4.18.1", "fs-extra": "^10.1.0", - "jembadb": "^4.3.0", + "jembadb": "^5.0.1", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", @@ -5022,9 +5022,9 @@ } }, "node_modules/jembadb": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-4.3.0.tgz", - "integrity": "sha512-DJ1w+sbZ1lk7jd39fkn6sz86d6QR/y+rcGYZ/XTdoiG/bWsyr8/mbSDmecbI5+MVca2KOFA1BjKYJ51cPp2LJg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.1.tgz", + "integrity": "sha512-fOKfUK8vOQJDtugJAYCllfQiYD3G0GWKNhOMOLXF8jkOcgnsCxQRxAVDIM/K5s07q1I96DIQcsWFBBDfYA/heA==", "engines": { "node": ">=16.16.0" } @@ -12468,9 +12468,9 @@ "dev": true }, "jembadb": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-4.3.0.tgz", - "integrity": "sha512-DJ1w+sbZ1lk7jd39fkn6sz86d6QR/y+rcGYZ/XTdoiG/bWsyr8/mbSDmecbI5+MVca2KOFA1BjKYJ51cPp2LJg==" + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.1.tgz", + "integrity": "sha512-fOKfUK8vOQJDtugJAYCllfQiYD3G0GWKNhOMOLXF8jkOcgnsCxQRxAVDIM/K5s07q1I96DIQcsWFBBDfYA/heA==" }, "jest-worker": { "version": "27.5.1", diff --git a/package.json b/package.json index 4df3615..c5d85b9 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "axios": "^0.27.2", "express": "^4.18.1", "fs-extra": "^10.1.0", - "jembadb": "^4.3.0", + "jembadb": "^5.0.1", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", From 87d418a32a4756a1e957747a72ff8daf52463724 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 18:09:52 +0700 Subject: [PATCH 029/133] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20=D0=BA=D0=BE=D0=BD=D1=81=D0=BE=D0=BB=D1=8C=D0=BD?= =?UTF-8?q?=D1=8B=D0=B9=20=D0=BB=D0=BE=D0=B3=20=D0=B2=20=D1=80=D0=B5=D0=B6?= =?UTF-8?q?=D0=B8=D0=BC=D0=B5=20loggingEnabled:=20false?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/AppLogger.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/core/AppLogger.js b/server/core/AppLogger.js index 68c8e1f..1e1f27b 100644 --- a/server/core/AppLogger.js +++ b/server/core/AppLogger.js @@ -37,6 +37,10 @@ class AppLogger { {log: 'FileLog', fileName: this.errLogFileName, exclude: [LM_OK, LM_INFO, LM_TOTAL]}, {log: 'FileLog', fileName: this.fatalLogFileName, exclude: [LM_OK, LM_INFO, LM_WARN, LM_ERR, LM_TOTAL]},//LM_FATAL only ]; + } else { + loggerParams = [ + {log: 'ConsoleLog'}, + ]; } this._logger = new Logger(loggerParams); From 46829a9b08af138e73adfea8ddecd42f540f89c0 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 18:21:30 +0700 Subject: [PATCH 030/133] =?UTF-8?q?=D0=90=D0=B4=D0=B0=D0=BF=D1=82=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D1=8F=20=D0=BA=20jembadb=205.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 589323b..ff8ae7c 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -44,7 +44,7 @@ class DbSearcher { where = `@@indexIter('value', (v) => { const enru = new Set(${db.esc(enruArr)}); return !v || (!enru.has(v[0].toLowerCase()) && v.indexOf(${db.esc(a)}) >= 0); - });`; + })`; } else { where = `@@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`; } @@ -64,22 +64,22 @@ class DbSearcher { const authorRows = await db.select({ table: 'author', - dirtyIdsOnly: true, - where + rawResult: true, + where: `return Array.from(${where.substring(1)})`, }); - for (const row of authorRows) - authorIds.push(row.id); + if (authorRows.length) + authorIds = authorRows[0].rawResult; } else {//все авторы if (!this.searchCache.authorIdsAll) { const authorRows = await db.select({ table: 'author', - dirtyIdsOnly: true, + rawResult: true, + where: `return Array.from(@all())`, }); - for (const row of authorRows) { - authorIds.push(row.id); - } + if (authorRows.length) + authorIds = authorRows[0].rawResult; this.searchCache.authorIdsAll = authorIds; } else {//оптимизация @@ -407,7 +407,7 @@ class DbSearcher { } this.searchCache = null; - + if (this.timer) { clearTimeout(this.timer); this.timer = null; From 5a19af3c7d5c7f5ab7d6c4c6ab4c9a40c1b0112f Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 18:52:20 +0700 Subject: [PATCH 031/133] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=D1=8B=20=D0=BF=D0=BE=D0=B4=D1=81=D1=87=D0=B5=D1=82=20?= =?UTF-8?q?=D0=B8=20=D0=BE=D1=82=D0=BE=D0=B1=D1=80=D0=B0=D0=B6=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D1=81=D1=82=D0=B0=D1=82=D0=B8=D1=81=D1=82=D0=B8?= =?UTF-8?q?=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 12 +++++---- server/core/DbCreator.js | 39 +++++++++++++++++++---------- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 196fb8a..e0fbb4d 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -680,19 +680,21 @@ class Search { info += `
`; info += ` -
Всего файлов книг:
${stat.filesCount}
+
Всего файлов книг:
${stat.filesCountAll}
+
Из них актуальных:
${stat.filesCount}
+
Помеченных как удаленные:
${stat.filesDelCount}

-
Обработано ссылок на книги:
${stat.bookCountAll}
+
Обработано ссылок на файлы:
${stat.bookCountAll}
Из них актуальных:
${stat.bookCount}
Помеченных как удаленные:
${stat.bookDelCount}
Актуальных без автора:
${stat.noAuthorBookCount}

-
Всего записей об авторах:
${stat.authorCountAll}
-
Записей без соавторов:
${stat.authorCount}
+
Всего имен авторов:
${stat.authorCountAll}
+
Уникальных имен без соавторов:
${stat.authorCount}
С соавторами:
${stat.authorCountAll- stat.authorCount}

Уникальных названий книг:
${stat.titleCount}
-
Уникальных серий:
${stat.seriesCount}
+
Уникальных названий серий:
${stat.seriesCount}
Найдено жанров:
${stat.genreCount}
Найдено языков:
${stat.langCount}
`; diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 998bc61..29e43ac 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -464,7 +464,9 @@ class DbCreator { //stats const stats = { - filesCount: 0, + filesCount: 0,//вычислим позднее + filesCountAll: 0,//вычислим позднее + filesDelCount: 0,//вычислим позднее recsLoaded, authorCount, authorCountAll: authorArr.length, @@ -554,10 +556,10 @@ class DbCreator { cacheSize: (config.lowMemoryMode ? 5 : 500), }); - callback({job: 'series optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0}); + callback({job: 'optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0}); await this.optimizeSeries(db, callback); - callback({job: 'files count', jobMessage: 'Подсчет статистики', jobStep: 12, progress: 0}); + callback({job: 'stats count', jobMessage: 'Подсчет статистики', jobStep: 12, progress: 0}); await this.countStats(db, callback, stats); //чистка памяти, ибо жрет как не в себя @@ -676,22 +678,33 @@ class DbCreator { })(); //подчсет - const countRes = await db.select({table: 'book', count: true, where: ` - const filesSet = new Set(); + const countRes = await db.select({table: 'book', rawResult: true, where: ` + const files = new Set(); + const filesDel = new Set(); - @@iter(@all(), (r) => { + for (const id of @all()) { + const r = @row(id); const file = ${"`${r.folder}/${r.file}.${r.ext}`"}; - if (filesSet.has(file)) { - return false; + if (!r.del) { + files.add(file); } else { - filesSet.add(file); - return true; + filesDel.add(file); } - }); + } + + for (const file of filesDel) + if (files.has(file)) + filesDel.delete(file); + + return {filesCount: files.size, filesDelCount: filesDel.size}; `}); - if (countRes.length) - stats.filesCount = countRes[0].count; + if (countRes.length) { + const res = countRes[0].rawResult; + stats.filesCount = res.filesCount; + stats.filesCountAll = res.filesCount + res.filesDelCount; + stats.filesDelCount = res.filesDelCount; + } countDone = true; } From cd72de50140076772e350700a9d65a5167a3ac35 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 19:59:19 +0700 Subject: [PATCH 032/133] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 135 ++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 56 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index ff8ae7c..a47bc0f 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -35,18 +35,18 @@ class DbSearcher { //особая обработка префиксов if (a[0] == '=') { a = a.substring(1); - where = `@@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a)})`; + where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a)})`; } else if (a[0] == '*') { a = a.substring(1); - where = `@@indexIter('value', (v) => (v.indexOf(${db.esc(a)}) >= 0) )`; + where = `@indexIter('value', (v) => (v.indexOf(${db.esc(a)}) >= 0) )`; } else if (a[0] == '#') { a = a.substring(1); - where = `@@indexIter('value', (v) => { + where = `@indexIter('value', (v) => { const enru = new Set(${db.esc(enruArr)}); return !v || (!enru.has(v[0].toLowerCase()) && v.indexOf(${db.esc(a)}) >= 0); })`; } else { - where = `@@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`; + where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`; } return where; @@ -65,11 +65,10 @@ class DbSearcher { const authorRows = await db.select({ table: 'author', rawResult: true, - where: `return Array.from(${where.substring(1)})`, + where: `return Array.from(${where})`, }); - if (authorRows.length) - authorIds = authorRows[0].rawResult; + authorIds = authorRows[0].rawResult; } else {//все авторы if (!this.searchCache.authorIdsAll) { const authorRows = await db.select({ @@ -78,8 +77,7 @@ class DbSearcher { where: `return Array.from(@all())`, }); - if (authorRows.length) - authorIds = authorRows[0].rawResult; + authorIds = authorRows[0].rawResult; this.searchCache.authorIdsAll = authorIds; } else {//оптимизация @@ -95,17 +93,22 @@ class DbSearcher { const seriesRows = await db.select({ table: 'series', - map: `(r) => ({authorId: r.authorId})`, - where + rawResult: true, + where: ` + const ids = ${where}; + + const result = new Set(); + for (const id of ids) { + const row = @row(id); + for (const authorId of row.authorId) + result.add(authorId); + } + + return Array.from(result); + ` }); - const ids = new Set(); - for (const row of seriesRows) { - for (const id of row.authorId) - ids.add(id); - } - - idsArr.push(ids); + idsArr.push(new Set(seriesRows[0].rawResult)); } //названия @@ -114,19 +117,25 @@ class DbSearcher { let titleRows = await db.select({ table: 'title', - map: `(r) => ({authorId: r.authorId})`, - where + rawResult: true, + where: ` + const ids = ${where}; + + const result = new Set(); + for (const id of ids) { + const row = @row(id); + for (const authorId of row.authorId) + result.add(authorId); + } + + return Array.from(result); + ` }); - const ids = new Set(); - for (const row of titleRows) { - for (const id of row.authorId) - ids.add(id); - } - idsArr.push(ids); + idsArr.push(new Set(titleRows[0].rawResult)); //чистки памяти при тяжелых запросах - if (query.title[0] == '*') { + if (this.config.lowMemoryMode && query.title[0] == '*') { titleRows = null; utils.freeMemory(); await db.freeMemory(); @@ -135,44 +144,58 @@ class DbSearcher { //жанры if (query.genre) { - const genres = query.genre.split(','); + const genreRows = await db.select({ + table: 'genre', + rawResult: true, + where: ` + const genres = ${db.esc(query.genre.split(','))}; - const ids = new Set(); - for (const g of genres) { - const genreRows = await db.select({ - table: 'genre', - map: `(r) => ({authorId: r.authorId})`, - where: `@@indexLR('value', ${db.esc(g)}, ${db.esc(g)})`, - }); + const ids = new Set(); + for (const g of genres) { + for (const id of @indexLR('value', g, g)) + ids.add(id); + } + + const result = new Set(); + for (const id of ids) { + const row = @row(id); + for (const authorId of row.authorId) + result.add(authorId); + } - for (const row of genreRows) { - for (const id of row.authorId) - ids.add(id); - } - } + return Array.from(result); + ` + }); - idsArr.push(ids); + idsArr.push(new Set(genreRows[0].rawResult)); } //языки if (query.lang) { - const langs = query.lang.split(','); + const langRows = await db.select({ + table: 'lang', + rawResult: true, + where: ` + const langs = ${db.esc(query.lang.split(','))}; - const ids = new Set(); - for (const l of langs) { - const langRows = await db.select({ - table: 'lang', - map: `(r) => ({authorId: r.authorId})`, - where: `@@indexLR('value', ${db.esc(l)}, ${db.esc(l)})`, - }); + const ids = new Set(); + for (const l of langs) { + for (const id of @indexLR('value', l, l)) + ids.add(id); + } + + const result = new Set(); + for (const id of ids) { + const row = @row(id); + for (const authorId of row.authorId) + result.add(authorId); + } - for (const row of langRows) { - for (const id of row.authorId) - ids.add(id); - } - } - - idsArr.push(ids); + return Array.from(result); + ` + }); + + idsArr.push(new Set(langRows[0].rawResult)); } if (idsArr.length) { From 93634b113715424f4996034a5d3aeca824c459b6 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 21 Oct 2022 20:18:32 +0700 Subject: [PATCH 033/133] =?UTF-8?q?=D0=94=D0=BE=D1=80=D0=B0=D0=B1=D0=BE?= =?UTF-8?q?=D1=82=D0=BA=D0=B0=20vueComponent,=20=D1=82=D0=B5=D0=BF=D0=B5?= =?UTF-8?q?=D1=80=D1=8C=20=D0=BF=D0=BE=D0=BD=D0=B8=D0=BC=D0=B0=D0=B5=D1=82?= =?UTF-8?q?=20=D0=B8=D0=B5=D1=80=D0=B0=D1=80=D1=85=D0=B8=D1=8E=20=D0=BA?= =?UTF-8?q?=D0=BB=D0=B0=D1=81=D1=81=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/vueComponent.js | 35 ++++++++++++++++++------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/client/components/vueComponent.js b/client/components/vueComponent.js index 67f8c2e..5ac69ad 100644 --- a/client/components/vueComponent.js +++ b/client/components/vueComponent.js @@ -17,7 +17,7 @@ export default function(componentClass) { } } } else if (prop === '_props') { - comp['props'] = obj[prop]; + comp.props = obj[prop]; } } else {//usual prop data[prop] = obj[prop]; @@ -26,23 +26,28 @@ export default function(componentClass) { comp.data = () => _.cloneDeep(data); //methods - const classProto = Object.getPrototypeOf(obj); - const classMethods = Object.getOwnPropertyNames(classProto); const methods = {}; const computed = {}; - for (const method of classMethods) { - const desc = Object.getOwnPropertyDescriptor(classProto, method); - if (desc.get) {//has getter, computed - computed[method] = {get: desc.get}; - if (desc.set) - computed[method].set = desc.set; - } else if ( ['beforeCreate', 'created', 'beforeMount', 'mounted', 'beforeUpdate', 'updated', 'activated',//life cycle hooks - 'deactivated', 'beforeUnmount', 'unmounted', 'errorCaptured', 'renderTracked', 'renderTriggered',//life cycle hooks - 'setup'].includes(method) ) { - comp[method] = obj[method]; - } else if (method !== 'constructor') {//usual - methods[method] = obj[method]; + + let classProto = Object.getPrototypeOf(obj); + while (classProto) { + const classMethods = Object.getOwnPropertyNames(classProto); + for (const method of classMethods) { + const desc = Object.getOwnPropertyDescriptor(classProto, method); + if (desc.get) {//has getter, computed + computed[method] = {get: desc.get}; + if (desc.set) + computed[method].set = desc.set; + } else if ( ['beforeCreate', 'created', 'beforeMount', 'mounted', 'beforeUpdate', 'updated', 'activated',//life cycle hooks + 'deactivated', 'beforeUnmount', 'unmounted', 'errorCaptured', 'renderTracked', 'renderTriggered',//life cycle hooks + 'setup'].includes(method) ) { + comp[method] = obj[method]; + } else if (method !== 'constructor') {//usual + methods[method] = obj[method]; + } } + + classProto = Object.getPrototypeOf(classProto); } comp.methods = methods; comp.computed = computed; From 6a8f08fca8eb6353e015ca64645cc21b3d5bfb05 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sat, 22 Oct 2022 00:32:32 +0700 Subject: [PATCH 034/133] "jembadb": "^5.0.2" --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9428d45..47748ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "axios": "^0.27.2", "express": "^4.18.1", "fs-extra": "^10.1.0", - "jembadb": "^5.0.1", + "jembadb": "^5.0.2", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", @@ -5022,9 +5022,9 @@ } }, "node_modules/jembadb": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.1.tgz", - "integrity": "sha512-fOKfUK8vOQJDtugJAYCllfQiYD3G0GWKNhOMOLXF8jkOcgnsCxQRxAVDIM/K5s07q1I96DIQcsWFBBDfYA/heA==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.2.tgz", + "integrity": "sha512-0309Qo4wSkyf154xTokxNl0DuBP5f2Q2MzWGUNX1JmMzlRypFsPY/9VDYV/htkxhT53f2prlQ2NUguQjG2lCRA==", "engines": { "node": ">=16.16.0" } @@ -12468,9 +12468,9 @@ "dev": true }, "jembadb": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.1.tgz", - "integrity": "sha512-fOKfUK8vOQJDtugJAYCllfQiYD3G0GWKNhOMOLXF8jkOcgnsCxQRxAVDIM/K5s07q1I96DIQcsWFBBDfYA/heA==" + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.2.tgz", + "integrity": "sha512-0309Qo4wSkyf154xTokxNl0DuBP5f2Q2MzWGUNX1JmMzlRypFsPY/9VDYV/htkxhT53f2prlQ2NUguQjG2lCRA==" }, "jest-worker": { "version": "27.5.1", diff --git a/package.json b/package.json index c5d85b9..06826ea 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "axios": "^0.27.2", "express": "^4.18.1", "fs-extra": "^10.1.0", - "jembadb": "^5.0.1", + "jembadb": "^5.0.2", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", From 1d3a661973eaa7e9f553d8c5a404e05037ccc2b6 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sat, 22 Oct 2022 00:40:07 +0700 Subject: [PATCH 035/133] =?UTF-8?q?=D0=A3=D1=81=D0=BA=D0=BE=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BF=D1=80=D0=BE=D1=86=D0=B5=D0=B4=D1=83?= =?UTF-8?q?=D1=80=D1=8B=20=D0=BF=D0=BE=D0=B8=D1=81=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index a47bc0f..877aae2 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -99,7 +99,7 @@ class DbSearcher { const result = new Set(); for (const id of ids) { - const row = @row(id); + const row = @unsafeRow(id); for (const authorId of row.authorId) result.add(authorId); } @@ -123,7 +123,7 @@ class DbSearcher { const result = new Set(); for (const id of ids) { - const row = @row(id); + const row = @unsafeRow(id); for (const authorId of row.authorId) result.add(authorId); } @@ -158,7 +158,7 @@ class DbSearcher { const result = new Set(); for (const id of ids) { - const row = @row(id); + const row = @unsafeRow(id); for (const authorId of row.authorId) result.add(authorId); } @@ -186,7 +186,7 @@ class DbSearcher { const result = new Set(); for (const id of ids) { - const row = @row(id); + const row = @unsafeRow(id); for (const authorId of row.authorId) result.add(authorId); } From 19aaba2492afbab9329455babcd7dcd483b3b120 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sat, 22 Oct 2022 00:49:45 +0700 Subject: [PATCH 036/133] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BF=D0=BE=D1=82=D0=B5=D0=BD?= =?UTF-8?q?=D1=86=D0=B8=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D1=85=20=D0=B1=D0=B0?= =?UTF-8?q?=D0=B3=D0=BE=D0=B2=20=D0=BF=D1=80=D0=B8=20=D0=B8=D1=81=D0=BF?= =?UTF-8?q?=D0=BE=D0=BB=D1=8C=D0=B7=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B8=20?= =?UTF-8?q?=D0=9E=D0=9E=D0=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/vueComponent.js | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/client/components/vueComponent.js b/client/components/vueComponent.js index 5ac69ad..3d4f11b 100644 --- a/client/components/vueComponent.js +++ b/client/components/vueComponent.js @@ -35,15 +35,19 @@ export default function(componentClass) { for (const method of classMethods) { const desc = Object.getOwnPropertyDescriptor(classProto, method); if (desc.get) {//has getter, computed - computed[method] = {get: desc.get}; - if (desc.set) - computed[method].set = desc.set; - } else if ( ['beforeCreate', 'created', 'beforeMount', 'mounted', 'beforeUpdate', 'updated', 'activated',//life cycle hooks - 'deactivated', 'beforeUnmount', 'unmounted', 'errorCaptured', 'renderTracked', 'renderTriggered',//life cycle hooks - 'setup'].includes(method) ) { - comp[method] = obj[method]; + if (!computed[method]) { + computed[method] = {get: desc.get}; + if (desc.set) + computed[method].set = desc.set; + } + } else if ( ['beforeCreate', 'created', 'beforeMount', 'mounted', 'beforeUpdate', 'updated', 'activated', + 'deactivated', 'beforeUnmount', 'unmounted', 'errorCaptured', 'renderTracked', 'renderTriggered', + 'setup'].includes(method) ) {//life cycle hooks + if (!comp[method]) + comp[method] = obj[method]; } else if (method !== 'constructor') {//usual - methods[method] = obj[method]; + if (!methods[method]) + methods[method] = obj[method]; } } From 12e5c325781309bfb4d22eb36b863dd3077d983b Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sat, 22 Oct 2022 13:18:25 +0700 Subject: [PATCH 037/133] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B8=D0=B9=20?= =?UTF-8?q?=D1=80=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE=D1=80=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index e0fbb4d..a8b615a 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -21,12 +21,12 @@
- +
Коллекция @@ -709,8 +709,8 @@ class Search { this.$root.stdDialog.alert(info, 'Статистика по коллекции', {iconName: 'la la-info-circle'}); } - newSearch() { - window.location = window.location.origin; + get newSearchLink() { + return window.location.origin; } async hideTooltip() { From 8d5792d456e2c08cc22a48f307f5af5dd21a9225 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sat, 22 Oct 2022 19:50:51 +0700 Subject: [PATCH 038/133] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=BE=20=D0=BE=D1=82=D0=BE=D0=B1=D1=80=D0=B0=D0=B6?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B2=D0=B5=D1=80=D1=81=D0=B8=D0=B8?= =?UTF-8?q?=20=D0=91=D0=94=20=D0=B2=20=D1=81=D1=82=D0=B0=D1=82=D0=B8=D1=81?= =?UTF-8?q?=D1=82=D0=B8=D0=BA=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 2 ++ server/config/base.js | 6 +++++- server/core/InpxHashCreator.js | 6 +----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index a8b615a..5413311 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -697,6 +697,8 @@ class Search {
Уникальных названий серий:
${stat.seriesCount}
Найдено жанров:
${stat.genreCount}
Найдено языков:
${stat.langCount}
+
+
Версия поисковой БД:
${this.config.dbVersion}
`; info += ` diff --git a/server/config/base.js b/server/config/base.js index 77c0d8a..ec38128 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -14,7 +14,11 @@ module.exports = { bookReadLink: '', loggingEnabled: true, + //поправить в случае, если были критические изменения в DbCreator + //иначе будет рассинхронизация между сервером и клиентом на уровне БД + dbVersion: '3', dbCacheSize: 5, + maxPayloadSize: 500,//in MB maxFilesDirSize: 1024*1024*1024,//1Gb queryCacheEnabled: true, @@ -22,7 +26,7 @@ module.exports = { inpxCheckInterval: 60,//minutes lowMemoryMode: false, - webConfigParams: ['name', 'version', 'branch', 'bookReadLink'], + webConfigParams: ['name', 'version', 'branch', 'bookReadLink', 'dbVersion'], allowRemoteLib: false, remoteLib: false, diff --git a/server/core/InpxHashCreator.js b/server/core/InpxHashCreator.js index 0248615..418f556 100644 --- a/server/core/InpxHashCreator.js +++ b/server/core/InpxHashCreator.js @@ -2,10 +2,6 @@ const fs = require('fs-extra'); const utils = require('./utils'); -//поправить в случае, если были критические изменения в DbCreator -//иначе будет рассинхронизация между сервером и клиентом на уровне БД -const dbCreatorVersion = '3'; - class InpxHashCreator { constructor(config) { this.config = config; @@ -18,7 +14,7 @@ class InpxHashCreator { if (await fs.pathExists(config.inpxFilterFile)) inpxFilterHash = await utils.getFileHash(config.inpxFilterFile, 'sha256', 'hex'); - const joinedHash = dbCreatorVersion + inpxFilterHash + + const joinedHash = this.config.dbVersion + inpxFilterHash + await utils.getFileHash(config.inpxFile, 'sha256', 'hex'); return utils.getBufHash(joinedHash, 'sha256', 'hex'); From 86aa3511bfdf148be8896fd19682a6d5fac31da3 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 23 Oct 2022 16:32:55 +0700 Subject: [PATCH 039/133] =?UTF-8?q?=D0=9D=D0=BE=D0=B2=D1=8B=D0=B9=20=D0=BC?= =?UTF-8?q?=D0=BE=D0=B4=D1=83=D0=BB=D1=8C=20HeavyCalc=20=D0=B4=D0=BB=D1=8F?= =?UTF-8?q?=20=D1=82=D1=8F=D0=B6=D0=B5=D0=BB=D1=8B=D1=85=20=D0=B2=D1=8B?= =?UTF-8?q?=D1=87=D0=B8=D1=81=D0=BB=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=B2=20?= =?UTF-8?q?=D0=BE=D1=82=D0=B4=D0=B5=D0=BB=D1=8C=D0=BD=D0=BE=D0=BC=20=D0=BF?= =?UTF-8?q?=D0=BE=D1=82=D0=BE=D0=BA=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/HeavyCalc.js | 133 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 server/core/HeavyCalc.js diff --git a/server/core/HeavyCalc.js b/server/core/HeavyCalc.js new file mode 100644 index 0000000..a12f31e --- /dev/null +++ b/server/core/HeavyCalc.js @@ -0,0 +1,133 @@ +const { Worker } = require('worker_threads'); + +class CalcThread { + constructor() { + this.worker = null; + this.listeners = new Map(); + this.requestId = 0; + + this.runWorker(); + } + + terminate() { + if (this.worker) { + this.worker.terminate(); + + for (const listener of this.listeners.values()) { + listener({error: 'Worker terminated'}); + } + } + this.worker = null; + } + + runWorker() { + const workerProc = () => { + const { parentPort } = require('worker_threads'); + + const sleep = (ms) => { + return new Promise(resolve => setTimeout(resolve, ms)); + }; + + if (parentPort) { + parentPort.on('message', async(mes) => { + let result = {}; + try { + const fn = new Function(`'use strict'; return ${mes.fn}`)(); + result.result = await fn(mes.args, sleep); + } catch (e) { + result = {error: e.message}; + } + + result.requestId = mes.requestId; + parentPort.postMessage(result); + }); + } + }; + + const worker = new Worker(`const wp = ${workerProc.toString()}; wp();`, {eval: true}); + + worker.on('message', (mes) => { + const listener = this.listeners.get(mes.requestId); + if (listener) { + this.listeners.delete(mes.requestId); + listener(mes); + } + }); + + worker.on('error', (err) => { + console.error(err); + }); + + worker.on('exit', () => { + this.terminate(); + }); + + this.worker = worker; + } + + //async + run(params) {//args, fn + return new Promise((resolve, reject) => { + this.requestId++; + + this.listeners.set(this.requestId, (mes) => { + if (mes.error) + reject(new Error(mes.error)); + else + resolve(mes.result); + }); + + if (this.worker) { + this.worker.postMessage({requestId: this.requestId, args: params.args, fn: params.fn.toString()}); + } else { + reject(new Error('Worker does not exist')); + } + }); + } +} + +class HeavyCalc { + constructor(opts = {}) { + this.threads = opts.threads || 1; + this.singleton = opts.singleton || false; + this.terminated = false; + + this.workers = []; + this.load = []; + for (let i = 0; i < this.threads; i++) { + const worker = new CalcThread(); + this.workers.push(worker); + this.load.push(0); + } + } + + async run(params) { + if (this.terminated || !this.workers.length) + throw new Error('All workers terminated'); + + //находим поток с минимальной нагрузкой + let found = 0; + for (let i = 1; i < this.load.length; i++) { + if (this.load[i] < this.load[found]) + found = i; + } + + try { + this.load[found]++; + return await this.workers[found].run(params); + } finally { + this.load[found]--; + } + } + + terminate() { + for (let i = 0; i < this.workers.length; i++) { + this.workers[i].terminate(); + } + this.workers = []; + this.load = []; + this.terminated = true; + } +} + +module.exports = HeavyCalc; \ No newline at end of file From ab4d54cc85c0135db600506f30c7a5480a8f8bcb Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 23 Oct 2022 18:01:20 +0700 Subject: [PATCH 040/133] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=B2=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D1=8C=20=D0=B8=D1=81=D0=BF=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D0=B7=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F=20HeavyCalc=20=D0=BA?= =?UTF-8?q?=D0=B0=D0=BA=20singleton?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/HeavyCalc.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/core/HeavyCalc.js b/server/core/HeavyCalc.js index a12f31e..2c08891 100644 --- a/server/core/HeavyCalc.js +++ b/server/core/HeavyCalc.js @@ -86,10 +86,17 @@ class CalcThread { } } +//singleton +let instance = null; + class HeavyCalc { constructor(opts = {}) { + const singleton = opts.singleton || false; + + if (singleton && instance) + return instance; + this.threads = opts.threads || 1; - this.singleton = opts.singleton || false; this.terminated = false; this.workers = []; @@ -99,6 +106,10 @@ class HeavyCalc { this.workers.push(worker); this.load.push(0); } + + if (singleton) { + instance = this; + } } async run(params) { From 91d3d4c2540d7a4eff80ed6d8999f608305a6117 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 23 Oct 2022 18:09:02 +0700 Subject: [PATCH 041/133] =?UTF-8?q?=D0=A2=D1=8F=D0=B6=D0=B5=D0=BB=D1=8B?= =?UTF-8?q?=D0=B5=20=D0=B2=D1=8B=D1=87=D0=B8=D1=81=D0=BB=D0=B5=D0=BD=D0=B8?= =?UTF-8?q?=D1=8F=20=D0=B2=D1=8B=D0=BD=D0=B5=D1=81=D0=B5=D0=BD=D1=8B=20?= =?UTF-8?q?=D0=B2=20=D0=BE=D1=82=D0=B4=D0=B5=D0=BB=D1=8C=D0=BD=D1=8B=D0=B9?= =?UTF-8?q?=20=D0=BF=D0=BE=D1=82=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 78 ++++++++++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 877aae2..c0180bd 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -1,5 +1,5 @@ //const _ = require('lodash'); - +const HeavyCalc = require('./HeavyCalc'); const utils = require('./utils'); const maxMemCacheSize = 100; @@ -18,6 +18,8 @@ class DbSearcher { this.timer = null; this.closed = false; + this.heavyCalc = new HeavyCalc({threads: 4}); + this.searchCache = { memCache: new Map(), authorIdsAll: false, @@ -108,7 +110,7 @@ class DbSearcher { ` }); - idsArr.push(new Set(seriesRows[0].rawResult)); + idsArr.push(seriesRows[0].rawResult); } //названия @@ -132,7 +134,7 @@ class DbSearcher { ` }); - idsArr.push(new Set(titleRows[0].rawResult)); + idsArr.push(titleRows[0].rawResult); //чистки памяти при тяжелых запросах if (this.config.lowMemoryMode && query.title[0] == '*') { @@ -167,7 +169,7 @@ class DbSearcher { ` }); - idsArr.push(new Set(genreRows[0].rawResult)); + idsArr.push(genreRows[0].rawResult); } //языки @@ -195,17 +197,75 @@ class DbSearcher { ` }); - idsArr.push(new Set(langRows[0].rawResult)); + idsArr.push(langRows[0].rawResult); } - if (idsArr.length) { - //ищем пересечение множеств - idsArr.push(new Set(authorIds)); - authorIds = Array.from(utils.intersectSet(idsArr)); +/* + //ищем пересечение множеств + idsArr.push(authorIds); + + if (idsArr.length > 1) { + const idsSetArr = idsArr.map(ids => new Set(ids)); + authorIds = Array.from(utils.intersectSet(idsSetArr)); } //сортировка authorIds.sort((a, b) => a - b); +*/ + //ищем пересечение множеств в отдельном потоке + idsArr.push(authorIds); + authorIds = await this.heavyCalc.run({ + args: idsArr, + fn: (args) => { + //из utils.intersectSet + const intersectSet = (arrSet) => { + if (!arrSet.length) + return new Set(); + + let min = 0; + let size = arrSet[0].size; + for (let i = 1; i < arrSet.length; i++) { + if (arrSet[i].size < size) { + min = i; + size = arrSet[i].size; + } + } + + const result = new Set(); + for (const elem of arrSet[min]) { + let inAll = true; + for (let i = 0; i < arrSet.length; i++) { + if (i === min) + continue; + if (!arrSet[i].has(elem)) { + inAll = false; + break; + } + } + + if (inAll) + result.add(elem); + } + + return result; + }; + + //считаем пересечение, если надо + let result = []; + + if (args.length > 1) { + const arrSet = args.map(ids => new Set(ids)); + result = Array.from(intersectSet(arrSet)); + } else if (args.length == 1) { + result = args[0]; + } + + //сортировка + result.sort((a, b) => a - b); + + return result; + } + }); return authorIds; } From 54f0cfec76757daf51181b45ba91d1d31708d967 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 23 Oct 2022 18:13:05 +0700 Subject: [PATCH 042/133] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20heavyCalc.terminate()=20=D0=B2=20close?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index c0180bd..5b6888a 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -496,6 +496,7 @@ class DbSearcher { this.timer = null; } + this.heavyCalc.terminate(); this.closed = true; } } From ea6d61ac6fb6ecdb994aa988c4be6c777f582783 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 23 Oct 2022 18:43:14 +0700 Subject: [PATCH 043/133] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 117 ++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 54 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 5b6888a..f186a2e 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -54,6 +54,61 @@ class DbSearcher { return where; } + async calcIntersect(idsArr) { + return await this.heavyCalc.run({ + args: idsArr, + fn: (args) => { + //из utils.intersectSet + const intersectSet = (arrSet) => { + if (!arrSet.length) + return new Set(); + + let min = 0; + let size = arrSet[0].size; + for (let i = 1; i < arrSet.length; i++) { + if (arrSet[i].size < size) { + min = i; + size = arrSet[i].size; + } + } + + const result = new Set(); + for (const elem of arrSet[min]) { + let inAll = true; + for (let i = 0; i < arrSet.length; i++) { + if (i === min) + continue; + if (!arrSet[i].has(elem)) { + inAll = false; + break; + } + } + + if (inAll) + result.add(elem); + } + + return result; + }; + + //считаем пересечение, если надо + let result = []; + + if (args.length > 1) { + const arrSet = args.map(ids => new Set(ids)); + result = Array.from(intersectSet(arrSet)); + } else if (args.length == 1) { + result = args[0]; + } + + //сортировка + result.sort((a, b) => a - b); + + return result; + } + }); + } + async selectAuthorIds(query) { const db = this.db; @@ -212,60 +267,14 @@ class DbSearcher { //сортировка authorIds.sort((a, b) => a - b); */ - //ищем пересечение множеств в отдельном потоке - idsArr.push(authorIds); - authorIds = await this.heavyCalc.run({ - args: idsArr, - fn: (args) => { - //из utils.intersectSet - const intersectSet = (arrSet) => { - if (!arrSet.length) - return new Set(); - - let min = 0; - let size = arrSet[0].size; - for (let i = 1; i < arrSet.length; i++) { - if (arrSet[i].size < size) { - min = i; - size = arrSet[i].size; - } - } - - const result = new Set(); - for (const elem of arrSet[min]) { - let inAll = true; - for (let i = 0; i < arrSet.length; i++) { - if (i === min) - continue; - if (!arrSet[i].has(elem)) { - inAll = false; - break; - } - } - - if (inAll) - result.add(elem); - } - - return result; - }; - - //считаем пересечение, если надо - let result = []; - - if (args.length > 1) { - const arrSet = args.map(ids => new Set(ids)); - result = Array.from(intersectSet(arrSet)); - } else if (args.length == 1) { - result = args[0]; - } - - //сортировка - result.sort((a, b) => a - b); - - return result; - } - }); + if (idsArr.length) { + //ищем пересечение множеств в отдельном потоке + idsArr.push(authorIds); + authorIds = await this.calcIntersect(idsArr); + } else { + //просто сортировка + authorIds.sort((a, b) => a - b); + } return authorIds; } From 784dda03fda215018eef791da6ba5d1cce316dd9 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 23 Oct 2022 19:05:17 +0700 Subject: [PATCH 044/133] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=B1=D0=B0=D0=B3=D0=BE=D0=B2=20=D0=BF=D0=BE?= =?UTF-8?q?=D0=B8=D1=81=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 7 ++++--- server/core/DbSearcher.js | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 5413311..9340b4b 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -1124,6 +1124,7 @@ class Search { const s = this.search; const emptyFieldValue = '?'; + const maxUtf8Char = String.fromCodePoint(0xFFFFF); const ruAlphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'; const enAlphabet = 'abcdefghijklmnopqrstuvwxyz'; const enru = new Set((ruAlphabet + enAlphabet).split('')); @@ -1151,7 +1152,7 @@ class Search { if (searchValue[0] == '=') { searchValue = searchValue.substring(1); - return bookValue == searchValue; + return bookValue.localeCompare(searchValue) == 0; } else if (searchValue[0] == '*') { searchValue = searchValue.substring(1); @@ -1163,8 +1164,8 @@ class Search { } else if (searchValue[0] == '?') { return bookValue == '' || bookValue.indexOf(searchValue) == 0; } else { - - return bookValue.indexOf(searchValue) == 0; + //where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`; + return bookValue.localeCompare(searchValue) >= 0 && bookValue.localeCompare(searchValue + maxUtf8Char) <= 0; } }; diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index f186a2e..78cdba4 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -45,7 +45,7 @@ class DbSearcher { a = a.substring(1); where = `@indexIter('value', (v) => { const enru = new Set(${db.esc(enruArr)}); - return !v || (!enru.has(v[0].toLowerCase()) && v.indexOf(${db.esc(a)}) >= 0); + return !v || (!enru.has(v[0]) && v.indexOf(${db.esc(a)}) >= 0); })`; } else { where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`; From f1267879af810e23df0ae0d50932204cfd0be110 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 23 Oct 2022 19:10:13 +0700 Subject: [PATCH 045/133] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=B1=D0=B0=D0=B3=D0=BE=D0=B2=20=D0=BF=D0=BE?= =?UTF-8?q?=D0=B8=D1=81=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 2 +- server/core/DbSearcher.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 9340b4b..853302b 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -1160,7 +1160,7 @@ class Search { } else if (searchValue[0] == '#') { searchValue = searchValue.substring(1); - return !bookValue || (!enru.has(bookValue[0]) && bookValue.indexOf(searchValue) >= 0); + return !bookValue || (bookValue !== emptyFieldValue && !enru.has(bookValue[0]) && bookValue.indexOf(searchValue) >= 0); } else if (searchValue[0] == '?') { return bookValue == '' || bookValue.indexOf(searchValue) == 0; } else { diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 78cdba4..5d9cc2e 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -4,6 +4,7 @@ const utils = require('./utils'); const maxMemCacheSize = 100; +const emptyFieldValue = '?'; const maxUtf8Char = String.fromCodePoint(0xFFFFF); const ruAlphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'; const enAlphabet = 'abcdefghijklmnopqrstuvwxyz'; @@ -45,7 +46,7 @@ class DbSearcher { a = a.substring(1); where = `@indexIter('value', (v) => { const enru = new Set(${db.esc(enruArr)}); - return !v || (!enru.has(v[0]) && v.indexOf(${db.esc(a)}) >= 0); + return !v || (v !== ${db.esc(emptyFieldValue)} && !enru.has(v[0]) && v.indexOf(${db.esc(a)}) >= 0); })`; } else { where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`; From 723af3ea8b3d9e85240bbdd5a3eb1e1aeaf50b7b Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 23 Oct 2022 19:26:59 +0700 Subject: [PATCH 046/133] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=B1=D0=B0=D0=B3=D0=BE=D0=B2=20=D0=BF=D0=BE?= =?UTF-8?q?=D0=B8=D1=81=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Search/Search.vue | 7 ++++--- server/core/DbSearcher.js | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 853302b..50561ef 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -1145,6 +1145,9 @@ class Search { if (!searchValue) return true; + if (!bookValue) + bookValue = emptyFieldValue; + bookValue = bookValue.toLowerCase(); searchValue = searchValue.toLowerCase(); @@ -1156,13 +1159,11 @@ class Search { } else if (searchValue[0] == '*') { searchValue = searchValue.substring(1); - return bookValue.indexOf(searchValue) >= 0; + return bookValue !== emptyFieldValue && bookValue.indexOf(searchValue) >= 0; } else if (searchValue[0] == '#') { searchValue = searchValue.substring(1); return !bookValue || (bookValue !== emptyFieldValue && !enru.has(bookValue[0]) && bookValue.indexOf(searchValue) >= 0); - } else if (searchValue[0] == '?') { - return bookValue == '' || bookValue.indexOf(searchValue) == 0; } else { //where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`; return bookValue.localeCompare(searchValue) >= 0 && bookValue.localeCompare(searchValue + maxUtf8Char) <= 0; diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 5d9cc2e..f4e24f6 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -41,7 +41,7 @@ class DbSearcher { where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a)})`; } else if (a[0] == '*') { a = a.substring(1); - where = `@indexIter('value', (v) => (v.indexOf(${db.esc(a)}) >= 0) )`; + where = `@indexIter('value', (v) => (v !== ${db.esc(emptyFieldValue)} && v.indexOf(${db.esc(a)}) >= 0) )`; } else if (a[0] == '#') { a = a.substring(1); where = `@indexIter('value', (v) => { From 63dfdaf2c2445e61ee05868a3e6db543f629fffc Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 23 Oct 2022 20:51:56 +0700 Subject: [PATCH 047/133] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 85 +++++++++++++++++++-------------------- server/core/HeavyCalc.js | 8 ++-- 2 files changed, 45 insertions(+), 48 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index f4e24f6..d5e4ed5 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -56,57 +56,54 @@ class DbSearcher { } async calcIntersect(idsArr) { - return await this.heavyCalc.run({ - args: idsArr, - fn: (args) => { - //из utils.intersectSet - const intersectSet = (arrSet) => { - if (!arrSet.length) - return new Set(); + return await this.heavyCalc.run(idsArr, (args) => { + //из utils.intersectSet + const intersectSet = (arrSet) => { + if (!arrSet.length) + return new Set(); - let min = 0; - let size = arrSet[0].size; - for (let i = 1; i < arrSet.length; i++) { - if (arrSet[i].size < size) { - min = i; - size = arrSet[i].size; - } + let min = 0; + let size = arrSet[0].size; + for (let i = 1; i < arrSet.length; i++) { + if (arrSet[i].size < size) { + min = i; + size = arrSet[i].size; } - - const result = new Set(); - for (const elem of arrSet[min]) { - let inAll = true; - for (let i = 0; i < arrSet.length; i++) { - if (i === min) - continue; - if (!arrSet[i].has(elem)) { - inAll = false; - break; - } - } - - if (inAll) - result.add(elem); - } - - return result; - }; - - //считаем пересечение, если надо - let result = []; - - if (args.length > 1) { - const arrSet = args.map(ids => new Set(ids)); - result = Array.from(intersectSet(arrSet)); - } else if (args.length == 1) { - result = args[0]; } - //сортировка - result.sort((a, b) => a - b); + const result = new Set(); + for (const elem of arrSet[min]) { + let inAll = true; + for (let i = 0; i < arrSet.length; i++) { + if (i === min) + continue; + if (!arrSet[i].has(elem)) { + inAll = false; + break; + } + } + + if (inAll) + result.add(elem); + } return result; + }; + + //считаем пересечение, если надо + let result = []; + + if (args.length > 1) { + const arrSet = args.map(ids => new Set(ids)); + result = Array.from(intersectSet(arrSet)); + } else if (args.length == 1) { + result = args[0]; } + + //сортировка + result.sort((a, b) => a - b); + + return result; }); } diff --git a/server/core/HeavyCalc.js b/server/core/HeavyCalc.js index 2c08891..4f8ea97 100644 --- a/server/core/HeavyCalc.js +++ b/server/core/HeavyCalc.js @@ -66,7 +66,7 @@ class CalcThread { } //async - run(params) {//args, fn + run(args, fn) { return new Promise((resolve, reject) => { this.requestId++; @@ -78,7 +78,7 @@ class CalcThread { }); if (this.worker) { - this.worker.postMessage({requestId: this.requestId, args: params.args, fn: params.fn.toString()}); + this.worker.postMessage({requestId: this.requestId, args, fn: fn.toString()}); } else { reject(new Error('Worker does not exist')); } @@ -112,7 +112,7 @@ class HeavyCalc { } } - async run(params) { + async run(args, fn) { if (this.terminated || !this.workers.length) throw new Error('All workers terminated'); @@ -125,7 +125,7 @@ class HeavyCalc { try { this.load[found]++; - return await this.workers[found].run(params); + return await this.workers[found].run(args, fn); } finally { this.load[found]--; } From bc2a54c4ab5a22fca0a8a69a365ad7d0c8ba030d Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 24 Oct 2022 16:51:27 +0700 Subject: [PATCH 048/133] =?UTF-8?q?=D0=94=D0=B5=D0=BA=D0=BE=D0=BC=D0=BF?= =?UTF-8?q?=D0=BE=D0=B7=D0=B8=D1=86=D0=B8=D1=8F,=20=D0=B2=D1=8B=D0=B4?= =?UTF-8?q?=D0=B5=D0=BB=D0=B5=D0=BD=20AuthorList?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Search/AuthorList/AuthorList.vue | 911 ++++++++++++++++++ client/components/Search/BaseList.js | 2 + .../components/Search/BookView/BookView.vue | 12 +- .../Search/LoadingMessage/LoadingMessage.vue | 33 + client/components/Search/Search.vue | 862 +---------------- 5 files changed, 995 insertions(+), 825 deletions(-) create mode 100644 client/components/Search/AuthorList/AuthorList.vue create mode 100644 client/components/Search/BaseList.js create mode 100644 client/components/Search/LoadingMessage/LoadingMessage.vue diff --git a/client/components/Search/AuthorList/AuthorList.vue b/client/components/Search/AuthorList/AuthorList.vue new file mode 100644 index 0000000..f8be95c --- /dev/null +++ b/client/components/Search/AuthorList/AuthorList.vue @@ -0,0 +1,911 @@ + + + + + diff --git a/client/components/Search/BaseList.js b/client/components/Search/BaseList.js new file mode 100644 index 0000000..4053afe --- /dev/null +++ b/client/components/Search/BaseList.js @@ -0,0 +1,2 @@ +export default class BaseList { +} \ No newline at end of file diff --git a/client/components/Search/BookView/BookView.vue b/client/components/Search/BookView/BookView.vue index 0492a1d..300cad5 100644 --- a/client/components/Search/BookView/BookView.vue +++ b/client/components/Search/BookView/BookView.vue @@ -85,7 +85,7 @@ class BookView { _options = componentOptions; _props = { book: Object, - genreTree: Array, + genreMap: Object, showAuthor: Boolean, showReadLink: Boolean, titleColor: { type: String, default: 'text-blue-10'}, @@ -140,12 +140,12 @@ class BookView { get bookGenre() { let result = []; - const genre = new Set(this.book.genre.split(',')); + const genre = this.book.genre.split(','); - for (const section of this.genreTree) { - for (const g of section.value) - if (genre.has(g.value)) - result.push(g.name); + for (const g of genre) { + const name = this.genreMap.get(g); + if (name) + result.push(name); } return `(${result.join(' / ')})`; diff --git a/client/components/Search/LoadingMessage/LoadingMessage.vue b/client/components/Search/LoadingMessage/LoadingMessage.vue new file mode 100644 index 0000000..cc5d3ee --- /dev/null +++ b/client/components/Search/LoadingMessage/LoadingMessage.vue @@ -0,0 +1,33 @@ + + + + + diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 50561ef..582b646 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -1,23 +1,5 @@