diff --git a/client/components/Api/Api.vue b/client/components/Api/Api.vue
index 0898738..98daf9b 100644
--- a/client/components/Api/Api.vue
+++ b/client/components/Api/Api.vue
@@ -42,18 +42,15 @@ const rotor = '|/-\\';
const stepBound = [
0,
0,// jobStep = 1
- 18,// jobStep = 2
- 20,// jobStep = 3
- 50,// jobStep = 4
- 62,// jobStep = 5
- 62,// jobStep = 6
- 64,// jobStep = 7
- 65,// jobStep = 8
- 69,// jobStep = 9
- 69,// jobStep = 10
- 70,// jobStep = 11
- 95,// jobStep = 12
- 100,// jobStep = 13
+ 40,// jobStep = 2
+ 50,// jobStep = 3
+ 54,// jobStep = 4
+ 58,// jobStep = 5
+ 69,// jobStep = 6
+ 69,// jobStep = 7
+ 70,// jobStep = 8
+ 95,// jobStep = 9
+ 100,// jobStep = 10
];
const componentOptions = {
@@ -218,16 +215,8 @@ class Api {
}
}
- async authorSearch(query) {
- return await this.request({action: 'author-search', query});
- }
-
- async seriesSearch(query) {
- return await this.request({action: 'series-search', query});
- }
-
- async titleSearch(query) {
- return await this.request({action: 'title-search', query});
+ async search(from, query) {
+ return await this.request({action: 'search', from, query});
}
async getAuthorBookList(authorId) {
diff --git a/client/components/Search/AuthorList/AuthorList.vue b/client/components/Search/AuthorList/AuthorList.vue
index 2680c80..7008e7b 100644
--- a/client/components/Search/AuthorList/AuthorList.vue
+++ b/client/components/Search/AuthorList/AuthorList.vue
@@ -104,10 +104,10 @@
-
+
@@ -317,7 +317,7 @@ class AuthorList extends BaseList {
let result = [];
const expandedSet = new Set(this.expandedAuthor);
- const authors = this.searchResult.author;
+ const authors = this.searchResult.found;
if (!authors)
return;
@@ -399,7 +399,7 @@ class AuthorList extends BaseList {
(async() => {
await utils.sleep(500);
if (this.refreshing)
- this.loadingMessage = 'Поиск серий...';
+ this.loadingMessage = 'Поиск авторов...';
})();
try {
@@ -408,13 +408,13 @@ class AuthorList extends BaseList {
this.queryExecute = null;
try {
- const result = await this.api.authorSearch(query);
+ const response = await this.api.search('author', query);
- this.list.queryFound = result.author.length;
- this.list.totalFound = result.totalFound;
- this.list.inpxHash = result.inpxHash;
+ this.list.queryFound = response.found.length;
+ this.list.totalFound = response.totalFound;
+ this.list.inpxHash = response.inpxHash;
- this.searchResult = result;
+ this.searchResult = response;
await utils.sleep(1);
if (!this.queryExecute) {
diff --git a/client/components/Search/SeriesList/SeriesList.vue b/client/components/Search/SeriesList/SeriesList.vue
index 3a15987..b50b50e 100644
--- a/client/components/Search/SeriesList/SeriesList.vue
+++ b/client/components/Search/SeriesList/SeriesList.vue
@@ -181,7 +181,7 @@ class SeriesList extends BaseList {
let result = [];
const expandedSet = new Set(this.expandedSeries);
- const series = this.searchResult.series;
+ const series = this.searchResult.found;
if (!series)
return;
@@ -256,13 +256,13 @@ class SeriesList extends BaseList {
this.queryExecute = null;
try {
- const result = await this.api.seriesSearch(query);
+ const response = await this.api.search('series', query);
- this.list.queryFound = result.series.length;
- this.list.totalFound = result.totalFound;
- this.list.inpxHash = result.inpxHash;
+ this.list.queryFound = response.found.length;
+ this.list.totalFound = response.totalFound;
+ this.list.inpxHash = response.inpxHash;
- this.searchResult = result;
+ this.searchResult = response;
await utils.sleep(1);
if (!this.queryExecute) {
diff --git a/client/components/Search/TitleList/TitleList.vue b/client/components/Search/TitleList/TitleList.vue
index 84693ec..f0bf34d 100644
--- a/client/components/Search/TitleList/TitleList.vue
+++ b/client/components/Search/TitleList/TitleList.vue
@@ -49,7 +49,7 @@ class TitleList extends BaseList {
async updateTableData() {
let result = [];
- const title = this.searchResult.title;
+ const title = this.searchResult.found;
if (!title)
return;
@@ -116,13 +116,13 @@ class TitleList extends BaseList {
this.queryExecute = null;
try {
- const result = await this.api.titleSearch(query);
+ const response = await this.api.search('title', query);
- this.list.queryFound = result.title.length;
- this.list.totalFound = result.totalFound;
- this.list.inpxHash = result.inpxHash;
+ this.list.queryFound = response.found.length;
+ this.list.totalFound = response.totalFound;
+ this.list.inpxHash = response.inpxHash;
- this.searchResult = result;
+ this.searchResult = response;
await utils.sleep(1);
if (!this.queryExecute) {
diff --git a/server/config/base.js b/server/config/base.js
index 952abaf..4c5d66c 100644
--- a/server/config/base.js
+++ b/server/config/base.js
@@ -17,7 +17,7 @@ module.exports = {
//поправить в случае, если были критические изменения в DbCreator
//иначе будет рассинхронизация между сервером и клиентом на уровне БД
- dbVersion: '4',
+ dbVersion: '5',
dbCacheSize: 5,
maxPayloadSize: 500,//in MB
@@ -26,6 +26,7 @@ module.exports = {
cacheCleanInterval: 60,//minutes
inpxCheckInterval: 60,//minutes
lowMemoryMode: false,
+ fullOptimization: false,
webConfigParams: ['name', 'version', 'branch', 'bookReadLink', 'dbVersion', 'extendedSearch'],
diff --git a/server/config/index.js b/server/config/index.js
index 5b34533..5f1332d 100644
--- a/server/config/index.js
+++ b/server/config/index.js
@@ -15,6 +15,7 @@ const propsToSave = [
'cacheCleanInterval',
'inpxCheckInterval',
'lowMemoryMode',
+ 'fullOptimization',
'allowRemoteLib',
'remoteLib',
'server',
diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js
index 53d9735..0dac9f6 100644
--- a/server/controllers/WebSocketController.js
+++ b/server/controllers/WebSocketController.js
@@ -74,12 +74,8 @@ class WebSocketController {
await this.getConfig(req, ws); break;
case 'get-worker-state':
await this.getWorkerState(req, ws); break;
- case 'author-search':
- await this.authorSearch(req, ws); break;
- case 'series-search':
- await this.seriesSearch(req, ws); break;
- case 'title-search':
- await this.titleSearch(req, ws); break;
+ case 'search':
+ await this.search(req, ws); break;
case 'get-author-book-list':
await this.getAuthorBookList(req, ws); break;
case 'get-series-book-list':
@@ -137,35 +133,13 @@ class WebSocketController {
this.send((state ? state : {}), req, ws);
}
- async authorSearch(req, ws) {
+ async search(req, ws) {
if (!req.query)
throw new Error(`query is empty`);
+ if (!req.from)
+ throw new Error(`from is empty`);
- const result = await this.webWorker.authorSearch(req.query);
-
- this.send(result, req, ws);
- }
-
- async seriesSearch(req, ws) {
- if (!this.config.extendedSearch)
- throw new Error(`Extended search disabled`);
-
- if (!req.query)
- throw new Error(`query is empty`);
-
- const result = await this.webWorker.seriesSearch(req.query);
-
- this.send(result, req, ws);
- }
-
- async titleSearch(req, ws) {
- if (!this.config.extendedSearch)
- throw new Error(`Extended search disabled`);
-
- if (!req.query)
- throw new Error(`query is empty`);
-
- const result = await this.webWorker.titleSearch(req.query);
+ const result = await this.webWorker.search(req.from, req.query);
this.send(result, req, ws);
}
diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js
index d29cdbe..ef54dc7 100644
--- a/server/core/DbCreator.js
+++ b/server/core/DbCreator.js
@@ -138,6 +138,84 @@ class DbCreator {
callback({progress: (readState.current || 0)/totalFiles});
};
+ const parseField = (fieldValue, fieldMap, fieldArr, bookId, rec, fillBookIds = true) => {
+ let value = fieldValue;
+
+ if (typeof(fieldValue) == 'string') {
+ if (!fieldValue)
+ fieldValue = emptyFieldValue;
+
+ value = fieldValue.toLowerCase();
+ }
+
+ let fieldRec;
+ if (fieldMap.has(value)) {
+ const fieldId = fieldMap.get(value);
+ fieldRec = fieldArr[fieldId];
+ } else {
+ fieldRec = {id: fieldArr.length, value, bookIds: new Set()};
+ if (rec !== undefined) {
+ fieldRec.name = fieldValue;
+ fieldRec.bookCount = 0;
+ fieldRec.bookDelCount = 0;
+ }
+ fieldArr.push(fieldRec);
+ fieldMap.set(value, fieldRec.id);
+ }
+
+ if (fieldValue !== emptyFieldValue || fillBookIds)
+ fieldRec.bookIds.add(bookId);
+
+ if (rec !== undefined) {
+ if (!rec.del)
+ fieldRec.bookCount++;
+ else
+ fieldRec.bookDelCount++;
+ }
+ };
+
+ const parseBookRec = (rec) => {
+ //авторы
+ const author = splitAuthor(rec.author);
+
+ for (let i = 0; i < author.length; i++) {
+ const a = author[i];
+
+ //статистика
+ if (!authorMap.has(a.toLowerCase()) && (author.length == 1 || i < author.length - 1)) //без соавторов
+ authorCount++;
+
+ parseField(a, authorMap, authorArr, rec.id, rec);
+ }
+
+ //серии
+ parseField(rec.series, seriesMap, seriesArr, rec.id, rec, false);
+
+ //названия
+ parseField(rec.title, titleMap, titleArr, rec.id, rec);
+
+ //жанры
+ let genre = rec.genre || emptyFieldValue;
+ genre = rec.genre.split(',');
+
+ for (let g of genre) {
+ parseField(g, genreMap, genreArr, rec.id);
+ }
+
+ //языки
+ parseField(rec.lang, langMap, langArr, rec.id);
+
+ //удаленные
+ parseField(rec.del, delMap, delArr, rec.id);
+
+ //дата поступления
+ parseField(rec.date, dateMap, dateArr, rec.id);
+
+ //оценка
+ parseField(rec.librate, librateMap, librateArr, rec.id);
+ };
+
+ //основная процедура парсинга
let id = 0;
const parsedCallback = async(chunk) => {
let filtered = false;
@@ -159,40 +237,7 @@ class DbCreator {
bookDelCount++;
}
- //авторы
- const author = splitAuthor(rec.author);
-
- for (let i = 0; i < author.length; i++) {
- const a = author[i];
- const value = a.toLowerCase();
-
- let authorRec;
- if (authorMap.has(value)) {
- const authorTmpId = authorMap.get(value);
- authorRec = authorArr[authorTmpId];
- } else {
- authorRec = {tmpId: authorArr.length, author: a, value, bookCount: 0, bookDelCount: 0, bookId: []};
- authorArr.push(authorRec);
- authorMap.set(value, authorRec.tmpId);
-
- if (author.length == 1 || i < author.length - 1) //без соавторов
- authorCount++;
- }
-
- //это нужно для того, чтобы имя автора начиналось с заглавной
- if (a[0].toUpperCase() === a[0])
- authorRec.author = a;
-
- //счетчики
- if (!rec.del) {
- authorRec.bookCount++;
- } else {
- authorRec.bookDelCount++;
- }
-
- //ссылки на книги
- authorRec.bookId.push(id);
- }
+ parseBookRec(rec);
}
let saveChunk = [];
@@ -211,246 +256,10 @@ class DbCreator {
utils.freeMemory();
};
- //парсинг 1
+ //парсинг
const parser = new InpxParser();
await parser.parse(config.inpxFile, readFileCallback, parsedCallback);
- utils.freeMemory();
-
- //отсортируем авторов и выдадим им правильные id
- //порядок id соответствует ASC-сортировке по author.toLowerCase
- callback({job: 'author sort', jobMessage: 'Сортировка авторов', jobStep: 2, progress: 0});
- await utils.sleep(100);
- authorArr.sort((a, b) => a.value.localeCompare(b.value));
-
- id = 0;
- authorMap = new Map();
- for (const authorRec of authorArr) {
- authorRec.id = ++id;
- authorMap.set(authorRec.author, id);
- delete authorRec.tmpId;
- }
-
- utils.freeMemory();
-
- //подготовка к сохранению author_book
- const saveBookChunk = async(authorChunk, callback) => {
- callback(0);
-
- const ids = [];
- for (const a of authorChunk) {
- for (const id of a.bookId) {
- ids.push(id);
- }
- }
-
- ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb
-
- callback(0.1);
- const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
- callback(0.6);
- await utils.sleep(100);
-
- const bookArr = new Map();
- for (const row of rows)
- bookArr.set(row.id, row);
-
- const abRows = [];
- for (const a of authorChunk) {
- const aBooks = [];
- for (const id of a.bookId) {
- const rec = bookArr.get(id);
- aBooks.push(rec);
- }
-
- abRows.push({id: a.id, author: a.author, books: JSON.stringify(aBooks)});
-
- delete a.bookId;//в дальнейшем не понадобится, authorArr сохраняем без него
- }
-
- callback(0.7);
- await db.insert({
- table: 'author_book',
- rows: abRows,
- });
- callback(1);
- };
-
- callback({job: 'book sort', jobMessage: 'Сортировка книг', jobStep: 3, progress: 0});
-
- //сохранение author_book
- await db.create({
- table: 'author_book',
- });
-
- let idsLen = 0;
- let aChunk = [];
- let prevI = 0;
- for (let i = 0; i < authorArr.length; i++) {// eslint-disable-line
- const author = authorArr[i];
-
- aChunk.push(author);
- idsLen += author.bookId.length;
-
- if (idsLen > 50000) {//константа выяснена эмпирическим путем "память/скорость"
- await saveBookChunk(aChunk, (p) => {
- callback({progress: (prevI + (i - prevI)*p)/authorArr.length});
- });
-
- prevI = i;
- idsLen = 0;
- aChunk = [];
- await utils.sleep(100);
- utils.freeMemory();
- await db.freeMemory();
- }
- }
- if (aChunk.length) {
- await saveBookChunk(aChunk, () => {});
- aChunk = null;
- }
-
- callback({progress: 1});
-
- //чистка памяти, ибо жрет как не в себя
- await db.close({table: 'book'});
- await db.freeMemory();
- utils.freeMemory();
-
- //парсинг 2, подготовка
- const parseField = (fieldValue, fieldMap, fieldArr, authorIds, bookId) => {
- let addBookId = bookId;
- let value = fieldValue;
-
- if (typeof(fieldValue) == 'string') {
- if (!fieldValue) {
- fieldValue = emptyFieldValue;
- addBookId = 0;//!!!
- }
-
- value = fieldValue.toLowerCase();
- }
-
- let fieldRec;
- if (fieldMap.has(value)) {
- const fieldId = fieldMap.get(value);
- fieldRec = fieldArr[fieldId];
- } else {
- fieldRec = {id: fieldArr.length, value, authorId: new Set()};
- if (bookId)
- fieldRec.bookId = new Set();
- fieldArr.push(fieldRec);
- fieldMap.set(value, fieldRec.id);
- }
-
- for (const id of authorIds) {
- fieldRec.authorId.add(id);
- }
-
- if (addBookId)
- fieldRec.bookId.add(addBookId);
- };
-
- const parseBookRec = (rec) => {
- //авторы
- const author = splitAuthor(rec.author);
-
- const authorIds = [];
- for (const a of author) {
- const authorId = authorMap.get(a);
- if (!authorId) //подстраховка
- continue;
- authorIds.push(authorId);
- }
-
- //серии
- parseField(rec.series, seriesMap, seriesArr, authorIds, rec.id);
-
- //названия
- parseField(rec.title, titleMap, titleArr, authorIds, rec.id);
-
- //жанры
- let genre = rec.genre || emptyFieldValue;
- genre = rec.genre.split(',');
-
- for (let g of genre) {
- if (!g)
- g = emptyFieldValue;
-
- let genreRec;
- if (genreMap.has(g)) {
- const genreId = genreMap.get(g);
- genreRec = genreArr[genreId];
- } else {
- genreRec = {id: genreArr.length, value: g, authorId: new Set()};
- genreArr.push(genreRec);
- genreMap.set(g, genreRec.id);
- }
-
- for (const id of authorIds) {
- genreRec.authorId.add(id);
- }
- }
-
- //языки
- parseField(rec.lang, langMap, langArr, authorIds);
-
- //удаленные
- parseField(rec.del, delMap, delArr, authorIds);
-
- //дата поступления
- parseField(rec.date, dateMap, dateArr, authorIds);
-
- //оценка
- parseField(rec.librate, librateMap, librateArr, authorIds);
- };
-
- callback({job: 'search tables create', jobMessage: 'Создание поисковых таблиц', jobStep: 4, progress: 0});
-
- //парсинг 2, теперь можно создавать остальные поисковые таблицы
- let proc = 0;
- while (1) {// eslint-disable-line
- const rows = await db.select({
- table: 'author_book',
- where: `
- let iter = @getItem('parse_book');
- if (!iter) {
- iter = @all();
- @setItem('parse_book', iter);
- }
-
- const ids = new Set();
- let id = iter.next();
- while (!id.done) {
- ids.add(id.value);
- if (ids.size >= 10000)
- break;
- id = iter.next();
- }
-
- return ids;
- `
- });
-
- if (rows.length) {
- for (const row of rows) {
- const books = JSON.parse(row.books);
- for (const rec of books)
- parseBookRec(rec);
- }
-
- proc += rows.length;
- callback({progress: proc/authorArr.length});
- } else
- break;
-
- if (config.lowMemoryMode) {
- await utils.sleep(100);
- utils.freeMemory();
- await db.freeMemory();
- }
- }
-
//чистка памяти, ибо жрет как не в себя
authorMap = null;
seriesMap = null;
@@ -461,25 +270,42 @@ class DbCreator {
dateMap = null;
librateMap = null;
+ await db.close({table: 'book'});
+ await db.freeMemory();
utils.freeMemory();
- //сортировка серий
- callback({job: 'sort', jobMessage: 'Сортировка', jobStep: 5, progress: 0});
+ //отсортируем таблицы выдадим им правильные id
+ //порядок id соответствует ASC-сортировке по value
+ callback({job: 'sort', jobMessage: 'Сортировка', jobStep: 2, progress: 0});
await utils.sleep(100);
- seriesArr.sort((a, b) => a.value.localeCompare(b.value));
+ //сортировка авторов
+ authorArr.sort((a, b) => a.value.localeCompare(b.value));
+ callback({progress: 0.2});
await utils.sleep(100);
+
+ id = 0;
+ for (const authorRec of authorArr) {
+ authorRec.id = ++id;
+ }
callback({progress: 0.3});
+ await utils.sleep(100);
+
+ //сортировка серий
+ seriesArr.sort((a, b) => a.value.localeCompare(b.value));
+ callback({progress: 0.5});
+ await utils.sleep(100);
+
id = 0;
for (const seriesRec of seriesArr) {
seriesRec.id = ++id;
}
+ callback({progress: 0.6});
+ await utils.sleep(100);
- await utils.sleep(100);
- callback({progress: 0.5});
- //заодно и названия
+ //сортировка названий
titleArr.sort((a, b) => a.value.localeCompare(b.value));
- await utils.sleep(100);
- callback({progress: 0.7});
+ callback({progress: 0.8});
+ await utils.sleep(100);
id = 0;
for (const titleRec of titleArr) {
titleRec.id = ++id;
@@ -507,7 +333,7 @@ class DbCreator {
//сохраним поисковые таблицы
const chunkSize = 10000;
- const saveTable = async(table, arr, nullArr, authorIdToArray = false, bookIdToArray = false, indexType = 'string') => {
+ const saveTable = async(table, arr, nullArr, indexType = 'string') => {
if (indexType == 'string')
arr.sort((a, b) => a.value.localeCompare(b.value));
@@ -523,21 +349,14 @@ class DbCreator {
for (let i = 0; i < arr.length; i += chunkSize) {
const chunk = arr.slice(i, i + chunkSize);
- if (authorIdToArray) {
- for (const rec of chunk)
- rec.authorId = Array.from(rec.authorId);
- }
-
- if (bookIdToArray) {
- for (const rec of chunk)
- rec.bookId = Array.from(rec.bookId);
- }
+ for (const rec of chunk)
+ rec.bookIds = Array.from(rec.bookIds);
await db.insert({table, rows: chunk});
if (i % 5 == 0) {
await db.freeMemory();
- await utils.sleep(100);
+ await utils.sleep(10);
}
callback({progress: i/arr.length});
@@ -550,33 +369,33 @@ class DbCreator {
};
//author
- callback({job: 'author save', jobMessage: 'Сохранение индекса авторов', jobStep: 6, progress: 0});
+ callback({job: 'author save', jobMessage: 'Сохранение индекса авторов', jobStep: 3, progress: 0});
await saveTable('author', authorArr, () => {authorArr = null});
//series
- callback({job: 'series save', jobMessage: 'Сохранение индекса серий', jobStep: 7, progress: 0});
- await saveTable('series', seriesArr, () => {seriesArr = null}, true, true);
+ callback({job: 'series save', jobMessage: 'Сохранение индекса серий', jobStep: 4, progress: 0});
+ await saveTable('series', seriesArr, () => {seriesArr = null});
//title
- callback({job: 'title save', jobMessage: 'Сохранение индекса названий', jobStep: 8, progress: 0});
- await saveTable('title', titleArr, () => {titleArr = null}, true, true);
+ callback({job: 'title save', jobMessage: 'Сохранение индекса названий', jobStep: 5, progress: 0});
+ await saveTable('title', titleArr, () => {titleArr = null});
//genre
- callback({job: 'genre save', jobMessage: 'Сохранение индекса жанров', jobStep: 9, progress: 0});
- await saveTable('genre', genreArr, () => {genreArr = null}, true);
+ callback({job: 'genre save', jobMessage: 'Сохранение индекса жанров', jobStep: 6, progress: 0});
+ await saveTable('genre', genreArr, () => {genreArr = null});
- callback({job: 'others save', jobMessage: 'Сохранение остальных индексов', jobStep: 10, progress: 0});
+ callback({job: 'others save', jobMessage: 'Сохранение остальных индексов', jobStep: 7, progress: 0});
//lang
- await saveTable('lang', langArr, () => {langArr = null}, true);
+ await saveTable('lang', langArr, () => {langArr = null});
//del
- await saveTable('del', delArr, () => {delArr = null}, true, false, 'number');
+ await saveTable('del', delArr, () => {delArr = null}, 'number');
//date
- await saveTable('date', dateArr, () => {dateArr = null}, true);
+ await saveTable('date', dateArr, () => {dateArr = null});
//librate
- await saveTable('librate', librateArr, () => {librateArr = null}, true, false, 'number');
+ await saveTable('librate', librateArr, () => {librateArr = null}, 'number');
//кэш-таблицы запросов
await db.create({table: 'query_cache'});
@@ -591,23 +410,28 @@ class DbCreator {
cacheSize: (config.lowMemoryMode ? 5 : 500),
});
- callback({job: 'optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0});
- await this.optimizeTable('series', 'series_book', 'series', db, (p) => {
+ callback({job: 'optimization', jobMessage: 'Оптимизация', jobStep: 8, progress: 0});
+ await this.optimizeTable('author', db, (p) => {
if (p.progress)
- p.progress = 0.2*p.progress;
+ p.progress = 0.3*p.progress;
callback(p);
});
- await this.optimizeTable('title', 'title_book', 'title', db, (p) => {
+ await this.optimizeTable('series', db, (p) => {
if (p.progress)
- p.progress = 0.2 + 0.8*p.progress;
+ p.progress = 0.3 + 0.2*p.progress;
+ callback(p);
+ });
+ await this.optimizeTable('title', db, (p) => {
+ if (p.progress)
+ p.progress = 0.5 + 0.5*p.progress;
callback(p);
});
- callback({job: 'stats count', jobMessage: 'Подсчет статистики', jobStep: 12, progress: 0});
+ callback({job: 'stats count', jobMessage: 'Подсчет статистики', jobStep: 9, progress: 0});
await this.countStats(db, callback, stats);
//чистка памяти, ибо жрет как не в себя
- await db.drop({table: 'book'});//больше не понадобится
+ await db.close({table: 'book'});
await db.freeMemory();
utils.freeMemory();
@@ -627,61 +451,61 @@ class DbCreator {
callback({job: 'done', jobMessage: ''});
}
- async optimizeTable(from, to, restoreProp, db, callback) {
- //оптимизация таблицы from, превращаем массив bookId в books, кладем все в таблицу to
- await db.open({table: from});
+ async optimizeTable(from, db, callback) {
+ const config = this.config;
- await db.create({
- table: to,
- flag: {name: 'toDel', check: 'r => r.toDel'},
- });
+ const to = `${from}_book`;
+ const toId = `${from}_id`;
+
+ await db.open({table: from});
+ await db.create({table: to});
+
+ const bookId2RecId = new Map();
const saveChunk = async(chunk) => {
const ids = [];
- for (const s of chunk) {
- for (const id of s.bookId) {
+ for (const rec of chunk) {
+ for (const id of rec.bookIds) {
+ let b2r = bookId2RecId.get(id);
+ if (!b2r) {
+ b2r = [];
+ bookId2RecId.set(id, b2r);
+ }
+ b2r.push(rec.id);
+
ids.push(id);
}
}
- ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb
+ if (config.fullOptimization) {
+ ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb
- const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
+ const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
- const bookArr = new Map();
- for (const row of rows)
- bookArr.set(row.id, row);
+ const bookArr = new Map();
+ for (const row of rows)
+ bookArr.set(row.id, row);
- for (const s of chunk) {
- s.books = [];
- s.bookCount = 0;
- s.bookDelCount = 0;
- for (const id of s.bookId) {
- const rec = bookArr.get(id);
- if (rec) {//на всякий случай
- s.books.push(rec);
- if (!rec.del)
- s.bookCount++;
- else
- s.bookDelCount++;
+ for (const rec of chunk) {
+ rec.books = [];
+
+ for (const id of rec.bookIds) {
+ const book = bookArr.get(id);
+ if (book) {//на всякий случай
+ rec.books.push(book);
+ }
}
+
+ delete rec.name;
+ delete rec.value;
+ delete rec.bookIds;
}
- if (s.books.length) {
- s[restoreProp] = s.books[0][restoreProp];
- } else {
- s.toDel = 1;
- }
-
- delete s.value;
- delete s.authorId;
- delete s.bookId;
+ await db.insert({
+ table: to,
+ rows: chunk,
+ });
}
-
- await db.insert({
- table: to,
- rows: chunk,
- });
};
const rows = await db.select({table: from, count: true});
@@ -699,11 +523,16 @@ class DbCreator {
}
const ids = new Set();
+ let bookIdsLen = 0;
let id = iter.next();
while (!id.done) {
ids.add(id.value);
- if (ids.size >= 20000)
+
+ const row = @row(id.value);
+ bookIdsLen += row.bookIds.length;
+ if (bookIdsLen >= 50000)
break;
+
id = iter.next();
}
@@ -726,9 +555,16 @@ class DbCreator {
}
}
- await db.delete({table: to, where: `@@flag('toDel')`});
await db.close({table: to});
await db.close({table: from});
+
+ await db.create({table: toId});
+ const idRows = [];
+ for (const [id, value] of bookId2RecId) {
+ idRows.push({id, value});
+ }
+ await db.insert({table: toId, rows: idRows});
+ await db.close({table: toId});
}
async countStats(db, callback, stats) {
diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js
index 7d2da99..8c67fdc 100644
--- a/server/core/DbSearcher.js
+++ b/server/core/DbSearcher.js
@@ -20,6 +20,7 @@ class DbSearcher {
this.closed = false;
this.memCache = new Map();
+ this.bookIdMap = {};
this.periodicCleanCache();//no await
}
@@ -54,116 +55,80 @@ class DbSearcher {
return where;
}
- async selectAuthorIds(query) {
+ async selectBookIds(query) {
const db = this.db;
- const authorKеy = `author-ids-author-${query.author}`;
- let authorIds = await this.getCached(authorKеy);
-
- //сначала выберем все id авторов по фильтру
- //порядок id соответствует ASC-сортировке по author
- if (authorIds === null) {
- if (query.author && query.author !== '*') {
- const where = this.getWhere(query.author);
-
- const authorRows = await db.select({
- table: 'author',
- rawResult: true,
- where: `return Array.from(${where})`,
- });
-
- authorIds = authorRows[0].rawResult;
- } else {//все авторы
- const authorRows = await db.select({
- table: 'author',
- rawResult: true,
- where: `return Array.from(@all())`,
- });
-
- authorIds = authorRows[0].rawResult;
- }
-
- await this.putCached(authorKеy, authorIds);
- }
-
const idsArr = [];
- //серии
- if (query.series && query.series !== '*') {
- const seriesKеy = `author-ids-series-${query.series}`;
- let seriesIds = await this.getCached(seriesKеy);
+ const tableBookIds = async(table, where) => {
+ const rows = await db.select({
+ table,
+ rawResult: true,
+ where: `
+ const ids = ${where};
- if (seriesIds === null) {
- const where = this.getWhere(query.series);
+ const result = new Set();
+ for (const id of ids) {
+ const row = @unsafeRow(id);
+ for (const bookId of row.bookIds)
+ result.add(bookId);
+ }
- const seriesRows = await db.select({
- table: 'series',
- rawResult: true,
- where: `
- const ids = ${where};
+ return Array.from(result);
+ `
+ });
- const result = new Set();
- for (const id of ids) {
- const row = @unsafeRow(id);
- for (const authorId of row.authorId)
- result.add(authorId);
- }
+ return rows[0].rawResult;
+ };
- return Array.from(result);
- `
- });
+ //авторы
+ if (query.author && query.author !== '*') {
+ const key = `book-ids-author-${query.author}`;
+ let ids = await this.getCached(key);
- seriesIds = seriesRows[0].rawResult;
- await this.putCached(seriesKеy, seriesIds);
+ if (ids === null) {
+ ids = await tableBookIds('author', this.getWhere(query.author));
+
+ await this.putCached(key, ids);
}
- idsArr.push(seriesIds);
+ idsArr.push(ids);
+ }
+
+ //серии
+ if (query.series && query.series !== '*') {
+ const key = `book-ids-series-${query.series}`;
+ let ids = await this.getCached(key);
+
+ if (ids === null) {
+ ids = await tableBookIds('series', this.getWhere(query.series));
+
+ await this.putCached(key, ids);
+ }
+
+ idsArr.push(ids);
}
//названия
if (query.title && query.title !== '*') {
- const titleKey = `author-ids-title-${query.title}`;
- let titleIds = await this.getCached(titleKey);
+ const key = `book-ids-title-${query.title}`;
+ let ids = await this.getCached(key);
- if (titleIds === null) {
- const where = this.getWhere(query.title);
+ if (ids === null) {
+ ids = await tableBookIds('title', this.getWhere(query.title));
- let titleRows = await db.select({
- table: 'title',
- rawResult: true,
- where: `
- const ids = ${where};
-
- const result = new Set();
- for (const id of ids) {
- const row = @unsafeRow(id);
- for (const authorId of row.authorId)
- result.add(authorId);
- }
-
- return Array.from(result);
- `
- });
-
- titleIds = titleRows[0].rawResult;
- await this.putCached(titleKey, titleIds);
+ await this.putCached(key, ids);
}
- idsArr.push(titleIds);
-
- //чистки памяти при тяжелых запросах
- if (this.config.lowMemoryMode && query.title[0] == '*') {
- utils.freeMemory();
- await db.freeMemory();
- }
+ idsArr.push(ids);
}
//жанры
if (query.genre) {
- const genreKey = `author-ids-genre-${query.genre}`;
- let genreIds = await this.getCached(genreKey);
+ const key = `book-ids-genre-${query.genre}`;
+ let ids = await this.getCached(key);
- if (genreIds === null) {
+ if (ids === null) {
const genreRows = await db.select({
table: 'genre',
rawResult: true,
@@ -179,27 +144,27 @@ class DbSearcher {
const result = new Set();
for (const id of ids) {
const row = @unsafeRow(id);
- for (const authorId of row.authorId)
- result.add(authorId);
+ for (const bookId of row.bookIds)
+ result.add(bookId);
}
return Array.from(result);
`
});
- genreIds = genreRows[0].rawResult;
- await this.putCached(genreKey, genreIds);
+ ids = genreRows[0].rawResult;
+ await this.putCached(key, ids);
}
- idsArr.push(genreIds);
+ idsArr.push(ids);
}
//языки
if (query.lang) {
- const langKey = `author-ids-lang-${query.lang}`;
- let langIds = await this.getCached(langKey);
+ const key = `book-ids-lang-${query.lang}`;
+ let ids = await this.getCached(key);
- if (langIds === null) {
+ if (ids === null) {
const langRows = await db.select({
table: 'lang',
rawResult: true,
@@ -215,89 +180,56 @@ class DbSearcher {
const result = new Set();
for (const id of ids) {
const row = @unsafeRow(id);
- for (const authorId of row.authorId)
- result.add(authorId);
+ for (const bookId of row.bookIds)
+ result.add(bookId);
}
return Array.from(result);
`
});
- langIds = langRows[0].rawResult;
- await this.putCached(langKey, langIds);
+ ids = langRows[0].rawResult;
+ await this.putCached(key, ids);
}
- idsArr.push(langIds);
+ idsArr.push(ids);
}
//удаленные
if (query.del !== undefined) {
- const delKey = `author-ids-del-${query.del}`;
- let delIds = await this.getCached(delKey);
+ const key = `book-ids-del-${query.del}`;
+ let ids = await this.getCached(key);
- if (delIds === null) {
- const delRows = await db.select({
- table: 'del',
- rawResult: true,
- where: `
- const ids = @indexLR('value', ${db.esc(query.del)}, ${db.esc(query.del)});
-
- const result = new Set();
- for (const id of ids) {
- const row = @unsafeRow(id);
- for (const authorId of row.authorId)
- result.add(authorId);
- }
+ if (ids === null) {
+ ids = await tableBookIds('del', `@indexLR('value', ${db.esc(query.del)}, ${db.esc(query.del)})`);
- return Array.from(result);
- `
- });
-
- delIds = delRows[0].rawResult;
- await this.putCached(delKey, delIds);
+ await this.putCached(key, ids);
}
- idsArr.push(delIds);
+ idsArr.push(ids);
}
//дата поступления
if (query.date) {
- const dateKey = `author-ids-date-${query.date}`;
- let dateIds = await this.getCached(dateKey);
+ const key = `book-ids-date-${query.date}`;
+ let ids = await this.getCached(key);
- if (dateIds === null) {
+ if (ids === null) {
let [from = '', to = ''] = query.date.split(',');
+ ids = await tableBookIds('date', `@indexLR('value', ${db.esc(from)} || undefined, ${db.esc(to)} || undefined)`);
- const dateRows = await db.select({
- table: 'date',
- rawResult: true,
- where: `
- const ids = @indexLR('value', ${db.esc(from)} || undefined, ${db.esc(to)} || undefined);
-
- const result = new Set();
- for (const id of ids) {
- const row = @unsafeRow(id);
- for (const authorId of row.authorId)
- result.add(authorId);
- }
-
- return Array.from(result);
- `
- });
-
- dateIds = dateRows[0].rawResult;
- await this.putCached(dateKey, dateIds);
+ await this.putCached(key, ids);
}
- idsArr.push(dateIds);
+ idsArr.push(ids);
}
//оценка
if (query.librate) {
- const librateKey = `author-ids-librate-${query.librate}`;
- let librateIds = await this.getCached(librateKey);
+ const key = `book-ids-librate-${query.librate}`;
+ let ids = await this.getCached(key);
- if (librateIds === null) {
+ if (ids === null) {
const dateRows = await db.select({
table: 'librate',
rawResult: true,
@@ -313,37 +245,23 @@ class DbSearcher {
const result = new Set();
for (const id of ids) {
const row = @unsafeRow(id);
- for (const authorId of row.authorId)
- result.add(authorId);
+ for (const bookId of row.bookIds)
+ result.add(bookId);
}
return Array.from(result);
`
});
- librateIds = dateRows[0].rawResult;
- await this.putCached(librateKey, librateIds);
+ ids = dateRows[0].rawResult;
+ await this.putCached(key, ids);
}
- idsArr.push(librateIds);
+ idsArr.push(ids);
}
-/*
- //ищем пересечение множеств
- idsArr.push(authorIds);
if (idsArr.length > 1) {
- const idsSetArr = idsArr.map(ids => new Set(ids));
- authorIds = Array.from(utils.intersectSet(idsSetArr));
- }
-
- //сортировка
- authorIds.sort((a, b) => a - b);
-*/
-
- //ищем пересечение множеств, работает быстрее предыдущего
- if (idsArr.length) {
- idsArr.push(authorIds);
-
+ //ищем пересечение множеств
let proc = 0;
let nextProc = 0;
let inter = new Set(idsArr[0]);
@@ -363,340 +281,271 @@ class DbSearcher {
}
inter = newInter;
}
- authorIds = Array.from(inter);
- }
- //сортировка
- authorIds.sort((a, b) => a - b);
- return authorIds;
+ return Array.from(inter);
+ } else if (idsArr.length == 1) {
+ return idsArr[0];
+ } else {
+ return false;
+ }
}
- getWhere2(query, ids, exclude = '') {
- const db = this.db;
+ async fillBookIdMap(from) {
+ if (!this.bookIdMap[from]) {
+ const db = this.db;
+ const map = new Map();
+ const table = `${from}_id`;
- const filterBySearch = (searchValue) => {
- searchValue = searchValue.toLowerCase();
+ await db.open({table});
+ const rows = await db.select({table});
+ await db.close({table});
- //особая обработка префиксов
- if (searchValue[0] == '=') {
+ for (const row of rows) {
+ if (!row.value.length)
+ continue;
- searchValue = searchValue.substring(1);
- return `bookValue.localeCompare(${db.esc(searchValue)}) == 0`;
- } else if (searchValue[0] == '*') {
-
- searchValue = searchValue.substring(1);
- return `bookValue !== ${db.esc(emptyFieldValue)} && bookValue.indexOf(${db.esc(searchValue)}) >= 0`;
- } else if (searchValue[0] == '#') {
-
- searchValue = searchValue.substring(1);
- return `!bookValue || (bookValue !== ${db.esc(emptyFieldValue)} && !enru.has(bookValue[0]) && bookValue.indexOf(${db.esc(searchValue)}) >= 0)`;
- } else {
- return `bookValue.localeCompare(${db.esc(searchValue)}) >= 0 && bookValue.localeCompare(${db.esc(searchValue + maxUtf8Char)}) <= 0`;
+ if (row.value.length > 1)
+ map.set(row.id, row.value);
+ else
+ map.set(row.id, row.value[0]);
}
- };
- //подготовка фильтра
- let filter = '';
- let closures = '';
-
- //порядок важен, более простые проверки вперед
-
- //удаленные
- if (query.del !== undefined) {
- filter += `
- if (book.del !== ${db.esc(query.del)})
- return false;
- `;
+ this.bookIdMap[from] = map;
}
-
- //дата поступления
- if (query.date) {
- let [from = '0000-00-00', to = '9999-99-99'] = query.date.split(',');
- filter += `
- if (!(book.date >= ${db.esc(from)} && book.date <= ${db.esc(to)}))
- return false;
- `;
- }
-
- //оценка
- if (query.librate) {
- closures += `
- const searchLibrate = new Set(${db.esc(query.librate.split(',').map(n => parseInt(n, 10)).filter(n => !isNaN(n)))});
- `;
- filter += `
- if (!searchLibrate.has(book.librate))
- return false;
- `;
- }
-
- //серии
- if (exclude !== 'series' && query.series && query.series !== '*') {
- closures += `
- const checkSeries = (bookValue) => {
- if (!bookValue)
- bookValue = ${db.esc(emptyFieldValue)};
-
- bookValue = bookValue.toLowerCase();
-
- return ${filterBySearch(query.series)};
- };
- `;
- filter += `
- if (!checkSeries(book.series))
- return false;
- `;
- }
-
- //названия
- if (exclude !== 'title' && query.title && query.title !== '*') {
- closures += `
- const checkTitle = (bookValue) => {
- if (!bookValue)
- bookValue = ${db.esc(emptyFieldValue)};
-
- bookValue = bookValue.toLowerCase();
-
- return ${filterBySearch(query.title)};
- };
- `;
- filter += `
- if (!checkTitle(book.title))
- return false;
- `;
- }
-
- //языки
- if (exclude !== 'lang' && query.lang) {
- const queryLangs = query.lang.split(',');
-
- closures += `
- const queryLangs = new Set(${db.esc(queryLangs)});
-
- const checkLang = (bookValue) => {
- if (!bookValue)
- bookValue = ${db.esc(emptyFieldValue)};
-
- return queryLangs.has(bookValue);
- };
- `;
- filter += `
- if (!checkLang(book.lang))
- return false;
- `;
- }
-
- //жанры
- if (exclude !== 'genre' && query.genre) {
- const queryGenres = query.genre.split(',');
-
- closures += `
- const queryGenres = new Set(${db.esc(queryGenres)});
-
- const checkGenre = (bookValue) => {
- if (!bookValue)
- bookValue = ${db.esc(emptyFieldValue)};
-
- return queryGenres.has(bookValue);
- };
- `;
- filter += `
- const genres = book.genre.split(',');
- found = false;
- for (const g of genres) {
- if (checkGenre(g)) {
- found = true;
- break;
- }
- }
-
- if (!found)
- return false;
- `;
- }
-
- //авторы
- if (exclude !== 'author' && query.author && query.author !== '*') {
- closures += `
- const splitAuthor = (author) => {
- if (!author)
- author = ${db.esc(emptyFieldValue)};
-
- const result = author.split(',');
- if (result.length > 1)
- result.push(author);
-
- return result;
- };
-
- const checkAuthor = (bookValue) => {
- if (!bookValue)
- bookValue = ${db.esc(emptyFieldValue)};
-
- bookValue = bookValue.toLowerCase();
-
- return ${filterBySearch(query.author)};
- };
- `;
-
- filter += `
- const author = splitAuthor(book.author);
- found = false;
- for (const a of author) {
- if (checkAuthor(a)) {
- found = true;
- break;
- }
- }
-
- if (!found)
- return false;
- `;
- }
-
- //формируем where
- let where = '';
- if (filter) {
- where = `
- const enru = new Set(${db.esc(enruArr)});
-
- ${closures}
-
- const filterBook = (book) => {
- let found = false;
- ${filter}
- return true;
- };
-
- let ids;
- if (${!ids}) {
- ids = @all();
- } else {
- ids = ${db.esc(ids)};
- }
-
- const result = new Set();
- for (const id of ids) {
- const row = @unsafeRow(id);
-
- if (row) {
- for (const book of row.books) {
- if (filterBook(book)) {
- result.add(id);
- break;
- }
- }
- }
- }
-
- return Array.from(result);
- `;
- }
-
- return where;
+ return this.bookIdMap[from];
}
- async selectSeriesIds(query) {
- const db = this.db;
+ async filterTableIds(tableIds, from, query) {
+ let result = tableIds;
- let seriesIds = false;
- let isAll = !(query.series && query.series !== '*');
+ //т.к. авторы у книги идут списком, то дополнительно фильтруем
+ if (from == 'author' && query.author && query.author !== '*') {
+ const key = `filter-ids-author-${query.author}`;
+ let authorIds = await this.getCached(key);
- //серии
- const seriesKеy = `series-ids-series-${query.series}`;
- seriesIds = await this.getCached(seriesKеy);
-
- if (seriesIds === null) {
- if (query.series && query.series !== '*') {
- const where = this.getWhere(query.series);
-
- const seriesRows = await db.select({
- table: 'series',
+ if (authorIds === null) {
+ const rows = await this.db.select({
+ table: 'author',
rawResult: true,
- where: `return Array.from(${where})`,
+ where: `return Array.from(${this.getWhere(query.author)})`
});
- seriesIds = seriesRows[0].rawResult;
- } else {
- const seriesRows = await db.select({
- table: 'series',
- rawResult: true,
- where: `return Array.from(@all())`,
- });
+ authorIds = rows[0].rawResult;
- seriesIds = seriesRows[0].rawResult;
+ await this.putCached(key, authorIds);
}
- seriesIds.sort((a, b) => a - b);
-
- await this.putCached(seriesKеy, seriesIds);
+ //пересечение tableIds и authorIds
+ result = [];
+ const authorIdsSet = new Set(authorIds);
+ for (const id of tableIds)
+ if (authorIdsSet.has(id))
+ result.push(id);
}
- const where = this.getWhere2(query, (isAll ? false : seriesIds), 'series');
+ return result;
+ }
- if (where) {
- //тяжелый запрос перебором в series_book
- const rows = await db.select({
- table: 'series_book',
- rawResult: true,
- where,
+ async selectTableIds(from, query) {
+ const db = this.db;
+ const queryKey = this.queryKey(query);
+ const tableKey = `${from}-table-ids-${queryKey}`;
+ let tableIds = await this.getCached(tableKey);
+
+ if (tableIds === null) {
+ const bookKey = `book-ids-${queryKey}`;
+ let bookIds = await this.getCached(bookKey);
+
+ if (bookIds === null) {
+ bookIds = await this.selectBookIds(query);
+ await this.putCached(bookKey, bookIds);
+ }
+
+ if (bookIds) {
+ const tableIdsSet = new Set();
+ const bookIdMap = await this.fillBookIdMap(from);
+ for (const bookId of bookIds) {
+ const tableIdValue = bookIdMap.get(bookId);
+ if (!tableIdValue)
+ continue;
+
+ if (Array.isArray(tableIdValue)) {
+ for (const tableId of tableIdValue)
+ tableIdsSet.add(tableId);
+ } else
+ tableIdsSet.add(tableIdValue);
+ }
+
+ tableIds = Array.from(tableIdsSet);
+ } else {
+ const rows = await db.select({
+ table: from,
+ rawResult: true,
+ where: `return Array.from(@all())`
+ });
+
+ tableIds = rows[0].rawResult;
+ }
+
+ tableIds = await this.filterTableIds(tableIds, from, query);
+
+ tableIds.sort((a, b) => a - b);
+
+ await this.putCached(tableKey, tableIds);
+ }
+
+ return tableIds;
+ }
+
+ async restoreBooks(from, ids) {
+ const db = this.db;
+ const bookTable = `${from}_book`;
+
+ const rows = await db.select({
+ table: bookTable,
+ where: `@@id(${db.esc(ids)})`
+ });
+
+ if (rows.length == ids.length)
+ return rows;
+
+ const idsSet = new Set(rows.map(r => r.id));
+
+ for (const id of ids) {
+ if (!idsSet.has(id)) {
+ const bookIds = await db.select({
+ table: from,
+ where: `@@id(${db.esc(id)})`
+ });
+
+ if (!bookIds.length)
+ continue;
+
+ let books = await db.select({
+ table: 'book',
+ where: `@@id(${db.esc(bookIds[0].bookIds)})`
+ });
+
+ if (!books.length)
+ continue;
+
+ rows.push({id, name: bookIds[0].name, books});
+
+ await db.insert({table: bookTable, ignore: true, rows});
+ }
+ }
+
+ return rows;
+ }
+
+ async search(from, query) {
+ if (this.closed)
+ throw new Error('DbSearcher closed');
+
+ if (!['author', 'series', 'title'].includes(from))
+ throw new Error(`Unknown value for param 'from'`);
+
+ this.searchFlag++;
+
+ try {
+ const db = this.db;
+
+ const ids = await this.selectTableIds(from, query);
+
+ const totalFound = ids.length;
+ let limit = (query.limit ? query.limit : 100);
+ limit = (limit > maxLimit ? maxLimit : limit);
+ const offset = (query.offset ? query.offset : 0);
+
+ //выборка найденных значений
+ const found = await db.select({
+ table: from,
+ map: `(r) => ({id: r.id, ${from}: r.name, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
+ where: `@@id(${db.esc(ids.slice(offset, offset + limit))})`
});
- seriesIds = rows[0].rawResult;
- }
+ //для title восстановим books
+ if (from == 'title') {
+ const bookIds = found.map(r => r.id);
+ const rows = await this.restoreBooks(from, bookIds);
+ const rowsMap = new Map();
+ for (const row of rows)
+ rowsMap.set(row.id, row);
- return seriesIds;
- }
-
- async selectTitleIds(query) {
- const db = this.db;
-
- let titleIds = false;
- let isAll = !(query.title && query.title !== '*');
-
- //серии
- const titleKеy = `title-ids-title-${query.title}`;
- titleIds = await this.getCached(titleKеy);
-
- if (titleIds === null) {
- if (query.title && query.title !== '*') {
- const where = this.getWhere(query.title);
-
- const titleRows = await db.select({
- table: 'title',
- rawResult: true,
- where: `return Array.from(${where})`,
- });
-
- titleIds = titleRows[0].rawResult;
- } else {
- const titleRows = await db.select({
- table: 'title',
- rawResult: true,
- where: `return Array.from(@all())`,
- });
-
- titleIds = titleRows[0].rawResult;
+ for (const f of found) {
+ const b = rowsMap.get(f.id);
+ if (b)
+ f.books = b.books;
+ }
}
- titleIds.sort((a, b) => a - b);
-
- await this.putCached(titleKеy, titleIds);
+ return {found, totalFound};
+ } finally {
+ this.searchFlag--;
}
+ }
- const where = this.getWhere2(query, (isAll ? false : titleIds), 'title');
+ async getAuthorBookList(authorId) {
+ if (this.closed)
+ throw new Error('DbSearcher closed');
- if (where) {
- //тяжелый запрос перебором в title_book
- const rows = await db.select({
- table: 'title_book',
+ if (!authorId)
+ return {author: '', books: ''};
+
+ this.searchFlag++;
+
+ try {
+ //выборка книг автора по authorId
+ const rows = await this.restoreBooks('author', [authorId])
+
+ let author = '';
+ let books = '';
+
+ if (rows.length) {
+ author = rows[0].name;
+ books = rows[0].books;
+ }
+
+ return {author, books: (books && books.length ? JSON.stringify(books) : '')};
+ } finally {
+ this.searchFlag--;
+ }
+ }
+
+ async getSeriesBookList(series) {
+ if (this.closed)
+ throw new Error('DbSearcher closed');
+
+ if (!series)
+ return {books: ''};
+
+ this.searchFlag++;
+
+ try {
+ const db = this.db;
+
+ series = series.toLowerCase();
+
+ //выборка серии по названию серии
+ let rows = await db.select({
+ table: 'series',
rawResult: true,
- where,
+ where: `return Array.from(@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)}))`
});
- titleIds = rows[0].rawResult;
- }
+ let books;
+ if (rows.length && rows[0].rawResult.length) {
+ //выборка книг серии
+ const bookRows = await this.restoreBooks('series', [rows[0].rawResult[0]])
- return titleIds;
+ if (bookRows.length)
+ books = bookRows[0].books;
+ }
+
+ return {books: (books && books.length ? JSON.stringify(books) : '')};
+ } finally {
+ this.searchFlag--;
+ }
}
async getCached(key) {
@@ -757,200 +606,24 @@ class DbSearcher {
}
}
- //кладем в таблицу
- await db.insert({
- table: 'query_cache',
- replace: true,
- rows: [{id: key, value}],
- });
-
- await db.insert({
- table: 'query_time',
- replace: true,
- rows: [{id: key, time: Date.now()}],
- });
- }
-
- async authorSearch(query) {
- if (this.closed)
- throw new Error('DbSearcher closed');
-
- this.searchFlag++;
-
- try {
- const db = this.db;
-
- const key = `author-ids-${this.queryKey(query)}`;
-
- //сначала попробуем найти в кеше
- let authorIds = await this.getCached(key);
- if (authorIds === null) {//не нашли в кеше, ищем в поисковых таблицах
- authorIds = await this.selectAuthorIds(query);
-
- await this.putCached(key, authorIds);
- }
-
- const totalFound = authorIds.length;
- let limit = (query.limit ? query.limit : 100);
- limit = (limit > maxLimit ? maxLimit : limit);
- const offset = (query.offset ? query.offset : 0);
-
- //выборка найденных авторов
- const result = await db.select({
- table: 'author',
- map: `(r) => ({id: r.id, author: r.author, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
- where: `@@id(${db.esc(authorIds.slice(offset, offset + limit))})`
- });
-
- return {result, totalFound};
- } finally {
- this.searchFlag--;
- }
- }
-
- async seriesSearch(query) {
- if (this.closed)
- throw new Error('DbSearcher closed');
-
- this.searchFlag++;
-
- try {
- const db = this.db;
-
- const key = `series-ids-${this.queryKey(query)}`;
-
- //сначала попробуем найти в кеше
- let seriesIds = await this.getCached(key);
- if (seriesIds === null) {//не нашли в кеше, ищем в поисковых таблицах
- seriesIds = await this.selectSeriesIds(query);
-
- await this.putCached(key, seriesIds);
- }
-
- const totalFound = seriesIds.length;
- let limit = (query.limit ? query.limit : 100);
- limit = (limit > maxLimit ? maxLimit : limit);
- const offset = (query.offset ? query.offset : 0);
-
- //выборка найденных авторов
- const result = await db.select({
- table: 'series_book',
- map: `(r) => ({id: r.id, series: r.series, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
- where: `@@id(${db.esc(seriesIds.slice(offset, offset + limit))})`
- });
-
- return {result, totalFound};
- } finally {
- this.searchFlag--;
- }
- }
-
- async titleSearch(query) {
- if (this.closed)
- throw new Error('DbSearcher closed');
-
- this.searchFlag++;
-
- try {
- const db = this.db;
-
- const key = `title-ids-${this.queryKey(query)}`;
-
- //сначала попробуем найти в кеше
- let titleIds = await this.getCached(key);
- if (titleIds === null) {//не нашли в кеше, ищем в поисковых таблицах
- titleIds = await this.selectTitleIds(query);
-
- await this.putCached(key, titleIds);
- }
-
- const totalFound = titleIds.length;
- let limit = (query.limit ? query.limit : 100);
- limit = (limit > maxLimit ? maxLimit : limit);
- const offset = (query.offset ? query.offset : 0);
-
- //выборка найденных авторов
- const result = await db.select({
- table: 'title_book',
- map: `(r) => ({id: r.id, title: r.title, books: r.books, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
- where: `@@id(${db.esc(titleIds.slice(offset, offset + limit))})`
- });
-
- return {result, totalFound};
- } finally {
- this.searchFlag--;
- }
- }
-
- async getAuthorBookList(authorId) {
- if (this.closed)
- throw new Error('DbSearcher closed');
-
- if (!authorId)
- return {author: '', books: ''};
-
- this.searchFlag++;
-
- try {
- const db = this.db;
-
- //выборка книг автора по authorId
- const rows = await db.select({
- table: 'author_book',
- where: `@@id(${db.esc(authorId)})`
- });
-
- let author = '';
- let books = '';
-
- if (rows.length) {
- author = rows[0].author;
- books = rows[0].books;
- }
-
- return {author, books};
- } finally {
- this.searchFlag--;
- }
- }
-
- async getSeriesBookList(series) {
- if (this.closed)
- throw new Error('DbSearcher closed');
-
- if (!series)
- return {books: ''};
-
- this.searchFlag++;
-
- try {
- const db = this.db;
-
- series = series.toLowerCase();
-
- //выборка серии по названию серии
- let rows = await db.select({
- table: 'series',
- rawResult: true,
- where: `return Array.from(@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)}))`
- });
-
- let books;
- if (rows.length && rows[0].rawResult.length) {
- //выборка книг серии
- rows = await db.select({
- table: 'series_book',
- where: `@@id(${rows[0].rawResult[0]})`
+ //кладем в таблицу асинхронно
+ (async() => {
+ try {
+ await db.insert({
+ table: 'query_cache',
+ replace: true,
+ rows: [{id: key, value}],
});
- if (rows.length)
- books = rows[0].books;
+ await db.insert({
+ table: 'query_time',
+ replace: true,
+ rows: [{id: key, time: Date.now()}],
+ });
+ } catch(e) {
+ console.error(`putCached: ${e.message}`);
}
-
- return {books: (books && books.length ? JSON.stringify(books) : '')};
- } finally {
- this.searchFlag--;
- }
+ })();
}
async periodicCleanCache() {
diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js
index 2c5fdaa..895d0ab 100644
--- a/server/core/WebWorker.js
+++ b/server/core/WebWorker.js
@@ -187,14 +187,8 @@ class WebWorker {
},
});
- //открываем почти все таблицы
- await db.openAll({exclude: ['author', 'title_book']});
-
- //откроем таблицу 'author' с бОльшим размером кеша блоков, для ускорения выборки
- await db.open({table: 'author', cacheSize: (config.dbCacheSize > 100 ? config.dbCacheSize : 100)});
-
- if (config.extendedSearch)
- await db.open({table: 'title_book'});
+ //открываем таблицы
+ await db.openAll({exclude: ['author_id', 'series_id', 'title_id']});
this.dbSearcher = new DbSearcher(config, db);
@@ -242,43 +236,15 @@ class WebWorker {
return db.wwCache.config;
}
- async authorSearch(query) {
+ async search(from, query) {
this.checkMyState();
- const config = await this.dbConfig();
- const result = await this.dbSearcher.authorSearch(query);
-
- return {
- author: result.result,
- totalFound: result.totalFound,
- inpxHash: (config.inpxHash ? config.inpxHash : ''),
- };
- }
-
- async seriesSearch(query) {
- this.checkMyState();
+ const result = await this.dbSearcher.search(from, query);
const config = await this.dbConfig();
- const result = await this.dbSearcher.seriesSearch(query);
+ result.inpxHash = (config.inpxHash ? config.inpxHash : '');
- return {
- series: result.result,
- totalFound: result.totalFound,
- inpxHash: (config.inpxHash ? config.inpxHash : ''),
- };
- }
-
- async titleSearch(query) {
- this.checkMyState();
-
- const config = await this.dbConfig();
- const result = await this.dbSearcher.titleSearch(query);
-
- return {
- title: result.result,
- totalFound: result.totalFound,
- inpxHash: (config.inpxHash ? config.inpxHash : ''),
- };
+ return result;
}
async getAuthorBookList(authorId) {