Работа над новым поиском

This commit is contained in:
Book Pauk
2022-10-30 13:55:54 +07:00
parent 2ba8297594
commit 3f39b55ca0
6 changed files with 187 additions and 531 deletions

View File

@@ -218,16 +218,8 @@ class Api {
} }
} }
async authorSearch(query) { async search(from, query) {
return await this.request({action: 'author-search', query}); return await this.request({action: 'search', from, query});
}
async seriesSearch(query) {
return await this.request({action: 'series-search', query});
}
async titleSearch(query) {
return await this.request({action: 'title-search', query});
} }
async getAuthorBookList(authorId) { async getAuthorBookList(authorId) {

View File

@@ -317,7 +317,7 @@ class AuthorList extends BaseList {
let result = []; let result = [];
const expandedSet = new Set(this.expandedAuthor); const expandedSet = new Set(this.expandedAuthor);
const authors = this.searchResult.author; const authors = this.searchResult.found;
if (!authors) if (!authors)
return; return;
@@ -399,7 +399,7 @@ class AuthorList extends BaseList {
(async() => { (async() => {
await utils.sleep(500); await utils.sleep(500);
if (this.refreshing) if (this.refreshing)
this.loadingMessage = 'Поиск серий...'; this.loadingMessage = 'Поиск авторов...';
})(); })();
try { try {
@@ -408,13 +408,13 @@ class AuthorList extends BaseList {
this.queryExecute = null; this.queryExecute = null;
try { try {
const result = await this.api.authorSearch(query); const response = await this.api.search('author', query);
this.list.queryFound = result.author.length; this.list.queryFound = response.found.length;
this.list.totalFound = result.totalFound; this.list.totalFound = response.totalFound;
this.list.inpxHash = result.inpxHash; this.list.inpxHash = response.inpxHash;
this.searchResult = result; this.searchResult = response;
await utils.sleep(1); await utils.sleep(1);
if (!this.queryExecute) { if (!this.queryExecute) {

View File

@@ -74,12 +74,8 @@ class WebSocketController {
await this.getConfig(req, ws); break; await this.getConfig(req, ws); break;
case 'get-worker-state': case 'get-worker-state':
await this.getWorkerState(req, ws); break; await this.getWorkerState(req, ws); break;
case 'author-search': case 'search':
await this.authorSearch(req, ws); break; await this.search(req, ws); break;
case 'series-search':
await this.seriesSearch(req, ws); break;
case 'title-search':
await this.titleSearch(req, ws); break;
case 'get-author-book-list': case 'get-author-book-list':
await this.getAuthorBookList(req, ws); break; await this.getAuthorBookList(req, ws); break;
case 'get-series-book-list': case 'get-series-book-list':
@@ -137,35 +133,13 @@ class WebSocketController {
this.send((state ? state : {}), req, ws); this.send((state ? state : {}), req, ws);
} }
async authorSearch(req, ws) { async search(req, ws) {
if (!req.query) if (!req.query)
throw new Error(`query is empty`); throw new Error(`query is empty`);
if (!req.from)
throw new Error(`from is empty`);
const result = await this.webWorker.authorSearch(req.query); const result = await this.webWorker.search(req.from, req.query);
this.send(result, req, ws);
}
async seriesSearch(req, ws) {
if (!this.config.extendedSearch)
throw new Error(`Extended search disabled`);
if (!req.query)
throw new Error(`query is empty`);
const result = await this.webWorker.seriesSearch(req.query);
this.send(result, req, ws);
}
async titleSearch(req, ws) {
if (!this.config.extendedSearch)
throw new Error(`Extended search disabled`);
if (!req.query)
throw new Error(`query is empty`);
const result = await this.webWorker.titleSearch(req.query);
this.send(result, req, ws); this.send(result, req, ws);
} }

View File

@@ -494,7 +494,9 @@ class DbCreator {
} }
if (rec.books.length) { if (rec.books.length) {
rec[restoreProp] = rec.books[0][restoreProp]; rec[restoreProp] = rec.value;//rec.books[0][restoreProp];
if (!rec[restoreProp])
rec[restoreProp] = emptyFieldValue;
} else { } else {
rec.toDel = 1; rec.toDel = 1;
} }

View File

@@ -20,6 +20,7 @@ class DbSearcher {
this.closed = false; this.closed = false;
this.memCache = new Map(); this.memCache = new Map();
this.bookIdMap = {};
this.periodicCleanCache();//no await this.periodicCleanCache();//no await
} }
@@ -54,40 +55,74 @@ class DbSearcher {
return where; return where;
} }
async selectAuthorIds(query) { async selectBookIds(query) {
const db = this.db; const db = this.db;
const authorKеy = `author-ids-author-${query.author}`;
let authorIds = await this.getCached(authorKеy);
//сначала выберем все id авторов по фильтру
//порядок id соответствует ASC-сортировке по author
if (authorIds === null) {
if (query.author && query.author !== '*') {
const where = this.getWhere(query.author);
const authorRows = await db.select({
table: 'author',
rawResult: true,
where: `return Array.from(${where})`,
});
authorIds = authorRows[0].rawResult;
} else {//все авторы
const authorRows = await db.select({
table: 'author',
rawResult: true,
where: `return Array.from(@all())`,
});
authorIds = authorRows[0].rawResult;
}
await this.putCached(authorKеy, authorIds);
}
const idsArr = []; const idsArr = [];
const tableBookIds = async(table, where) => {
const rows = await db.select({
table,
rawResult: true,
where: `
const ids = ${where};
const result = new Set();
for (const id of ids) {
const row = @unsafeRow(id);
for (const bookId of row.bookIds)
result.add(bookId);
}
return Array.from(result);
`
});
return rows[0].rawResult;
};
//авторы
if (query.author && query.author !== '*') {
const key = `book-ids-author-${query.author}`;
let ids = await this.getCached(key);
if (ids === null) {
ids = await tableBookIds('author', this.getWhere(query.author));
await this.putCached(key, ids);
}
idsArr.push(ids);
}
//серии
if (query.series && query.series !== '*') {
const key = `book-ids-series-${query.series}`;
let ids = await this.getCached(key);
if (ids === null) {
ids = await tableBookIds('series', this.getWhere(query.series));
await this.putCached(key, ids);
}
idsArr.push(ids);
}
//названия
if (query.title && query.title !== '*') {
const key = `book-ids-title-${query.title}`;
let ids = await this.getCached(key);
if (ids === null) {
ids = await tableBookIds('title', this.getWhere(query.title));
await this.putCached(key, ids);
}
idsArr.push(ids);
}
/*
//серии //серии
if (query.series && query.series !== '*') { if (query.series && query.series !== '*') {
const seriesKеy = `author-ids-series-${query.series}`; const seriesKеy = `author-ids-series-${query.series}`;
@@ -327,23 +362,9 @@ class DbSearcher {
idsArr.push(librateIds); idsArr.push(librateIds);
} }
/*
//ищем пересечение множеств
idsArr.push(authorIds);
if (idsArr.length > 1) {
const idsSetArr = idsArr.map(ids => new Set(ids));
authorIds = Array.from(utils.intersectSet(idsSetArr));
}
//сортировка
authorIds.sort((a, b) => a - b);
*/ */
if (idsArr.length > 1) {
//ищем пересечение множеств, работает быстрее предыдущего //ищем пересечение множеств
if (idsArr.length) {
idsArr.push(authorIds);
let proc = 0; let proc = 0;
let nextProc = 0; let nextProc = 0;
let inter = new Set(idsArr[0]); let inter = new Set(idsArr[0]);
@@ -363,340 +384,120 @@ class DbSearcher {
} }
inter = newInter; inter = newInter;
} }
authorIds = Array.from(inter);
}
//сортировка
authorIds.sort((a, b) => a - b);
return authorIds; return Array.from(inter);
} else if (idsArr.length == 1) {
return idsArr[0];
} else {
return false;
}
} }
getWhere2(query, ids, exclude = '') { async fillBookIdMap(from) {
const db = this.db; if (!this.bookIdMap[from]) {
const db = this.db;
const map = new Map();
const table = `${from}_id`;
const filterBySearch = (searchValue) => { await db.open({table});
searchValue = searchValue.toLowerCase(); const rows = await db.select({table});
await db.close({table});
//особая обработка префиксов for (const row of rows) {
if (searchValue[0] == '=') { if (!row.value.length)
continue;
searchValue = searchValue.substring(1); if (row.value.length > 1)
return `bookValue.localeCompare(${db.esc(searchValue)}) == 0`; map.set(row.id, row.value);
} else if (searchValue[0] == '*') { else
map.set(row.id, row.value[0]);
searchValue = searchValue.substring(1);
return `bookValue !== ${db.esc(emptyFieldValue)} && bookValue.indexOf(${db.esc(searchValue)}) >= 0`;
} else if (searchValue[0] == '#') {
searchValue = searchValue.substring(1);
return `!bookValue || (bookValue !== ${db.esc(emptyFieldValue)} && !enru.has(bookValue[0]) && bookValue.indexOf(${db.esc(searchValue)}) >= 0)`;
} else {
return `bookValue.localeCompare(${db.esc(searchValue)}) >= 0 && bookValue.localeCompare(${db.esc(searchValue + maxUtf8Char)}) <= 0`;
} }
};
//подготовка фильтра this.bookIdMap[from] = map;
let filter = '';
let closures = '';
//порядок важен, более простые проверки вперед
//удаленные
if (query.del !== undefined) {
filter += `
if (book.del !== ${db.esc(query.del)})
return false;
`;
} }
return this.bookIdMap[from];
//дата поступления
if (query.date) {
let [from = '0000-00-00', to = '9999-99-99'] = query.date.split(',');
filter += `
if (!(book.date >= ${db.esc(from)} && book.date <= ${db.esc(to)}))
return false;
`;
}
//оценка
if (query.librate) {
closures += `
const searchLibrate = new Set(${db.esc(query.librate.split(',').map(n => parseInt(n, 10)).filter(n => !isNaN(n)))});
`;
filter += `
if (!searchLibrate.has(book.librate))
return false;
`;
}
//серии
if (exclude !== 'series' && query.series && query.series !== '*') {
closures += `
const checkSeries = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
bookValue = bookValue.toLowerCase();
return ${filterBySearch(query.series)};
};
`;
filter += `
if (!checkSeries(book.series))
return false;
`;
}
//названия
if (exclude !== 'title' && query.title && query.title !== '*') {
closures += `
const checkTitle = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
bookValue = bookValue.toLowerCase();
return ${filterBySearch(query.title)};
};
`;
filter += `
if (!checkTitle(book.title))
return false;
`;
}
//языки
if (exclude !== 'lang' && query.lang) {
const queryLangs = query.lang.split(',');
closures += `
const queryLangs = new Set(${db.esc(queryLangs)});
const checkLang = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
return queryLangs.has(bookValue);
};
`;
filter += `
if (!checkLang(book.lang))
return false;
`;
}
//жанры
if (exclude !== 'genre' && query.genre) {
const queryGenres = query.genre.split(',');
closures += `
const queryGenres = new Set(${db.esc(queryGenres)});
const checkGenre = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
return queryGenres.has(bookValue);
};
`;
filter += `
const genres = book.genre.split(',');
found = false;
for (const g of genres) {
if (checkGenre(g)) {
found = true;
break;
}
}
if (!found)
return false;
`;
}
//авторы
if (exclude !== 'author' && query.author && query.author !== '*') {
closures += `
const splitAuthor = (author) => {
if (!author)
author = ${db.esc(emptyFieldValue)};
const result = author.split(',');
if (result.length > 1)
result.push(author);
return result;
};
const checkAuthor = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
bookValue = bookValue.toLowerCase();
return ${filterBySearch(query.author)};
};
`;
filter += `
const author = splitAuthor(book.author);
found = false;
for (const a of author) {
if (checkAuthor(a)) {
found = true;
break;
}
}
if (!found)
return false;
`;
}
//формируем where
let where = '';
if (filter) {
where = `
const enru = new Set(${db.esc(enruArr)});
${closures}
const filterBook = (book) => {
let found = false;
${filter}
return true;
};
let ids;
if (${!ids}) {
ids = @all();
} else {
ids = ${db.esc(ids)};
}
const result = new Set();
for (const id of ids) {
const row = @unsafeRow(id);
if (row) {
for (const book of row.books) {
if (filterBook(book)) {
result.add(id);
break;
}
}
}
}
return Array.from(result);
`;
}
return where;
} }
async selectSeriesIds(query) { async filterTableIds(tableIds, from, query) {
const db = this.db; let result = tableIds;
let seriesIds = false; //т.к. авторы идут списком, то дополнительно фильтруем
let isAll = !(query.series && query.series !== '*'); if (query.author && query.author !== '*') {
const key = `filter-ids-author-${query.author}`;
let authorIds = await this.getCached(key);
//серии if (authorIds === null) {
const seriesKеy = `series-ids-series-${query.series}`; const rows = await this.db.select({
seriesIds = await this.getCached(seriesKеy); table: 'author',
if (seriesIds === null) {
if (query.series && query.series !== '*') {
const where = this.getWhere(query.series);
const seriesRows = await db.select({
table: 'series',
rawResult: true, rawResult: true,
where: `return Array.from(${where})`, where: `return Array.from(${this.getWhere(query.author)})`
}); });
seriesIds = seriesRows[0].rawResult; authorIds = rows[0].rawResult;
} else {
const seriesRows = await db.select({
table: 'series',
rawResult: true,
where: `return Array.from(@all())`,
});
seriesIds = seriesRows[0].rawResult; await this.putCached(key, authorIds);
} }
seriesIds.sort((a, b) => a - b); //пересечение tableIds и authorIds
result = [];
await this.putCached(seriesKеy, seriesIds); const authorIdsSet = new Set(authorIds);
for (const id of tableIds)
if (authorIdsSet.has(id))
result.push(id);
} }
const where = this.getWhere2(query, (isAll ? false : seriesIds), 'series'); return result;
if (where) {
//тяжелый запрос перебором в series_book
const rows = await db.select({
table: 'series_book',
rawResult: true,
where,
});
seriesIds = rows[0].rawResult;
}
return seriesIds;
} }
async selectTitleIds(query) { async selectTableIds(from, query) {
const db = this.db; const db = this.db;
const queryKey = this.queryKey(query);
const tableKey = `${from}-table-ids-${queryKey}`;
let tableIds = await this.getCached(tableKey);
let titleIds = false; if (tableIds === null) {
let isAll = !(query.title && query.title !== '*'); const bookKey = `book-ids-${queryKey}`;
let bookIds = await this.getCached(bookKey);
//серии if (bookIds === null) {
const titleKеy = `title-ids-title-${query.title}`; bookIds = await this.selectBookIds(query);
titleIds = await this.getCached(titleKеy); await this.putCached(bookKey, bookIds);
if (titleIds === null) {
if (query.title && query.title !== '*') {
const where = this.getWhere(query.title);
const titleRows = await db.select({
table: 'title',
rawResult: true,
where: `return Array.from(${where})`,
});
titleIds = titleRows[0].rawResult;
} else {
const titleRows = await db.select({
table: 'title',
rawResult: true,
where: `return Array.from(@all())`,
});
titleIds = titleRows[0].rawResult;
} }
titleIds.sort((a, b) => a - b); if (bookIds) {
const tableIdsSet = new Set();
const bookIdMap = await this.fillBookIdMap(from);
for (const bookId of bookIds) {
const tableIdValue = bookIdMap.get(bookId);
if (!tableIdValue)
continue;
await this.putCached(titleKеy, titleIds); if (Array.isArray(tableIdValue)) {
for (const tableId of tableIdValue)
tableIdsSet.add(tableId);
} else
tableIdsSet.add(tableIdValue);
}
tableIds = Array.from(tableIdsSet);
} else {
const rows = await db.select({
table: from,
rawResult: true,
where: `return Array.from(@all())`
});
tableIds = rows[0].rawResult;
}
tableIds = await this.filterTableIds(tableIds, from, query);
tableIds.sort((a, b) => a - b);
await this.putCached(tableKey, tableIds);
} }
const where = this.getWhere2(query, (isAll ? false : titleIds), 'title'); return tableIds;
if (where) {
//тяжелый запрос перебором в title_book
const rows = await db.select({
table: 'title_book',
rawResult: true,
where,
});
titleIds = rows[0].rawResult;
}
return titleIds;
} }
async getCached(key) { async getCached(key) {
@@ -771,112 +572,33 @@ class DbSearcher {
}); });
} }
async authorSearch(query) { async search(from, query) {
if (this.closed) if (this.closed)
throw new Error('DbSearcher closed'); throw new Error('DbSearcher closed');
if (!['author', 'series', 'title'].includes(from))
throw new Error(`Unknown value for param 'from'`);
this.searchFlag++; this.searchFlag++;
try { try {
const db = this.db; const db = this.db;
const key = `author-ids-${this.queryKey(query)}`; const ids = await this.selectTableIds(from, query);
//сначала попробуем найти в кеше const totalFound = ids.length;
let authorIds = await this.getCached(key);
if (authorIds === null) {//не нашли в кеше, ищем в поисковых таблицах
authorIds = await this.selectAuthorIds(query);
await this.putCached(key, authorIds);
}
const totalFound = authorIds.length;
let limit = (query.limit ? query.limit : 100); let limit = (query.limit ? query.limit : 100);
limit = (limit > maxLimit ? maxLimit : limit); limit = (limit > maxLimit ? maxLimit : limit);
const offset = (query.offset ? query.offset : 0); const offset = (query.offset ? query.offset : 0);
//выборка найденных авторов //выборка найденных авторов
const result = await db.select({ const found = await db.select({
table: 'author', table: `${from}_book`,
map: `(r) => ({id: r.id, author: r.author, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, map: `(r) => ({id: r.id, ${from}: r.${from}, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
where: `@@id(${db.esc(authorIds.slice(offset, offset + limit))})` where: `@@id(${db.esc(ids.slice(offset, offset + limit))})`
}); });
return {result, totalFound}; return {found, totalFound};
} finally {
this.searchFlag--;
}
}
async seriesSearch(query) {
if (this.closed)
throw new Error('DbSearcher closed');
this.searchFlag++;
try {
const db = this.db;
const key = `series-ids-${this.queryKey(query)}`;
//сначала попробуем найти в кеше
let seriesIds = await this.getCached(key);
if (seriesIds === null) {//не нашли в кеше, ищем в поисковых таблицах
seriesIds = await this.selectSeriesIds(query);
await this.putCached(key, seriesIds);
}
const totalFound = seriesIds.length;
let limit = (query.limit ? query.limit : 100);
limit = (limit > maxLimit ? maxLimit : limit);
const offset = (query.offset ? query.offset : 0);
//выборка найденных авторов
const result = await db.select({
table: 'series_book',
map: `(r) => ({id: r.id, series: r.series, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
where: `@@id(${db.esc(seriesIds.slice(offset, offset + limit))})`
});
return {result, totalFound};
} finally {
this.searchFlag--;
}
}
async titleSearch(query) {
if (this.closed)
throw new Error('DbSearcher closed');
this.searchFlag++;
try {
const db = this.db;
const key = `title-ids-${this.queryKey(query)}`;
//сначала попробуем найти в кеше
let titleIds = await this.getCached(key);
if (titleIds === null) {//не нашли в кеше, ищем в поисковых таблицах
titleIds = await this.selectTitleIds(query);
await this.putCached(key, titleIds);
}
const totalFound = titleIds.length;
let limit = (query.limit ? query.limit : 100);
limit = (limit > maxLimit ? maxLimit : limit);
const offset = (query.offset ? query.offset : 0);
//выборка найденных авторов
const result = await db.select({
table: 'title_book',
map: `(r) => ({id: r.id, title: r.title, books: r.books, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
where: `@@id(${db.esc(titleIds.slice(offset, offset + limit))})`
});
return {result, totalFound};
} finally { } finally {
this.searchFlag--; this.searchFlag--;
} }
@@ -908,7 +630,7 @@ class DbSearcher {
books = rows[0].books; books = rows[0].books;
} }
return {author, books}; return {author, books: (books && books.length ? JSON.stringify(books) : '')};
} finally { } finally {
this.searchFlag--; this.searchFlag--;
} }

View File

@@ -187,14 +187,8 @@ class WebWorker {
}, },
}); });
//открываем почти все таблицы //открываем таблицы
await db.openAll({exclude: ['author', 'title_book']}); await db.openAll({exclude: ['author_id', 'series_id', 'title_id']});
//откроем таблицу 'author' с бОльшим размером кеша блоков, для ускорения выборки
await db.open({table: 'author', cacheSize: (config.dbCacheSize > 100 ? config.dbCacheSize : 100)});
if (config.extendedSearch)
await db.open({table: 'title_book'});
this.dbSearcher = new DbSearcher(config, db); this.dbSearcher = new DbSearcher(config, db);
@@ -242,43 +236,15 @@ class WebWorker {
return db.wwCache.config; return db.wwCache.config;
} }
async authorSearch(query) { async search(from, query) {
this.checkMyState(); this.checkMyState();
const config = await this.dbConfig(); const result = await this.dbSearcher.search(from, query);
const result = await this.dbSearcher.authorSearch(query);
return {
author: result.result,
totalFound: result.totalFound,
inpxHash: (config.inpxHash ? config.inpxHash : ''),
};
}
async seriesSearch(query) {
this.checkMyState();
const config = await this.dbConfig(); const config = await this.dbConfig();
const result = await this.dbSearcher.seriesSearch(query); result.inpxHash = (config.inpxHash ? config.inpxHash : '');
return { return result;
series: result.result,
totalFound: result.totalFound,
inpxHash: (config.inpxHash ? config.inpxHash : ''),
};
}
async titleSearch(query) {
this.checkMyState();
const config = await this.dbConfig();
const result = await this.dbSearcher.titleSearch(query);
return {
title: result.result,
totalFound: result.totalFound,
inpxHash: (config.inpxHash ? config.inpxHash : ''),
};
} }
async getAuthorBookList(authorId) { async getAuthorBookList(authorId) {
@@ -522,7 +488,7 @@ class WebWorker {
async periodicLogServerStats() { async periodicLogServerStats() {
while (1) {// eslint-disable-line while (1) {// eslint-disable-line
this.logServerStats(); this.logServerStats();
await utils.sleep(60*1000); await utils.sleep(1000)//(60*1000);
} }
} }