deadcxap/inpx-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Работа над новым поиском

Browse Source
This commit is contained in:
Book Pauk
2022-10-30 13:55:54 +07:00
parent 2ba8297594
commit 3f39b55ca0
6 changed files with 187 additions and 531 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
client/components/Api/Api.vue
View File

@@ -218,16 +218,8 @@ class Api {
}
}
async authorSearch(query) {
return await this.request({action: 'author-search', query});
}
async seriesSearch(query) {
return await this.request({action: 'series-search', query});
}
async titleSearch(query) {
return await this.request({action: 'title-search', query});
async search(from, query) {
return await this.request({action: 'search', from, query});
}
async getAuthorBookList(authorId) {

14
client/components/Search/AuthorList/AuthorList.vue
View File

@@ -317,7 +317,7 @@ class AuthorList extends BaseList {
let result = [];
const expandedSet = new Set(this.expandedAuthor);
const authors = this.searchResult.author;
const authors = this.searchResult.found;
if (!authors)
return;
@@ -399,7 +399,7 @@ class AuthorList extends BaseList {
(async() => {
await utils.sleep(500);
if (this.refreshing)
this.loadingMessage = 'Поиск серий...';
this.loadingMessage = 'Поиск авторов...';
})();
try {
@@ -408,13 +408,13 @@ class AuthorList extends BaseList {
this.queryExecute = null;
try {
const result = await this.api.authorSearch(query);
const response = await this.api.search('author', query);
this.list.queryFound = result.author.length;
this.list.totalFound = result.totalFound;
this.list.inpxHash = result.inpxHash;
this.list.queryFound = response.found.length;
this.list.totalFound = response.totalFound;
this.list.inpxHash = response.inpxHash;
this.searchResult = result;
this.searchResult = response;
await utils.sleep(1);
if (!this.queryExecute) {

38
server/controllers/WebSocketController.js
View File

@@ -74,12 +74,8 @@ class WebSocketController {
await this.getConfig(req, ws); break;
case 'get-worker-state':
await this.getWorkerState(req, ws); break;
case 'author-search':
await this.authorSearch(req, ws); break;
case 'series-search':
await this.seriesSearch(req, ws); break;
case 'title-search':
await this.titleSearch(req, ws); break;
case 'search':
await this.search(req, ws); break;
case 'get-author-book-list':
await this.getAuthorBookList(req, ws); break;
case 'get-series-book-list':
@@ -137,35 +133,13 @@ class WebSocketController {
this.send((state ? state : {}), req, ws);
}
async authorSearch(req, ws) {
async search(req, ws) {
if (!req.query)
throw new Error(`query is empty`);
if (!req.from)
throw new Error(`from is empty`);
const result = await this.webWorker.authorSearch(req.query);
this.send(result, req, ws);
}
async seriesSearch(req, ws) {
if (!this.config.extendedSearch)
throw new Error(`Extended search disabled`);
if (!req.query)
throw new Error(`query is empty`);
const result = await this.webWorker.seriesSearch(req.query);
this.send(result, req, ws);
}
async titleSearch(req, ws) {
if (!this.config.extendedSearch)
throw new Error(`Extended search disabled`);
if (!req.query)
throw new Error(`query is empty`);
const result = await this.webWorker.titleSearch(req.query);
const result = await this.webWorker.search(req.from, req.query);
this.send(result, req, ws);
}

4
server/core/DbCreator.js
View File

@@ -494,7 +494,9 @@ class DbCreator {
}
if (rec.books.length) {
rec[restoreProp] = rec.books[0][restoreProp];
rec[restoreProp] = rec.value;//rec.books[0][restoreProp];
if (!rec[restoreProp])
rec[restoreProp] = emptyFieldValue;
} else {
rec.toDel = 1;
}

608
server/core/DbSearcher.js
View File

@@ -20,6 +20,7 @@ class DbSearcher {
this.closed = false;
this.memCache = new Map();
this.bookIdMap = {};
this.periodicCleanCache();//no await
}
@@ -54,40 +55,74 @@ class DbSearcher {
return where;
}
async selectAuthorIds(query) {
async selectBookIds(query) {
const db = this.db;
const authorKеy = `author-ids-author-${query.author}`;
let authorIds = await this.getCached(authorKеy);
//сначала выберем все id авторов по фильтру
//порядок id соответствует ASC-сортировке по author
if (authorIds === null) {
if (query.author && query.author !== '*') {
const where = this.getWhere(query.author);
const authorRows = await db.select({
table: 'author',
rawResult: true,
where: `return Array.from(${where})`,
});
authorIds = authorRows[0].rawResult;
} else {//все авторы
const authorRows = await db.select({
table: 'author',
rawResult: true,
where: `return Array.from(@all())`,
});
authorIds = authorRows[0].rawResult;
}
await this.putCached(authorKеy, authorIds);
}
const idsArr = [];
const tableBookIds = async(table, where) => {
const rows = await db.select({
table,
rawResult: true,
where: `
const ids = ${where};
const result = new Set();
for (const id of ids) {
const row = @unsafeRow(id);
for (const bookId of row.bookIds)
result.add(bookId);
}
return Array.from(result);
`
});
return rows[0].rawResult;
};
//авторы
if (query.author && query.author !== '*') {
const key = `book-ids-author-${query.author}`;
let ids = await this.getCached(key);
if (ids === null) {
ids = await tableBookIds('author', this.getWhere(query.author));
await this.putCached(key, ids);
}
idsArr.push(ids);
}
//серии
if (query.series && query.series !== '*') {
const key = `book-ids-series-${query.series}`;
let ids = await this.getCached(key);
if (ids === null) {
ids = await tableBookIds('series', this.getWhere(query.series));
await this.putCached(key, ids);
}
idsArr.push(ids);
}
//названия
if (query.title && query.title !== '*') {
const key = `book-ids-title-${query.title}`;
let ids = await this.getCached(key);
if (ids === null) {
ids = await tableBookIds('title', this.getWhere(query.title));
await this.putCached(key, ids);
}
idsArr.push(ids);
}
/*
//серии
if (query.series && query.series !== '*') {
const seriesKеy = `author-ids-series-${query.series}`;
@@ -327,23 +362,9 @@ class DbSearcher {
idsArr.push(librateIds);
}
/*
//ищем пересечение множеств
idsArr.push(authorIds);
if (idsArr.length > 1) {
const idsSetArr = idsArr.map(ids => new Set(ids));
authorIds = Array.from(utils.intersectSet(idsSetArr));
}
//сортировка
authorIds.sort((a, b) => a - b);
*/
//ищем пересечение множеств, работает быстрее предыдущего
if (idsArr.length) {
idsArr.push(authorIds);
if (idsArr.length > 1) {
//ищем пересечение множеств
let proc = 0;
let nextProc = 0;
let inter = new Set(idsArr[0]);
@@ -363,340 +384,120 @@ class DbSearcher {
}
inter = newInter;
}
authorIds = Array.from(inter);
}
//сортировка
authorIds.sort((a, b) => a - b);
return authorIds;
}
getWhere2(query, ids, exclude = '') {
const db = this.db;
const filterBySearch = (searchValue) => {
searchValue = searchValue.toLowerCase();
//особая обработка префиксов
if (searchValue[0] == '=') {
searchValue = searchValue.substring(1);
return `bookValue.localeCompare(${db.esc(searchValue)}) == 0`;
} else if (searchValue[0] == '*') {
searchValue = searchValue.substring(1);
return `bookValue !== ${db.esc(emptyFieldValue)} && bookValue.indexOf(${db.esc(searchValue)}) >= 0`;
} else if (searchValue[0] == '#') {
searchValue = searchValue.substring(1);
return `!bookValue || (bookValue !== ${db.esc(emptyFieldValue)} && !enru.has(bookValue[0]) && bookValue.indexOf(${db.esc(searchValue)}) >= 0)`;
return Array.from(inter);
} else if (idsArr.length == 1) {
return idsArr[0];
} else {
return `bookValue.localeCompare(${db.esc(searchValue)}) >= 0 && bookValue.localeCompare(${db.esc(searchValue + maxUtf8Char)}) <= 0`;
}
};
//подготовка фильтра
let filter = '';
let closures = '';
//порядок важен, более простые проверки вперед
//удаленные
if (query.del !== undefined) {
filter += `
if (book.del !== ${db.esc(query.del)})
return false;
`;
}
//дата поступления
if (query.date) {
let [from = '0000-00-00', to = '9999-99-99'] = query.date.split(',');
filter += `
if (!(book.date >= ${db.esc(from)} && book.date <= ${db.esc(to)}))
return false;
`;
}
//оценка
if (query.librate) {
closures += `
const searchLibrate = new Set(${db.esc(query.librate.split(',').map(n => parseInt(n, 10)).filter(n => !isNaN(n)))});
`;
filter += `
if (!searchLibrate.has(book.librate))
return false;
`;
}
//серии
if (exclude !== 'series' && query.series && query.series !== '*') {
closures += `
const checkSeries = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
bookValue = bookValue.toLowerCase();
return ${filterBySearch(query.series)};
};
`;
filter += `
if (!checkSeries(book.series))
return false;
`;
}
//названия
if (exclude !== 'title' && query.title && query.title !== '*') {
closures += `
const checkTitle = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
bookValue = bookValue.toLowerCase();
return ${filterBySearch(query.title)};
};
`;
filter += `
if (!checkTitle(book.title))
return false;
`;
}
//языки
if (exclude !== 'lang' && query.lang) {
const queryLangs = query.lang.split(',');
closures += `
const queryLangs = new Set(${db.esc(queryLangs)});
const checkLang = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
return queryLangs.has(bookValue);
};
`;
filter += `
if (!checkLang(book.lang))
return false;
`;
}
//жанры
if (exclude !== 'genre' && query.genre) {
const queryGenres = query.genre.split(',');
closures += `
const queryGenres = new Set(${db.esc(queryGenres)});
const checkGenre = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
return queryGenres.has(bookValue);
};
`;
filter += `
const genres = book.genre.split(',');
found = false;
for (const g of genres) {
if (checkGenre(g)) {
found = true;
break;
}
}
if (!found)
return false;
`;
async fillBookIdMap(from) {
if (!this.bookIdMap[from]) {
const db = this.db;
const map = new Map();
const table = `${from}_id`;
await db.open({table});
const rows = await db.select({table});
await db.close({table});
for (const row of rows) {
if (!row.value.length)
continue;
if (row.value.length > 1)
map.set(row.id, row.value);
else
map.set(row.id, row.value[0]);
}
//авторы
if (exclude !== 'author' && query.author && query.author !== '*') {
closures += `
const splitAuthor = (author) => {
if (!author)
author = ${db.esc(emptyFieldValue)};
this.bookIdMap[from] = map;
}
return this.bookIdMap[from];
}
const result = author.split(',');
if (result.length > 1)
result.push(author);
async filterTableIds(tableIds, from, query) {
let result = tableIds;
//т.к. авторы идут списком, то дополнительно фильтруем
if (query.author && query.author !== '*') {
const key = `filter-ids-author-${query.author}`;
let authorIds = await this.getCached(key);
if (authorIds === null) {
const rows = await this.db.select({
table: 'author',
rawResult: true,
where: `return Array.from(${this.getWhere(query.author)})`
});
authorIds = rows[0].rawResult;
await this.putCached(key, authorIds);
}
//пересечение tableIds и authorIds
result = [];
const authorIdsSet = new Set(authorIds);
for (const id of tableIds)
if (authorIdsSet.has(id))
result.push(id);
}
return result;
};
const checkAuthor = (bookValue) => {
if (!bookValue)
bookValue = ${db.esc(emptyFieldValue)};
bookValue = bookValue.toLowerCase();
return ${filterBySearch(query.author)};
};
`;
filter += `
const author = splitAuthor(book.author);
found = false;
for (const a of author) {
if (checkAuthor(a)) {
found = true;
break;
}
}
if (!found)
return false;
`;
}
//формируем where
let where = '';
if (filter) {
where = `
const enru = new Set(${db.esc(enruArr)});
${closures}
const filterBook = (book) => {
let found = false;
${filter}
return true;
};
let ids;
if (${!ids}) {
ids = @all();
} else {
ids = ${db.esc(ids)};
}
const result = new Set();
for (const id of ids) {
const row = @unsafeRow(id);
if (row) {
for (const book of row.books) {
if (filterBook(book)) {
result.add(id);
break;
}
}
}
}
return Array.from(result);
`;
}
return where;
}
async selectSeriesIds(query) {
async selectTableIds(from, query) {
const db = this.db;
const queryKey = this.queryKey(query);
const tableKey = `${from}-table-ids-${queryKey}`;
let tableIds = await this.getCached(tableKey);
let seriesIds = false;
let isAll = !(query.series && query.series !== '*');
if (tableIds === null) {
const bookKey = `book-ids-${queryKey}`;
let bookIds = await this.getCached(bookKey);
//серии
const seriesKеy = `series-ids-series-${query.series}`;
seriesIds = await this.getCached(seriesKеy);
if (bookIds === null) {
bookIds = await this.selectBookIds(query);
await this.putCached(bookKey, bookIds);
}
if (seriesIds === null) {
if (query.series && query.series !== '*') {
const where = this.getWhere(query.series);
if (bookIds) {
const tableIdsSet = new Set();
const bookIdMap = await this.fillBookIdMap(from);
for (const bookId of bookIds) {
const tableIdValue = bookIdMap.get(bookId);
if (!tableIdValue)
continue;
const seriesRows = await db.select({
table: 'series',
rawResult: true,
where: `return Array.from(${where})`,
});
if (Array.isArray(tableIdValue)) {
for (const tableId of tableIdValue)
tableIdsSet.add(tableId);
} else
tableIdsSet.add(tableIdValue);
}
seriesIds = seriesRows[0].rawResult;
tableIds = Array.from(tableIdsSet);
} else {
const seriesRows = await db.select({
table: 'series',
rawResult: true,
where: `return Array.from(@all())`,
});
seriesIds = seriesRows[0].rawResult;
}
seriesIds.sort((a, b) => a - b);
await this.putCached(seriesKеy, seriesIds);
}
const where = this.getWhere2(query, (isAll ? false : seriesIds), 'series');
if (where) {
//тяжелый запрос перебором в series_book
const rows = await db.select({
table: 'series_book',
table: from,
rawResult: true,
where,
where: `return Array.from(@all())`
});
seriesIds = rows[0].rawResult;
tableIds = rows[0].rawResult;
}
return seriesIds;
tableIds = await this.filterTableIds(tableIds, from, query);
tableIds.sort((a, b) => a - b);
await this.putCached(tableKey, tableIds);
}
async selectTitleIds(query) {
const db = this.db;
let titleIds = false;
let isAll = !(query.title && query.title !== '*');
//серии
const titleKеy = `title-ids-title-${query.title}`;
titleIds = await this.getCached(titleKеy);
if (titleIds === null) {
if (query.title && query.title !== '*') {
const where = this.getWhere(query.title);
const titleRows = await db.select({
table: 'title',
rawResult: true,
where: `return Array.from(${where})`,
});
titleIds = titleRows[0].rawResult;
} else {
const titleRows = await db.select({
table: 'title',
rawResult: true,
where: `return Array.from(@all())`,
});
titleIds = titleRows[0].rawResult;
}
titleIds.sort((a, b) => a - b);
await this.putCached(titleKеy, titleIds);
}
const where = this.getWhere2(query, (isAll ? false : titleIds), 'title');
if (where) {
//тяжелый запрос перебором в title_book
const rows = await db.select({
table: 'title_book',
rawResult: true,
where,
});
titleIds = rows[0].rawResult;
}
return titleIds;
return tableIds;
}
async getCached(key) {
@@ -771,112 +572,33 @@ class DbSearcher {
});
}
async authorSearch(query) {
async search(from, query) {
if (this.closed)
throw new Error('DbSearcher closed');
if (!['author', 'series', 'title'].includes(from))
throw new Error(`Unknown value for param 'from'`);
this.searchFlag++;
try {
const db = this.db;
const key = `author-ids-${this.queryKey(query)}`;
const ids = await this.selectTableIds(from, query);
//сначала попробуем найти в кеше
let authorIds = await this.getCached(key);
if (authorIds === null) {//не нашли в кеше, ищем в поисковых таблицах
authorIds = await this.selectAuthorIds(query);
await this.putCached(key, authorIds);
}
const totalFound = authorIds.length;
const totalFound = ids.length;
let limit = (query.limit ? query.limit : 100);
limit = (limit > maxLimit ? maxLimit : limit);
const offset = (query.offset ? query.offset : 0);
//выборка найденных авторов
const result = await db.select({
table: 'author',
map: `(r) => ({id: r.id, author: r.author, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
where: `@@id(${db.esc(authorIds.slice(offset, offset + limit))})`
const found = await db.select({
table: `${from}_book`,
map: `(r) => ({id: r.id, ${from}: r.${from}, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
where: `@@id(${db.esc(ids.slice(offset, offset + limit))})`
});
return {result, totalFound};
} finally {
this.searchFlag--;
}
}
async seriesSearch(query) {
if (this.closed)
throw new Error('DbSearcher closed');
this.searchFlag++;
try {
const db = this.db;
const key = `series-ids-${this.queryKey(query)}`;
//сначала попробуем найти в кеше
let seriesIds = await this.getCached(key);
if (seriesIds === null) {//не нашли в кеше, ищем в поисковых таблицах
seriesIds = await this.selectSeriesIds(query);
await this.putCached(key, seriesIds);
}
const totalFound = seriesIds.length;
let limit = (query.limit ? query.limit : 100);
limit = (limit > maxLimit ? maxLimit : limit);
const offset = (query.offset ? query.offset : 0);
//выборка найденных авторов
const result = await db.select({
table: 'series_book',
map: `(r) => ({id: r.id, series: r.series, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
where: `@@id(${db.esc(seriesIds.slice(offset, offset + limit))})`
});
return {result, totalFound};
} finally {
this.searchFlag--;
}
}
async titleSearch(query) {
if (this.closed)
throw new Error('DbSearcher closed');
this.searchFlag++;
try {
const db = this.db;
const key = `title-ids-${this.queryKey(query)}`;
//сначала попробуем найти в кеше
let titleIds = await this.getCached(key);
if (titleIds === null) {//не нашли в кеше, ищем в поисковых таблицах
titleIds = await this.selectTitleIds(query);
await this.putCached(key, titleIds);
}
const totalFound = titleIds.length;
let limit = (query.limit ? query.limit : 100);
limit = (limit > maxLimit ? maxLimit : limit);
const offset = (query.offset ? query.offset : 0);
//выборка найденных авторов
const result = await db.select({
table: 'title_book',
map: `(r) => ({id: r.id, title: r.title, books: r.books, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
where: `@@id(${db.esc(titleIds.slice(offset, offset + limit))})`
});
return {result, totalFound};
return {found, totalFound};
} finally {
this.searchFlag--;
}
@@ -908,7 +630,7 @@ class DbSearcher {
books = rows[0].books;
}
return {author, books};
return {author, books: (books && books.length ? JSON.stringify(books) : '')};
} finally {
this.searchFlag--;
}

48
server/core/WebWorker.js
View File

@@ -187,14 +187,8 @@ class WebWorker {
},
});
//открываем почти все таблицы
await db.openAll({exclude: ['author', 'title_book']});
//откроем таблицу 'author' с бОльшим размером кеша блоков, для ускорения выборки
await db.open({table: 'author', cacheSize: (config.dbCacheSize > 100 ? config.dbCacheSize : 100)});
if (config.extendedSearch)
await db.open({table: 'title_book'});
//открываем таблицы
await db.openAll({exclude: ['author_id', 'series_id', 'title_id']});
this.dbSearcher = new DbSearcher(config, db);
@@ -242,43 +236,15 @@ class WebWorker {
return db.wwCache.config;
}
async authorSearch(query) {
async search(from, query) {
this.checkMyState();
const config = await this.dbConfig();
const result = await this.dbSearcher.authorSearch(query);
return {
author: result.result,
totalFound: result.totalFound,
inpxHash: (config.inpxHash ? config.inpxHash : ''),
};
}
async seriesSearch(query) {
this.checkMyState();
const result = await this.dbSearcher.search(from, query);
const config = await this.dbConfig();
const result = await this.dbSearcher.seriesSearch(query);
result.inpxHash = (config.inpxHash ? config.inpxHash : '');
return {
series: result.result,
totalFound: result.totalFound,
inpxHash: (config.inpxHash ? config.inpxHash : ''),
};
}
async titleSearch(query) {
this.checkMyState();
const config = await this.dbConfig();
const result = await this.dbSearcher.titleSearch(query);
return {
title: result.result,
totalFound: result.totalFound,
inpxHash: (config.inpxHash ? config.inpxHash : ''),
};
return result;
}
async getAuthorBookList(authorId) {
@@ -522,7 +488,7 @@ class WebWorker {
async periodicLogServerStats() {
while (1) {// eslint-disable-line
this.logServerStats();
await utils.sleep(60*1000);
await utils.sleep(1000)//(60*1000);
}
}