diff --git a/client/components/Api/Api.vue b/client/components/Api/Api.vue index 0898738..02e139b 100644 --- a/client/components/Api/Api.vue +++ b/client/components/Api/Api.vue @@ -218,16 +218,8 @@ class Api { } } - async authorSearch(query) { - return await this.request({action: 'author-search', query}); - } - - async seriesSearch(query) { - return await this.request({action: 'series-search', query}); - } - - async titleSearch(query) { - return await this.request({action: 'title-search', query}); + async search(from, query) { + return await this.request({action: 'search', from, query}); } async getAuthorBookList(authorId) { diff --git a/client/components/Search/AuthorList/AuthorList.vue b/client/components/Search/AuthorList/AuthorList.vue index 2680c80..620fcb1 100644 --- a/client/components/Search/AuthorList/AuthorList.vue +++ b/client/components/Search/AuthorList/AuthorList.vue @@ -317,7 +317,7 @@ class AuthorList extends BaseList { let result = []; const expandedSet = new Set(this.expandedAuthor); - const authors = this.searchResult.author; + const authors = this.searchResult.found; if (!authors) return; @@ -399,7 +399,7 @@ class AuthorList extends BaseList { (async() => { await utils.sleep(500); if (this.refreshing) - this.loadingMessage = 'Поиск серий...'; + this.loadingMessage = 'Поиск авторов...'; })(); try { @@ -408,13 +408,13 @@ class AuthorList extends BaseList { this.queryExecute = null; try { - const result = await this.api.authorSearch(query); + const response = await this.api.search('author', query); - this.list.queryFound = result.author.length; - this.list.totalFound = result.totalFound; - this.list.inpxHash = result.inpxHash; + this.list.queryFound = response.found.length; + this.list.totalFound = response.totalFound; + this.list.inpxHash = response.inpxHash; - this.searchResult = result; + this.searchResult = response; await utils.sleep(1); if (!this.queryExecute) { diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js index 53d9735..0dac9f6 100644 --- a/server/controllers/WebSocketController.js +++ b/server/controllers/WebSocketController.js @@ -74,12 +74,8 @@ class WebSocketController { await this.getConfig(req, ws); break; case 'get-worker-state': await this.getWorkerState(req, ws); break; - case 'author-search': - await this.authorSearch(req, ws); break; - case 'series-search': - await this.seriesSearch(req, ws); break; - case 'title-search': - await this.titleSearch(req, ws); break; + case 'search': + await this.search(req, ws); break; case 'get-author-book-list': await this.getAuthorBookList(req, ws); break; case 'get-series-book-list': @@ -137,35 +133,13 @@ class WebSocketController { this.send((state ? state : {}), req, ws); } - async authorSearch(req, ws) { + async search(req, ws) { if (!req.query) throw new Error(`query is empty`); + if (!req.from) + throw new Error(`from is empty`); - const result = await this.webWorker.authorSearch(req.query); - - this.send(result, req, ws); - } - - async seriesSearch(req, ws) { - if (!this.config.extendedSearch) - throw new Error(`Extended search disabled`); - - if (!req.query) - throw new Error(`query is empty`); - - const result = await this.webWorker.seriesSearch(req.query); - - this.send(result, req, ws); - } - - async titleSearch(req, ws) { - if (!this.config.extendedSearch) - throw new Error(`Extended search disabled`); - - if (!req.query) - throw new Error(`query is empty`); - - const result = await this.webWorker.titleSearch(req.query); + const result = await this.webWorker.search(req.from, req.query); this.send(result, req, ws); } diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 61af5d3..6118fa2 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -494,7 +494,9 @@ class DbCreator { } if (rec.books.length) { - rec[restoreProp] = rec.books[0][restoreProp]; + rec[restoreProp] = rec.value;//rec.books[0][restoreProp]; + if (!rec[restoreProp]) + rec[restoreProp] = emptyFieldValue; } else { rec.toDel = 1; } diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 7d2da99..6d35073 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -20,6 +20,7 @@ class DbSearcher { this.closed = false; this.memCache = new Map(); + this.bookIdMap = {}; this.periodicCleanCache();//no await } @@ -54,40 +55,74 @@ class DbSearcher { return where; } - async selectAuthorIds(query) { + async selectBookIds(query) { const db = this.db; - const authorKеy = `author-ids-author-${query.author}`; - let authorIds = await this.getCached(authorKеy); - - //сначала выберем все id авторов по фильтру - //порядок id соответствует ASC-сортировке по author - if (authorIds === null) { - if (query.author && query.author !== '*') { - const where = this.getWhere(query.author); - - const authorRows = await db.select({ - table: 'author', - rawResult: true, - where: `return Array.from(${where})`, - }); - - authorIds = authorRows[0].rawResult; - } else {//все авторы - const authorRows = await db.select({ - table: 'author', - rawResult: true, - where: `return Array.from(@all())`, - }); - - authorIds = authorRows[0].rawResult; - } - - await this.putCached(authorKеy, authorIds); - } - const idsArr = []; + const tableBookIds = async(table, where) => { + const rows = await db.select({ + table, + rawResult: true, + where: ` + const ids = ${where}; + + const result = new Set(); + for (const id of ids) { + const row = @unsafeRow(id); + for (const bookId of row.bookIds) + result.add(bookId); + } + + return Array.from(result); + ` + }); + + return rows[0].rawResult; + }; + + //авторы + if (query.author && query.author !== '*') { + const key = `book-ids-author-${query.author}`; + let ids = await this.getCached(key); + + if (ids === null) { + ids = await tableBookIds('author', this.getWhere(query.author)); + + await this.putCached(key, ids); + } + + idsArr.push(ids); + } + + //серии + if (query.series && query.series !== '*') { + const key = `book-ids-series-${query.series}`; + let ids = await this.getCached(key); + + if (ids === null) { + ids = await tableBookIds('series', this.getWhere(query.series)); + + await this.putCached(key, ids); + } + + idsArr.push(ids); + } + + //названия + if (query.title && query.title !== '*') { + const key = `book-ids-title-${query.title}`; + let ids = await this.getCached(key); + + if (ids === null) { + ids = await tableBookIds('title', this.getWhere(query.title)); + + await this.putCached(key, ids); + } + + idsArr.push(ids); + } +/* //серии if (query.series && query.series !== '*') { const seriesKеy = `author-ids-series-${query.series}`; @@ -327,23 +362,9 @@ class DbSearcher { idsArr.push(librateIds); } -/* - //ищем пересечение множеств - idsArr.push(authorIds); - +*/ if (idsArr.length > 1) { - const idsSetArr = idsArr.map(ids => new Set(ids)); - authorIds = Array.from(utils.intersectSet(idsSetArr)); - } - - //сортировка - authorIds.sort((a, b) => a - b); -*/ - - //ищем пересечение множеств, работает быстрее предыдущего - if (idsArr.length) { - idsArr.push(authorIds); - + //ищем пересечение множеств let proc = 0; let nextProc = 0; let inter = new Set(idsArr[0]); @@ -363,340 +384,120 @@ class DbSearcher { } inter = newInter; } - authorIds = Array.from(inter); - } - //сортировка - authorIds.sort((a, b) => a - b); - return authorIds; + return Array.from(inter); + } else if (idsArr.length == 1) { + return idsArr[0]; + } else { + return false; + } } - getWhere2(query, ids, exclude = '') { - const db = this.db; + async fillBookIdMap(from) { + if (!this.bookIdMap[from]) { + const db = this.db; + const map = new Map(); + const table = `${from}_id`; - const filterBySearch = (searchValue) => { - searchValue = searchValue.toLowerCase(); + await db.open({table}); + const rows = await db.select({table}); + await db.close({table}); - //особая обработка префиксов - if (searchValue[0] == '=') { + for (const row of rows) { + if (!row.value.length) + continue; - searchValue = searchValue.substring(1); - return `bookValue.localeCompare(${db.esc(searchValue)}) == 0`; - } else if (searchValue[0] == '*') { - - searchValue = searchValue.substring(1); - return `bookValue !== ${db.esc(emptyFieldValue)} && bookValue.indexOf(${db.esc(searchValue)}) >= 0`; - } else if (searchValue[0] == '#') { - - searchValue = searchValue.substring(1); - return `!bookValue || (bookValue !== ${db.esc(emptyFieldValue)} && !enru.has(bookValue[0]) && bookValue.indexOf(${db.esc(searchValue)}) >= 0)`; - } else { - return `bookValue.localeCompare(${db.esc(searchValue)}) >= 0 && bookValue.localeCompare(${db.esc(searchValue + maxUtf8Char)}) <= 0`; + if (row.value.length > 1) + map.set(row.id, row.value); + else + map.set(row.id, row.value[0]); } - }; - //подготовка фильтра - let filter = ''; - let closures = ''; - - //порядок важен, более простые проверки вперед - - //удаленные - if (query.del !== undefined) { - filter += ` - if (book.del !== ${db.esc(query.del)}) - return false; - `; + this.bookIdMap[from] = map; } - - //дата поступления - if (query.date) { - let [from = '0000-00-00', to = '9999-99-99'] = query.date.split(','); - filter += ` - if (!(book.date >= ${db.esc(from)} && book.date <= ${db.esc(to)})) - return false; - `; - } - - //оценка - if (query.librate) { - closures += ` - const searchLibrate = new Set(${db.esc(query.librate.split(',').map(n => parseInt(n, 10)).filter(n => !isNaN(n)))}); - `; - filter += ` - if (!searchLibrate.has(book.librate)) - return false; - `; - } - - //серии - if (exclude !== 'series' && query.series && query.series !== '*') { - closures += ` - const checkSeries = (bookValue) => { - if (!bookValue) - bookValue = ${db.esc(emptyFieldValue)}; - - bookValue = bookValue.toLowerCase(); - - return ${filterBySearch(query.series)}; - }; - `; - filter += ` - if (!checkSeries(book.series)) - return false; - `; - } - - //названия - if (exclude !== 'title' && query.title && query.title !== '*') { - closures += ` - const checkTitle = (bookValue) => { - if (!bookValue) - bookValue = ${db.esc(emptyFieldValue)}; - - bookValue = bookValue.toLowerCase(); - - return ${filterBySearch(query.title)}; - }; - `; - filter += ` - if (!checkTitle(book.title)) - return false; - `; - } - - //языки - if (exclude !== 'lang' && query.lang) { - const queryLangs = query.lang.split(','); - - closures += ` - const queryLangs = new Set(${db.esc(queryLangs)}); - - const checkLang = (bookValue) => { - if (!bookValue) - bookValue = ${db.esc(emptyFieldValue)}; - - return queryLangs.has(bookValue); - }; - `; - filter += ` - if (!checkLang(book.lang)) - return false; - `; - } - - //жанры - if (exclude !== 'genre' && query.genre) { - const queryGenres = query.genre.split(','); - - closures += ` - const queryGenres = new Set(${db.esc(queryGenres)}); - - const checkGenre = (bookValue) => { - if (!bookValue) - bookValue = ${db.esc(emptyFieldValue)}; - - return queryGenres.has(bookValue); - }; - `; - filter += ` - const genres = book.genre.split(','); - found = false; - for (const g of genres) { - if (checkGenre(g)) { - found = true; - break; - } - } - - if (!found) - return false; - `; - } - - //авторы - if (exclude !== 'author' && query.author && query.author !== '*') { - closures += ` - const splitAuthor = (author) => { - if (!author) - author = ${db.esc(emptyFieldValue)}; - - const result = author.split(','); - if (result.length > 1) - result.push(author); - - return result; - }; - - const checkAuthor = (bookValue) => { - if (!bookValue) - bookValue = ${db.esc(emptyFieldValue)}; - - bookValue = bookValue.toLowerCase(); - - return ${filterBySearch(query.author)}; - }; - `; - - filter += ` - const author = splitAuthor(book.author); - found = false; - for (const a of author) { - if (checkAuthor(a)) { - found = true; - break; - } - } - - if (!found) - return false; - `; - } - - //формируем where - let where = ''; - if (filter) { - where = ` - const enru = new Set(${db.esc(enruArr)}); - - ${closures} - - const filterBook = (book) => { - let found = false; - ${filter} - return true; - }; - - let ids; - if (${!ids}) { - ids = @all(); - } else { - ids = ${db.esc(ids)}; - } - - const result = new Set(); - for (const id of ids) { - const row = @unsafeRow(id); - - if (row) { - for (const book of row.books) { - if (filterBook(book)) { - result.add(id); - break; - } - } - } - } - - return Array.from(result); - `; - } - - return where; + return this.bookIdMap[from]; } - async selectSeriesIds(query) { - const db = this.db; + async filterTableIds(tableIds, from, query) { + let result = tableIds; - let seriesIds = false; - let isAll = !(query.series && query.series !== '*'); + //т.к. авторы идут списком, то дополнительно фильтруем + if (query.author && query.author !== '*') { + const key = `filter-ids-author-${query.author}`; + let authorIds = await this.getCached(key); - //серии - const seriesKеy = `series-ids-series-${query.series}`; - seriesIds = await this.getCached(seriesKеy); - - if (seriesIds === null) { - if (query.series && query.series !== '*') { - const where = this.getWhere(query.series); - - const seriesRows = await db.select({ - table: 'series', + if (authorIds === null) { + const rows = await this.db.select({ + table: 'author', rawResult: true, - where: `return Array.from(${where})`, + where: `return Array.from(${this.getWhere(query.author)})` }); - seriesIds = seriesRows[0].rawResult; - } else { - const seriesRows = await db.select({ - table: 'series', - rawResult: true, - where: `return Array.from(@all())`, - }); + authorIds = rows[0].rawResult; - seriesIds = seriesRows[0].rawResult; + await this.putCached(key, authorIds); } - seriesIds.sort((a, b) => a - b); - - await this.putCached(seriesKеy, seriesIds); + //пересечение tableIds и authorIds + result = []; + const authorIdsSet = new Set(authorIds); + for (const id of tableIds) + if (authorIdsSet.has(id)) + result.push(id); } - const where = this.getWhere2(query, (isAll ? false : seriesIds), 'series'); - - if (where) { - //тяжелый запрос перебором в series_book - const rows = await db.select({ - table: 'series_book', - rawResult: true, - where, - }); - - seriesIds = rows[0].rawResult; - } - - return seriesIds; + return result; } - async selectTitleIds(query) { + async selectTableIds(from, query) { const db = this.db; + const queryKey = this.queryKey(query); + const tableKey = `${from}-table-ids-${queryKey}`; + let tableIds = await this.getCached(tableKey); - let titleIds = false; - let isAll = !(query.title && query.title !== '*'); + if (tableIds === null) { + const bookKey = `book-ids-${queryKey}`; + let bookIds = await this.getCached(bookKey); - //серии - const titleKеy = `title-ids-title-${query.title}`; - titleIds = await this.getCached(titleKеy); - - if (titleIds === null) { - if (query.title && query.title !== '*') { - const where = this.getWhere(query.title); - - const titleRows = await db.select({ - table: 'title', - rawResult: true, - where: `return Array.from(${where})`, - }); - - titleIds = titleRows[0].rawResult; - } else { - const titleRows = await db.select({ - table: 'title', - rawResult: true, - where: `return Array.from(@all())`, - }); - - titleIds = titleRows[0].rawResult; + if (bookIds === null) { + bookIds = await this.selectBookIds(query); + await this.putCached(bookKey, bookIds); } - titleIds.sort((a, b) => a - b); + if (bookIds) { + const tableIdsSet = new Set(); + const bookIdMap = await this.fillBookIdMap(from); + for (const bookId of bookIds) { + const tableIdValue = bookIdMap.get(bookId); + if (!tableIdValue) + continue; - await this.putCached(titleKеy, titleIds); + if (Array.isArray(tableIdValue)) { + for (const tableId of tableIdValue) + tableIdsSet.add(tableId); + } else + tableIdsSet.add(tableIdValue); + } + + tableIds = Array.from(tableIdsSet); + } else { + const rows = await db.select({ + table: from, + rawResult: true, + where: `return Array.from(@all())` + }); + + tableIds = rows[0].rawResult; + } + + tableIds = await this.filterTableIds(tableIds, from, query); + + tableIds.sort((a, b) => a - b); + + await this.putCached(tableKey, tableIds); } - const where = this.getWhere2(query, (isAll ? false : titleIds), 'title'); - - if (where) { - //тяжелый запрос перебором в title_book - const rows = await db.select({ - table: 'title_book', - rawResult: true, - where, - }); - - titleIds = rows[0].rawResult; - } - - return titleIds; + return tableIds; } async getCached(key) { @@ -771,112 +572,33 @@ class DbSearcher { }); } - async authorSearch(query) { + async search(from, query) { if (this.closed) throw new Error('DbSearcher closed'); + if (!['author', 'series', 'title'].includes(from)) + throw new Error(`Unknown value for param 'from'`); + this.searchFlag++; try { const db = this.db; - const key = `author-ids-${this.queryKey(query)}`; + const ids = await this.selectTableIds(from, query); - //сначала попробуем найти в кеше - let authorIds = await this.getCached(key); - if (authorIds === null) {//не нашли в кеше, ищем в поисковых таблицах - authorIds = await this.selectAuthorIds(query); - - await this.putCached(key, authorIds); - } - - const totalFound = authorIds.length; + const totalFound = ids.length; let limit = (query.limit ? query.limit : 100); limit = (limit > maxLimit ? maxLimit : limit); const offset = (query.offset ? query.offset : 0); //выборка найденных авторов - const result = await db.select({ - table: 'author', - map: `(r) => ({id: r.id, author: r.author, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, - where: `@@id(${db.esc(authorIds.slice(offset, offset + limit))})` + const found = await db.select({ + table: `${from}_book`, + map: `(r) => ({id: r.id, ${from}: r.${from}, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, + where: `@@id(${db.esc(ids.slice(offset, offset + limit))})` }); - return {result, totalFound}; - } finally { - this.searchFlag--; - } - } - - async seriesSearch(query) { - if (this.closed) - throw new Error('DbSearcher closed'); - - this.searchFlag++; - - try { - const db = this.db; - - const key = `series-ids-${this.queryKey(query)}`; - - //сначала попробуем найти в кеше - let seriesIds = await this.getCached(key); - if (seriesIds === null) {//не нашли в кеше, ищем в поисковых таблицах - seriesIds = await this.selectSeriesIds(query); - - await this.putCached(key, seriesIds); - } - - const totalFound = seriesIds.length; - let limit = (query.limit ? query.limit : 100); - limit = (limit > maxLimit ? maxLimit : limit); - const offset = (query.offset ? query.offset : 0); - - //выборка найденных авторов - const result = await db.select({ - table: 'series_book', - map: `(r) => ({id: r.id, series: r.series, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, - where: `@@id(${db.esc(seriesIds.slice(offset, offset + limit))})` - }); - - return {result, totalFound}; - } finally { - this.searchFlag--; - } - } - - async titleSearch(query) { - if (this.closed) - throw new Error('DbSearcher closed'); - - this.searchFlag++; - - try { - const db = this.db; - - const key = `title-ids-${this.queryKey(query)}`; - - //сначала попробуем найти в кеше - let titleIds = await this.getCached(key); - if (titleIds === null) {//не нашли в кеше, ищем в поисковых таблицах - titleIds = await this.selectTitleIds(query); - - await this.putCached(key, titleIds); - } - - const totalFound = titleIds.length; - let limit = (query.limit ? query.limit : 100); - limit = (limit > maxLimit ? maxLimit : limit); - const offset = (query.offset ? query.offset : 0); - - //выборка найденных авторов - const result = await db.select({ - table: 'title_book', - map: `(r) => ({id: r.id, title: r.title, books: r.books, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, - where: `@@id(${db.esc(titleIds.slice(offset, offset + limit))})` - }); - - return {result, totalFound}; + return {found, totalFound}; } finally { this.searchFlag--; } @@ -908,7 +630,7 @@ class DbSearcher { books = rows[0].books; } - return {author, books}; + return {author, books: (books && books.length ? JSON.stringify(books) : '')}; } finally { this.searchFlag--; } diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index 2c5fdaa..c895aa6 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -187,14 +187,8 @@ class WebWorker { }, }); - //открываем почти все таблицы - await db.openAll({exclude: ['author', 'title_book']}); - - //откроем таблицу 'author' с бОльшим размером кеша блоков, для ускорения выборки - await db.open({table: 'author', cacheSize: (config.dbCacheSize > 100 ? config.dbCacheSize : 100)}); - - if (config.extendedSearch) - await db.open({table: 'title_book'}); + //открываем таблицы + await db.openAll({exclude: ['author_id', 'series_id', 'title_id']}); this.dbSearcher = new DbSearcher(config, db); @@ -242,43 +236,15 @@ class WebWorker { return db.wwCache.config; } - async authorSearch(query) { + async search(from, query) { this.checkMyState(); - const config = await this.dbConfig(); - const result = await this.dbSearcher.authorSearch(query); - - return { - author: result.result, - totalFound: result.totalFound, - inpxHash: (config.inpxHash ? config.inpxHash : ''), - }; - } - - async seriesSearch(query) { - this.checkMyState(); + const result = await this.dbSearcher.search(from, query); const config = await this.dbConfig(); - const result = await this.dbSearcher.seriesSearch(query); + result.inpxHash = (config.inpxHash ? config.inpxHash : ''); - return { - series: result.result, - totalFound: result.totalFound, - inpxHash: (config.inpxHash ? config.inpxHash : ''), - }; - } - - async titleSearch(query) { - this.checkMyState(); - - const config = await this.dbConfig(); - const result = await this.dbSearcher.titleSearch(query); - - return { - title: result.result, - totalFound: result.totalFound, - inpxHash: (config.inpxHash ? config.inpxHash : ''), - }; + return result; } async getAuthorBookList(authorId) { @@ -522,7 +488,7 @@ class WebWorker { async periodicLogServerStats() { while (1) {// eslint-disable-line this.logServerStats(); - await utils.sleep(60*1000); + await utils.sleep(1000)//(60*1000); } }