From 5c77b1711b18d9fc841789d2d2cf5aaafb4cb188 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 1 Dec 2022 19:05:34 +0700 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20=D0=BD?= =?UTF-8?q?=D0=B0=D0=B4=20=D1=80=D0=B0=D1=81=D1=88=D0=B8=D1=80=D0=B5=D0=BD?= =?UTF-8?q?=D0=BD=D1=8B=D0=BC=20=D0=BF=D0=BE=D0=B8=D1=81=D0=BA=D0=BE=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 158 +++++++++++++++++++++++++++++++------- server/core/InpxParser.js | 49 ++++++------ server/core/WebWorker.js | 1 + 3 files changed, 157 insertions(+), 51 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index fe3012f..130c924 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -1,6 +1,5 @@ const fs = require('fs-extra'); //const _ = require('lodash'); -const LockQueue = require('./LockQueue'); const utils = require('./utils'); const maxLimit = 1000; @@ -21,7 +20,6 @@ class DbSearcher { this.db = db; - this.lock = new LockQueue(); this.searchFlag = 0; this.timer = null; this.closed = false; @@ -30,11 +28,22 @@ class DbSearcher { this.bookIdMap = {}; this.periodicCleanCache();//no await - this.fillBookIdMapAll();//no await + } + + async init() { + await this.fillBookIdMap('author'); + await this.fillBookIdMap('series'); + await this.fillBookIdMap('title'); + await this.fillDbConfig(); } queryKey(q) { - return JSON.stringify([q.author, q.series, q.title, q.genre, q.lang, q.del, q.date, q.librate]); + const result = []; + for (const f of this.recStruct) { + result.push(q[f.field]); + } + + return JSON.stringify(result); } getWhere(a) { @@ -298,34 +307,28 @@ class DbSearcher { } } - async fillBookIdMap(from) { - if (this.bookIdMap[from]) - return this.bookIdMap[from]; + async fillDbConfig() { + if (!this.dbConfig) { + const rows = await this.db.select({table: 'config'}); + const config = {}; - await this.lock.get(); - try { - const data = await fs.readFile(`${this.config.dataDir}/db/${from}_id.map`, 'utf-8'); + for (const row of rows) { + config[row.id] = row.value; + } - const idMap = JSON.parse(data); - idMap.arr = new Uint32Array(idMap.arr); - idMap.map = new Map(idMap.map); - - this.bookIdMap[from] = idMap; - - return this.bookIdMap[from]; - } finally { - this.lock.ret(); + this.dbConfig = config; + this.recStruct = config.inpxInfo.recStruct; } } - async fillBookIdMapAll() { - try { - await this.fillBookIdMap('author'); - await this.fillBookIdMap('series'); - await this.fillBookIdMap('title'); - } catch (e) { - throw new Error(`DbSearcher.fillBookIdMapAll error: ${e.message}`) - } + async fillBookIdMap(from) { + const data = await fs.readFile(`${this.config.dataDir}/db/${from}_id.map`, 'utf-8'); + + const idMap = JSON.parse(data); + idMap.arr = new Uint32Array(idMap.arr); + idMap.map = new Map(idMap.map); + + this.bookIdMap[from] = idMap; } async tableIdsFilter(from, query) { @@ -376,7 +379,7 @@ class DbSearcher { const filter = await this.tableIdsFilter(from, query); const tableIdsSet = new Set(); - const idMap = await this.fillBookIdMap(from); + const idMap = this.bookIdMap[from]; let proc = 0; let nextProc = 0; for (const bookId of bookIds) { @@ -534,6 +537,105 @@ class DbSearcher { } } + async bookSearchIds(query) { + const ids = await this.selectBookIds(query); + const queryKey = this.queryKey(query); + const bookKey = `book-search-ids-${queryKey}`; + let bookIds = await this.getCached(bookKey); + + if (bookIds === null) { + const db = this.db; + const filterBySearch = (bookField, searchValue) => { + //особая обработка префиксов + if (searchValue[0] == '=') { + searchValue = searchValue.substring(1); + return `(row.${bookField}.localeCompare(${db.esc(searchValue)}) === 0)`; + } else if (searchValue[0] == '*') { + searchValue = searchValue.substring(1); + return `(row.${bookField} && row.${bookField}.indexOf(${db.esc(searchValue)}) >= 0)`; + } else if (searchValue[0] == '#') { + + //searchValue = searchValue.substring(1); + //return !bookValue || (bookValue !== emptyFieldValue && !enru.has(bookValue[0]) && bookValue.indexOf(searchValue) >= 0); + return 'true'; + } else { + return `(row.${bookField}.localeCompare(${db.esc(searchValue)}) >= 0 && row.${bookField}.localeCompare(${db.esc(searchValue)} + maxUtf8Char) <= 0)`; + } + }; + + const checks = ['true']; + for (const f of this.recStruct) { + if (query[f.field]) { + let searchValue = query[f.field]; + if (f.type === 'S') { + checks.push(filterBySearch(f.field, searchValue)); + } if (f.type === 'N') { + searchValue = parseInt(searchValue, 10); + checks.push(`row.${f.field} === ${searchValue}`); + } + } + } + + const rows = await db.select({ + table: 'book', + rawResult: true, + where: ` + const ids = ${(ids ? db.esc(Array.from(ids)) : '@all()')}; + + const checkBook = (row) => { + return ${checks.join(' && ')}; + }; + + const result = new Set(); + for (const id of ids) { + const row = @unsafeRow(id); + if (checkBook(row)) + result.add(row.id); + } + + return new Uint32Array(result); + ` + }); + + bookIds = rows[0].rawResult; + + await this.putCached(bookKey, bookIds); + } + + return bookIds; + } + + //неоптимизированный поиск по всем книгам, по всем полям + async bookSearch(query) { + if (this.closed) + throw new Error('DbSearcher closed'); + + this.searchFlag++; + + try { + const db = this.db; + + const ids = await this.bookSearchIds(query); + + const totalFound = ids.length; + let limit = (query.limit ? query.limit : 100); + limit = (limit > maxLimit ? maxLimit : limit); + const offset = (query.offset ? query.offset : 0); + + const slice = ids.slice(offset, offset + limit); + + //выборка найденных значений + const found = await db.select({ + table: 'book', + where: `@@id(${db.esc(Array.from(slice))})` + }); + + return {found, totalFound}; + } finally { + this.searchFlag--; + } + } + async opdsQuery(from, query) { if (this.closed) throw new Error('DbSearcher closed'); diff --git a/server/core/InpxParser.js b/server/core/InpxParser.js index a9e5418..2dc3018 100644 --- a/server/core/InpxParser.js +++ b/server/core/InpxParser.js @@ -9,23 +9,23 @@ const versionInfo = 'version.info'; const defaultStructure = 'AUTHOR;GENRE;TITLE;SERIES;SERNO;FILE;SIZE;LIBID;DEL;EXT;DATE;LANG;LIBRATE;KEYWORDS'; //'AUTHOR;GENRE;TITLE;SERIES;SERNO;FILE;SIZE;LIBID;DEL;EXT;DATE;INSNO;FOLDER;LANG;LIBRATE;KEYWORDS;' -const defaultRecStruct = { - author: 'S', - genre: 'S', - title: 'S', - series: 'S', - serno: 'N', - file: 'S', - size: 'N', - libid: 'S', - del: 'N', - ext: 'S', - date: 'S', - insno: 'N', - folder: 'S', - lang: 'S', - librate: 'N', - keywords: 'S', +const recStructType = { + author: 'S', + genre: 'S', + title: 'S', + series: 'S', + serno: 'N', + file: 'S', + size: 'N', + libid: 'S', + del: 'N', + ext: 'S', + date: 'S', + insno: 'N', + folder: 'S', + lang: 'S', + librate: 'N', + keywords: 'S', } class InpxParser { @@ -45,13 +45,16 @@ class InpxParser { } getRecStruct(structure) { - const result = {}; - for (const field of structure) - if (utils.hasProp(defaultRecStruct, field)) - result[field] = defaultRecStruct[field]; - + const result = []; + let struct = structure; //folder есть всегда - result['folder'] = defaultRecStruct['folder']; + if (!struct.includes('folder')) + struct = struct.concat(['folder']); + + for (const field of struct) { + if (utils.hasProp(recStructType, field)) + result.push({field, type: recStructType[field]}); + } return result; } diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index 50a89e4..7ce88ad 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -210,6 +210,7 @@ class WebWorker { //поисковый движок this.dbSearcher = new DbSearcher(config, db); + await this.dbSearcher.init(); //stuff db.wwCache = {};