From 904e3e6c2f477db9d7c892a428275df6984146f3 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 3 Oct 2022 16:25:51 +0700 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=B2=D0=BE=D0=B7=D0=BC=D0=BE=D0=B6=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D1=8C=20=D1=84=D0=B8=D0=BB=D1=8C=D1=82=D1=80=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D0=B8=20=D0=BF=D0=BE=20=D0=B0=D0=B2=D1=82=D0=BE?= =?UTF-8?q?=D1=80=D0=B0=D0=BC=20=D0=BF=D1=80=D0=B8=20=D1=84=D0=BE=D1=80?= =?UTF-8?q?=D0=BC=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B8=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B8=D1=81=D0=BA=D0=BE=D0=B2=D0=BE=D0=B9=20=D0=91=D0=94?= =?UTF-8?q?.=20=D0=9A=D1=80=D0=B8=D1=82=D0=B5=D1=80=D0=B8=D0=B8=20=D1=84?= =?UTF-8?q?=D0=B8=D0=BB=D1=8C=D1=82=D1=80=D0=B0=D1=86=D0=B8=D0=B8=20=D0=B4?= =?UTF-8?q?=D0=BE=D0=BB=D0=B6=D0=BD=D1=8B=20=D0=BD=D0=B0=D1=85=D0=BE=D0=B4?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=D1=81=D1=8F=20=D0=B2=20=D1=84=D0=B0=D0=B9?= =?UTF-8?q?=D0=BB=D0=B5=20inpx-web-filter.json?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbCreator.js | 80 +++++++++++++++++++++++++++++++--- server/core/InpxHashCreator.js | 28 ++++++++++++ server/core/WebWorker.js | 6 ++- server/index.js | 1 + 4 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 server/core/InpxHashCreator.js diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 6becb12..a071f64 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -1,4 +1,7 @@ +const fs = require('fs-extra'); + const InpxParser = require('./InpxParser'); +const InpxHashCreator = require('./InpxHashCreator'); const utils = require('./utils'); const emptyFieldValue = '?'; @@ -8,6 +11,29 @@ class DbCreator { this.config = config; } + async loadInpxFilter() { + const inpxFilterFile = this.config.inpxFilterFile; + + if (await fs.pathExists(inpxFilterFile)) { + let filter = await fs.readFile(inpxFilterFile, 'utf8'); + filter = JSON.parse(filter); + + if (filter.includeAuthors) { + filter.includeAuthors = filter.includeAuthors.map(a => a.toLowerCase()); + filter.includeSet = new Set(filter.includeAuthors); + } + + if (filter.excludeAuthors) { + filter.excludeAuthors = filter.excludeAuthors.map(a => a.toLowerCase()); + filter.excludeSet = new Set(filter.excludeAuthors); + } + + return filter; + } else { + return false; + } + } + //процедура формировани БД несколько усложнена, в целях экономии памяти async run(db, callback) { const config = this.config; @@ -44,10 +70,37 @@ class DbCreator { callback({recsLoaded}); let chunkNum = 0; + //фильтр по авторам + const inpxFilter = await this.loadInpxFilter(); + let filterAuthor = () => true; + if (inpxFilter) { + filterAuthor = (author) => { + if (!author) + author = emptyFieldValue; + + author = author.toLowerCase(); + + let excluded = false; + if (inpxFilter.excludeSet) { + const authors = author.split(','); + + for (const a of authors) { + if (inpxFilter.excludeSet.has(a)) { + excluded = true; + break; + } + } + } + + return (!inpxFilter.includeSet || inpxFilter.includeSet.has(author)) && !excluded + ; + }; + } + + //вспомогательные функции const splitAuthor = (author) => { - if (!author) { + if (!author) author = emptyFieldValue; - } const result = author.split(','); if (result.length > 1) @@ -69,7 +122,15 @@ class DbCreator { let id = 0; const parsedCallback = async(chunk) => { + let filtered = false; for (const rec of chunk) { + //сначала фильтр по авторам + if (!filterAuthor(rec.author)) { + rec.id = 0; + filtered = true; + continue; + } + rec.id = ++id; if (!rec.del) { @@ -116,7 +177,14 @@ class DbCreator { } } - await db.insert({table: 'book', rows: chunk}); + let saveChunk = []; + if (filtered) { + saveChunk = chunk.filter(r => r.id); + } else { + saveChunk = chunk; + } + + await db.insert({table: 'book', rows: saveChunk}); recsLoaded += chunk.length; callback({recsLoaded}); @@ -373,12 +441,12 @@ class DbCreator { }; //console.log(stats); - const inpxHash = await utils.getFileHash(config.inpxFile, 'sha256', 'hex'); + const inpxHashCreator = new InpxHashCreator(config); await db.insert({table: 'config', rows: [ - {id: 'inpxInfo', value: parser.info}, + {id: 'inpxInfo', value: (inpxFilter && inpxFilter.info ? inpxFilter.info : parser.info)}, {id: 'stats', value: stats}, - {id: 'inpxHash', value: inpxHash}, + {id: 'inpxHash', value: await inpxHashCreator.getHash()}, ]}); //сохраним поисковые таблицы diff --git a/server/core/InpxHashCreator.js b/server/core/InpxHashCreator.js new file mode 100644 index 0000000..88566de --- /dev/null +++ b/server/core/InpxHashCreator.js @@ -0,0 +1,28 @@ +const fs = require('fs-extra'); + +const utils = require('./utils'); + +//поправить в случае, если изменился порядок формирования id для авторов в DbCreator +//иначе будет неправильно работать кеширование на клиенте +const dbCreatorVersion = '1'; + +class InpxHashCreator { + constructor(config) { + this.config = config; + } + + async getHash() { + const config = this.config; + + let inpxFilterHash = ''; + if (await fs.pathExists(config.inpxFilterFile)) + inpxFilterHash = await utils.getFileHash(config.inpxFilterFile, 'sha256', 'hex'); + + const joinedHash = dbCreatorVersion + inpxFilterHash + + await utils.getFileHash(config.inpxFile, 'sha256', 'hex'); + + return utils.getBufHash(joinedHash, 'sha256', 'hex'); + } +} + +module.exports = InpxHashCreator; \ No newline at end of file diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index b285b7c..f8ab542 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -9,6 +9,7 @@ const WorkerState = require('./WorkerState'); const { JembaDbThread } = require('jembadb'); const DbCreator = require('./DbCreator'); const DbSearcher = require('./DbSearcher'); +const InpxHashCreator = require('./InpxHashCreator'); const ayncExit = new (require('./AsyncExit'))(); const log = new (require('./AppLogger'))().log;//singleton @@ -497,13 +498,14 @@ class WebWorker { if (!inpxCheckInterval) return; + const inpxHashCreator = new InpxHashCreator(this.config); + while (1) {// eslint-disable-line no-constant-condition try { while (this.myState != ssNormal) await utils.sleep(1000); - log('check inpx file for changes'); - const newInpxHash = await utils.getFileHash(this.config.inpxFile, 'sha256', 'hex'); + const newInpxHash = await inpxHashCreator.getHash(); const dbConfig = await this.dbConfig(); const currentInpxHash = (dbConfig.inpxHash ? dbConfig.inpxHash : ''); diff --git a/server/index.js b/server/index.js index fcdb39c..c3023c9 100644 --- a/server/index.js +++ b/server/index.js @@ -103,6 +103,7 @@ async function init() { } config.recreateDb = argv.recreate || false; + config.inpxFilterFile = `${config.execDir}/inpx-web-filter.json`; //app const appDir = `${config.publicDir}/app`;