From 0beaa611f649c2926db0d67e837ea6d9d811df6a Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 2 Dec 2021 18:36:49 +0700 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D1=85=D0=BE=D0=B4=20?= =?UTF-8?q?=D0=BD=D0=B0=20JembaDb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/config/base.js | 8 ++ server/controllers/ReaderController.js | 4 +- server/controllers/WebSocketController.js | 4 +- server/core/Reader/JembaReaderStorage.js | 122 ++++++++++++++++++ server/core/utils.js | 5 + server/db/JembaConnManager.js | 4 +- .../reader-storage/001-create.js | 3 +- server/index.js | 3 + 8 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 server/core/Reader/JembaReaderStorage.js diff --git a/server/config/base.js b/server/config/base.js index 8211cad4..6e400a07 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -37,6 +37,14 @@ module.exports = { } ], + jembaDb: [ + { + dbName: 'reader-storage', + thread: true, + openAll: true, + } + ], + servers: [ { serverName: '1', diff --git a/server/controllers/ReaderController.js b/server/controllers/ReaderController.js index 87f69464..0dea8925 100644 --- a/server/controllers/ReaderController.js +++ b/server/controllers/ReaderController.js @@ -1,12 +1,12 @@ const BaseController = require('./BaseController'); const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton -const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton +const JembaReaderStorage = require('../core/Reader/JembaReaderStorage');//singleton const WorkerState = require('../core/WorkerState');//singleton class ReaderController extends BaseController { constructor(config) { super(config); - this.readerStorage = new ReaderStorage(); + this.readerStorage = new JembaReaderStorage(); this.readerWorker = new ReaderWorker(config); this.workerState = new WorkerState(); } diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js index cb147288..1ef46135 100644 --- a/server/controllers/WebSocketController.js +++ b/server/controllers/WebSocketController.js @@ -2,7 +2,7 @@ const WebSocket = require ('ws'); const _ = require('lodash'); const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton -const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton +const JembaReaderStorage = require('../core/Reader/JembaReaderStorage');//singleton const WorkerState = require('../core/WorkerState');//singleton const log = new (require('../core/AppLogger'))().log;//singleton const utils = require('../core/utils'); @@ -15,7 +15,7 @@ class WebSocketController { this.config = config; this.isDevelopment = (config.branch == 'development'); - this.readerStorage = new ReaderStorage(); + this.readerStorage = new JembaReaderStorage(); this.readerWorker = new ReaderWorker(config); this.workerState = new WorkerState(); diff --git a/server/core/Reader/JembaReaderStorage.js b/server/core/Reader/JembaReaderStorage.js new file mode 100644 index 00000000..8e7a00e8 --- /dev/null +++ b/server/core/Reader/JembaReaderStorage.js @@ -0,0 +1,122 @@ +const _ = require('lodash'); + +const utils = require('../utils'); +const JembaConnManager = require('../../db/JembaConnManager');//singleton + +let instance = null; + +//singleton +class JembaReaderStorage { + constructor() { + if (!instance) { + this.connManager = new JembaConnManager(); + this.db = this.connManager.db['reader-storage']; + this.periodicCleanCache(3*3600*1000);//1 раз в 3 часа + + instance = this; + } + + return instance; + } + + async doAction(act) { + if (!_.isObject(act.items)) + throw new Error('items is not an object'); + + let result = {}; + switch (act.action) { + case 'check': + result = await this.checkItems(act.items); + break; + case 'get': + result = await this.getItems(act.items); + break; + case 'set': + result = await this.setItems(act.items, act.force); + break; + default: + throw new Error('Unknown action'); + } + + return result; + } + + async checkItems(items) { + let result = {state: 'success', items: {}}; + + const db = this.db; + + for (const id of Object.keys(items)) { + if (this.cache[id]) { + result.items[id] = this.cache[id]; + } else { + const rows = await db.select({//SQL`SELECT rev FROM storage WHERE id = ${id}` + table: 'storage', + map: '(r) => ({rev: r.rev})', + where: `@@id(${db.esc(id)})` + }); + const rev = (rows.length && rows[0].rev ? rows[0].rev : 0); + result.items[id] = {rev}; + this.cache[id] = result.items[id]; + } + } + + return result; + } + + async getItems(items) { + let result = {state: 'success', items: {}}; + + const db = this.db; + + for (const id of Object.keys(items)) { + const rows = await db.select({//SQL`SELECT rev, data FROM storage WHERE id = ${id}`); + table: 'storage', + where: `@@id(${db.esc(id)})` + }); + const rev = (rows.length && rows[0].rev ? rows[0].rev : 0); + const data = (rows.length && rows[0].data ? rows[0].data : ''); + result.items[id] = {rev, data}; + } + + return result; + } + + async setItems(items, force) { + let check = await this.checkItems(items); + + //сначала проверим совпадение ревизий + for (const id of Object.keys(items)) { + if (!_.isString(items[id].data)) + throw new Error('items.data is not a string'); + + if (!force && check.items[id].rev + 1 !== items[id].rev) + return {state: 'reject', items: check.items}; + } + + const db = this.db; + const newRev = {}; + for (const id of Object.keys(items)) { + await db.insert({//SQL`INSERT OR REPLACE INTO storage (id, rev, time, data) VALUES (${id}, ${items[id].rev}, strftime('%s','now'), ${items[id].data})`); + table: 'storage', + replace: true, + rows: [{id, rev: items[id].rev, time: utils.toUnixTime(Date.now()), data: items[id].data}], + }); + newRev[id] = {rev: items[id].rev}; + } + + Object.assign(this.cache, newRev); + + return {state: 'success'}; + } + + periodicCleanCache(timeout) { + this.cache = {}; + + setTimeout(() => { + this.periodicCleanCache(timeout); + }, timeout); + } +} + +module.exports = JembaReaderStorage; \ No newline at end of file diff --git a/server/core/utils.js b/server/core/utils.js index 587b72bf..18017223 100644 --- a/server/core/utils.js +++ b/server/core/utils.js @@ -38,6 +38,10 @@ function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } +function toUnixTime(time) { + return parseInt(time/1000); +} + function randomHexString(len) { return crypto.randomBytes(len).toString('hex') } @@ -126,6 +130,7 @@ module.exports = { bufferRemoveZeroes, getFileHash, sleep, + toUnixTime, randomHexString, touchFile, spawnProcess, diff --git a/server/db/JembaConnManager.js b/server/db/JembaConnManager.js index 50cc61d3..05d8a620 100644 --- a/server/db/JembaConnManager.js +++ b/server/db/JembaConnManager.js @@ -28,7 +28,7 @@ class JembaConnManager { this.config = config; this._db = {}; - for (const dbConfig of this.config.db) { + for (const dbConfig of this.config.jembaDb) { const dbPath = `${this.config.dataDir}/db/${dbConfig.dbName}`; //бэкап @@ -90,7 +90,7 @@ class JembaConnManager { if (!this.inited) return; - for (const dbConfig of this.config.db) { + for (const dbConfig of this.config.jembaDb) { await this._db[dbConfig.dbName].closeDb(); } diff --git a/server/db/jembaMigrations/reader-storage/001-create.js b/server/db/jembaMigrations/reader-storage/001-create.js index 99e7b9b1..bb14583b 100644 --- a/server/db/jembaMigrations/reader-storage/001-create.js +++ b/server/db/jembaMigrations/reader-storage/001-create.js @@ -2,8 +2,7 @@ module.exports = { up: [ //CREATE TABLE storage (id TEXT PRIMARY KEY, rev INTEGER, time INTEGER, data TEXT); ['create', { - table: 'storage', - hash: {field: 'id', type: 'string', depth: 100} + table: 'storage' }], ], down: [ diff --git a/server/index.js b/server/index.js index c4b8a408..10729808 100644 --- a/server/index.js +++ b/server/index.js @@ -46,6 +46,9 @@ async function init() { //connections const connManager = new (require('./db/ConnManager'))();//singleton await connManager.init(config); + + const jembaConnManager = new (require('./db/JembaConnManager'))();//singleton + await jembaConnManager.init(config); } async function main() {