Merge branch 'release/0.11.2'

0.11.1-2

.eslintrc

@@ -12,6 +12,7 @@
     "@babel"
   ],
   "env": {
+    "es6": true,
     "browser": true,
     "node": true
   },

build/webpack.base.config.js

@@ -30,7 +30,7 @@ module.exports = {
             },
             {
                 resourceQuery: /^\?vue/,
-                use: path.resolve('build/includer.js')
+                use: path.resolve(__dirname, 'includer.js')
             },
             {
                 test: /\.js$/,
@@ -62,34 +62,6 @@ module.exports = {
                     filename: 'fonts/[name]-[hash:6][ext]'
                 },
             },
-            /*{
-                test: /\.gif$/,
-                loader: "url-loader",
-                options: {
-                    name: "images/[name]-[hash:6].[ext]"
-                }
-            },
-            {
-                test: /\.png$/,
-                loader: "url-loader",
-                options: {
-                    name: "images/[name]-[hash:6].[ext]"
-                }
-            },
-            {
-                test: /\.jpg$/,
-                loader: "file-loader",
-                options: {
-                    name: "images/[name]-[hash:6].[ext]"
-                }
-            },
-            {
-                test: /\.(ttf|eot|woff|woff2)$/,
-                loader: "file-loader",
-                options: {
-                    name: "fonts/[name]-[hash:6].[ext]"
-                }
-            },*/
         ]
     },
 

client/components/Reader/Reader.vue

@@ -310,7 +310,7 @@ class Reader {
                 await this.$nextTick();
                 this.paramPosIgnore = false;
             }
-        }, 500, {maxWait: 5000});
+        }, 250, {maxWait: 5000});
 
         this.scrollingSetRecentBook = _.debounce((newValue) => {
             this.debouncedSetRecentBook(newValue);
@@ -585,7 +585,20 @@ class Reader {
             //сохранение в serverStorage
             if (value) {
                 await utils.sleep(500);
+                
+                let timer = setTimeout(() => {
+                    if (!this.offlineModeActive)
+                        this.$root.notify.error('Таймаут соединения');
+                }, 10000);
+
+                try {
                     await this.$refs.serverStorage.saveRecent(value);
+                } catch (e) {
+                    if (!this.offlineModeActive)
+                        this.$root.notify.error(e.message);
+                } finally {
+                    clearTimeout(timer);
+                }
             }
         }
     }

client/components/Reader/ServerStorage/ServerStorage.vue

@@ -576,7 +576,7 @@ class ServerStorage {
                 newRecentPatch.rev++;
                 newRecentPatch.data[itemKey] = _.cloneDeep(bm.recent[itemKey]);
 
-                let applyMod = this.cachedRecentMod.data;
+                const applyMod = this.cachedRecentMod.data;
                 if (applyMod && applyMod.key && newRecentPatch.data[applyMod.key])
                     newRecentPatch.data[applyMod.key] = utils.applyObjDiff(newRecentPatch.data[applyMod.key], applyMod.mod, {isAddChanged: true});
 
@@ -627,7 +627,7 @@ class ServerStorage {
                     this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`);
                 if (!recurse && itemKey) {
                     this.savingRecent = false;
-                    this.saveRecent(itemKey, true);
+                    await this.saveRecent(itemKey, true);
                     return;
                 }
             } else if (result.state == 'success') {

client/components/Reader/share/bookManager.js

@@ -63,48 +63,6 @@ class BookManager {
             }
 
             await this.cleanRecentBooks();
-
-            //TODO: убрать после 06.2021, когда bmRecentStoreOld устареет
-            {
-                await this.convertFileToDiskPrefix();
-                if (this.recentRev > 10)
-                    await bmRecentStoreOld.clear();
-            }
-        } else {//TODO: убрать после 06.2021, когда bmRecentStoreOld устареет
-            this.recentLast = await bmRecentStoreOld.getItem('recent-last');
-            if (this.recentLast) {
-                this.recent[this.recentLast.key] = this.recentLast;
-                const meta = await bmMetaStore.getItem(`bmMeta-${this.recentLast.key}`);
-                if (_.isObject(meta)) {
-                    this.books[meta.key] = meta;
-                }
-            }
-
-            let key = null;
-            const len = await bmRecentStoreOld.length();
-            for (let i = len - 1; i >= 0; i--) {
-                key = await bmRecentStoreOld.key(i);
-                if (key) {
-                    let r = await bmRecentStoreOld.getItem(key);
-                    if (_.isObject(r) && r.key) {
-                        this.recent[r.key] = r;
-                    }
-                } else  {
-                    await bmRecentStoreOld.removeItem(key);
-                }
-            }
-
-            //размножение для дебага
-            /*if (key) {
-                for (let i = 0; i < 1000; i++) {
-                    const k = this.keyFromUrl(i.toString());
-                    this.recent[k] = Object.assign({}, _.cloneDeep(this.recent[key]), {key: k, touchTime: Date.now() - 1000000, url: utils.randomHexString(300)});
-                }
-            }*/
-
-            await bmRecentStoreNew.setItem('recent', this.recent);
-            this.recentRev = 1;
-            await bmRecentStoreNew.setItem('rev', this.recentRev);
         }
 
         this.recentChanged = true;
@@ -374,7 +332,7 @@ class BookManager {
     //-- recent --------------------------------------------------------------
     async recentSetItem(item = null, skipCheck = false) {
         const rev = await bmRecentStoreNew.getItem('rev');
-        if (rev != this.recentRev && !skipCheck) {
+        if (rev != this.recentRev && !skipCheck) {//если изменение произошло в другой вкладке барузера
             const newRecent = await bmRecentStoreNew.getItem('recent');
             Object.assign(this.recent, newRecent);
             this.recentItem = await bmRecentStoreNew.getItem('recent-item');
@@ -455,33 +413,6 @@ class BookManager {
         return isDel;
     }
 
-    async convertFileToDiskPrefix() {
-        let isConverted = false;
-
-        const newRecent = {};
-        for (let key of Object.keys(this.recent)) {
-            let newKey = key;
-            let newUrl = this.recent[key].url;
-
-            if (newKey.indexOf('66696c65') == 0) {
-                newKey = newKey.replace(/^66696c65/, '6469736b');
-                if (newUrl)
-                    newUrl = newUrl.replace(/^file/, 'disk');
-                isConverted = true;
-            }
-
-            newRecent[newKey] = this.recent[key];
-            newRecent[newKey].key = newKey;
-            if (newUrl)
-                newRecent[newKey].url = newUrl;
-        }
-        if (isConverted) {
-            this.recent = newRecent;
-            await this.recentSetItem(null, true);
-        }
-        return isConverted;
-    }
-
     mostRecentBook() {
         if (this.recentLastKey) {
             return this.recent[this.recentLastKey];

client/components/Reader/versionHistory.js

@@ -1,4 +1,26 @@
 export const versionHistory = [
+{
+    showUntil: '2022-01-10',
+    header: '0.11.2 (2022-01-11)',
+    content:
+`
+<ul>
+    <li>переход на JembaDb вместо SQLite</li>
+</ul>
+`
+},
+
+{
+    showUntil: '2021-12-02',
+    header: '0.11.1 (2021-12-03)',
+    content:
+`
+<ul>
+    <li>переход на JembaDb вместо SQLite</li>
+</ul>
+`
+},
+
 {
     showUntil: '2021-11-17',
     header: '0.11.0 (2021-11-18)',

client/quasar.js

@@ -86,7 +86,6 @@ const plugins = {
 import '@quasar/extras/line-awesome/line-awesome.css';
 import lineAwesome from 'quasar/icon-set/line-awesome.js'
 
-//const q: {Quasar, QuasarOptions: { config, components, directives, plugins }};
 export default {
     quasar: Quasar,
     options: { config, components, directives, plugins }, 

docs/liberama.top/liberama

@@ -35,6 +35,7 @@ server {
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
+    proxy_read_timeout 600s;
   }
 
   location / {

docs/omnireader.ru/omnireader

@@ -24,6 +24,7 @@ server {
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
+    proxy_read_timeout 600s;
   }
 
   location / {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "Liberama",
-  "version": "0.11.0",
+  "version": "0.11.2",
   "author": "Book Pauk <bookpauk@gmail.com>",
   "license": "CC0-1.0",
   "repository": "bookpauk/liberama",
@@ -10,8 +10,8 @@
   "scripts": {
     "dev": "nodemon --inspect --ignore server/public --ignore server/data --ignore client --exec 'node server'",
     "build:client": "webpack --config build/webpack.prod.config.js",
-    "build:linux": "npm run build:client && node build/linux && pkg -t node14-linux-x64 -o dist/linux/liberama .",
+    "build:linux": "npm run build:client && node build/linux && pkg -t node14-linux-x64 -C GZip -o dist/linux/liberama .",
-    "build:win": "npm run build:client && node build/win && pkg -t node14-win-x64 -o dist/win/liberama .",
+    "build:win": "npm run build:client && node build/win && pkg -t node14-win-x64 -C GZip -o dist/win/liberama .",
     "lint": "eslint --ext=.js,.vue client server",
     "build:client-dev": "webpack --config build/webpack.dev.config.js",
     "postinstall": "npm run build:client-dev && node build/linux"
@@ -35,6 +35,7 @@
     "eslint-plugin-vue": "^8.0.3",
     "html-webpack-plugin": "^5.5.0",
     "mini-css-extract-plugin": "^2.4.4",
+    "pkg": "^5.5.1",
     "terser-webpack-plugin": "^5.2.5",
     "vue-eslint-parser": "^8.0.1",
     "vue-loader": "^16.8.3",
@@ -59,6 +60,7 @@
     "got": "^11.8.2",
     "he": "^1.2.0",
     "iconv-lite": "^0.6.3",
+    "jembadb": "^1.3.0",
     "localforage": "^1.10.0",
     "lodash": "^4.17.21",
     "minimist": "^1.2.5",
@@ -66,7 +68,6 @@
     "pako": "^2.0.4",
     "path-browserify": "^1.0.1",
     "pidusage": "^3.0.0",
-    "pkg": "^4.4.9",
     "quasar": "^2.3.2",
     "safe-buffer": "^5.2.1",
     "sanitize-html": "^2.5.3",

server/config/base.js

@@ -37,6 +37,14 @@ module.exports = {
         }
     ],
 
+    jembaDb: [
+        {
+            dbName: 'reader-storage',
+            thread: true,
+            openAll: true,
+        }
+    ],
+
     servers: [
         {
             serverName: '1',

server/controllers/ReaderController.js

@@ -1,12 +1,12 @@
 const BaseController = require('./BaseController');
 const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton
-const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton
+const JembaReaderStorage = require('../core/Reader/JembaReaderStorage');//singleton
 const WorkerState = require('../core/WorkerState');//singleton
 
 class ReaderController extends BaseController {
     constructor(config) {
         super(config);
-        this.readerStorage = new ReaderStorage();
+        this.readerStorage = new JembaReaderStorage();
         this.readerWorker = new ReaderWorker(config);
         this.workerState = new WorkerState();
     }

server/controllers/WebSocketController.js

@@ -2,7 +2,7 @@ const WebSocket = require ('ws');
 const _ = require('lodash');
 
 const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton
-const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton
+const JembaReaderStorage = require('../core/Reader/JembaReaderStorage');//singleton
 const WorkerState = require('../core/WorkerState');//singleton
 const log = new (require('../core/AppLogger'))().log;//singleton
 const utils = require('../core/utils');
@@ -15,7 +15,7 @@ class WebSocketController {
         this.config = config;
         this.isDevelopment = (config.branch == 'development');
 
-        this.readerStorage = new ReaderStorage();
+        this.readerStorage = new JembaReaderStorage();
         this.readerWorker = new ReaderWorker(config);
         this.workerState = new WorkerState();
 

server/core/AppLogger.js

@@ -7,10 +7,14 @@ let instance = null;
 class AppLogger {
     constructor() {
         if (!instance) {
+            this.inited = false;
+            this.logFileName = '';
+            this.errLogFileName = '';
+            this.fatalLogFileName = '';
+
             instance = this;
         }
 
-        this.inited = false;
         return instance;
     }
 
@@ -22,11 +26,16 @@ class AppLogger {
 
         if (config.loggingEnabled) {
             await fs.ensureDir(config.logDir);
+
+            this.logFileName = `${config.logDir}/${config.name}.log`;
+            this.errLogFileName = `${config.logDir}/${config.name}.err.log`;
+            this.fatalLogFileName = `${config.logDir}/${config.name}.fatal.log`;
+
             loggerParams = [
                 {log: 'ConsoleLog'},
-                {log: 'FileLog', fileName: `${config.logDir}/${config.name}.log`},
+                {log: 'FileLog', fileName: this.logFileName},
-                {log: 'FileLog', fileName: `${config.logDir}/${config.name}.err.log`, exclude: [LM_OK, LM_INFO, LM_TOTAL]},
+                {log: 'FileLog', fileName: this.errLogFileName, exclude: [LM_OK, LM_INFO, LM_TOTAL]},
-                {log: 'FileLog', fileName: `${config.logDir}/${config.name}.fatal.log`, exclude: [LM_OK, LM_INFO, LM_WARN, LM_ERR, LM_TOTAL]},//LM_FATAL only
+                {log: 'FileLog', fileName: this.fatalLogFileName, exclude: [LM_OK, LM_INFO, LM_WARN, LM_ERR, LM_TOTAL]},//LM_FATAL only
             ];
         }
 

server/core/AsyncExit.js

@@ -0,0 +1,108 @@
+let instance = null;
+
+const defaultTimeout = 15*1000;//15 sec
+const exitSignals = ['SIGINT', 'SIGTERM', 'SIGBREAK', 'SIGHUP', 'uncaughtException', 'SIGUSR2'];
+
+//singleton
+class AsyncExit {
+    constructor() {
+        if (!instance) {
+            this.onSignalCallbacks = new Map();
+            this.callbacks = new Map();
+            this.afterCallbacks = new Map();
+            this.exitTimeout = defaultTimeout;
+            this.inited = false;
+            instance = this;
+        }
+
+        return instance;
+    }
+
+    init(signals = exitSignals, codeOnSignal = 2) {
+        if (this.inited)
+            throw new Error('AsyncExit: initialized already');
+
+        const runSingalCallbacks = async(signal) => {
+            for (const signalCallback of this.onSignalCallbacks.keys()) {
+                try {
+                    await signalCallback(signal);
+                } catch(e) {
+                    console.error(e);
+                }
+            }
+        };
+
+        for (const signal of signals) {
+            process.once(signal, async() => {
+                await runSingalCallbacks(signal);
+                this.exit(codeOnSignal);
+            });
+        }
+
+        this.inited = true;
+    }
+
+    onSignal(signalCallback) {
+        if (!this.onSignalCallbacks.has(signalCallback)) {
+            this.onSignalCallbacks.set(signalCallback, true);
+        }
+    }
+
+    add(exitCallback) {
+        if (!this.callbacks.has(exitCallback)) {
+            this.callbacks.set(exitCallback, true);
+        }
+    }
+
+    addAfter(exitCallback) {
+        if (!this.afterCallbacks.has(exitCallback)) {
+            this.afterCallbacks.set(exitCallback, true);
+        }
+    }
+
+    remove(exitCallback) {
+        if (this.callbacks.has(exitCallback)) {
+            this.callbacks.delete(exitCallback);
+        }
+        if (this.afterCallbacks.has(exitCallback)) {
+            this.afterCallbacks.delete(exitCallback);
+        }
+    }
+
+    setExitTimeout(timeout) {
+        this.exitTimeout = timeout;
+    }
+
+    exit(code = 0) {
+        if (this.exiting)
+            return;
+
+        this.exiting = true;
+
+        const timer = setTimeout(() => { process.exit(code); }, this.exitTimeout);
+
+        (async() => {
+            for (const exitCallback of this.callbacks.keys()) {
+                try {
+                    await exitCallback();
+                } catch(e) {
+                    console.error(e);
+                }
+            }
+
+            for (const exitCallback of this.afterCallbacks.keys()) {
+                try {
+                    await exitCallback();
+                } catch(e) {
+                    console.error(e);
+                }
+            }
+
+            clearTimeout(timer);
+            //console.log('Exited gracefully');
+            process.exit(code);
+        })();
+    }
+}
+
+module.exports = AsyncExit;

server/core/LibSharedStorage/MegaStorage.js

@@ -25,7 +25,6 @@ class MegaStorage {
             this.debouncedSaveStats = _.debounce(() => {
                 this.saveStats().catch((e) => {
                     log(LM_ERR, `MegaStorage::saveStats ${e.message}`);
-                    //process.exit(1);
                 });
             }, 5000, {'maxWait':6000});
 

server/core/Logger.js

@@ -2,6 +2,9 @@
   Журналирование с буферизацией вывода
 */
 const fs = require('fs-extra');
+const ayncExit = new (require('./AsyncExit'))();
+
+const sleep = (ms) => { return new Promise(resolve => setTimeout(resolve, ms)) };
 
 global.LM_OK = 0;
 global.LM_INFO = 1;
@@ -46,12 +49,13 @@ class BaseLog {
         this.outputBuffer = [];
 
         await this.flushImpl(this.data)
-            .catch(e => { console.log(e); process.exit(1); } );
+            .catch(e => { console.log(e); ayncExit.exit(1); } );
         this.flushing = false;
     }
 
     log(msgType, message) {
-        if (this.closed) { console.log(`Logger fatal error: log was closed (message to log: ${message}})`); process.exit(1); }
+        if (this.closed)
+            return;
 
         if (!this.exclude.has(msgType)) {
             this.outputBuffer.push(message);
@@ -73,7 +77,7 @@ class BaseLog {
         }
     }
 
-    close() {
+    async close() {
         if (this.closed)
             return;
 
@@ -81,12 +85,13 @@ class BaseLog {
             clearInterval(this.iid);
 
         try {
-            if (this.flushing)
+            while (this.outputBufferLength) {
-                this.flushImplSync(this.data);
+                await this.flush();
-            this.flushImplSync(this.outputBuffer);
+                await sleep(1);
+            }
         } catch(e) {
             console.log(e);
-            process.exit(1);
+            ayncExit.exit(1);
         }
         this.outputBufferLength = 0;
         this.outputBuffer = [];
@@ -103,12 +108,14 @@ class FileLog extends BaseLog {
         this.rcid = 0;
     }
 
-    close() {
+    async close() {
         if (this.closed)
             return;
-        super.close();
+        await super.close();
-        if (this.fd)
+        if (this.fd) {
-            fs.closeSync(this.fd);
+            await fs.close(this.fd);
+            this.fd = null;
+        }
         if (this.rcid)
             clearTimeout(this.rcid);
     }
@@ -151,23 +158,15 @@ class FileLog extends BaseLog {
             }, LOG_ROTATE_FILE_CHECK_INTERVAL);
         }
 
+        if (this.fd)
             await fs.write(this.fd, Buffer.from(data.join('')));
     }
-
-    flushImplSync(data) {
-        fs.writeSync(this.fd, Buffer.from(data.join('')));
-    }
-
 }
 
 class ConsoleLog extends BaseLog {
     async flushImpl(data) {
         process.stdout.write(data.join(''));
     }
-
-    flushImplSync(data) {
-        process.stdout.write(data.join(''));
-    }
 }
 
 //------------------------------------------------------------------
@@ -178,7 +177,7 @@ const factory = {
 
 class Logger {
 
-    constructor(params = null, cleanupCallback = null) {        
+    constructor(params = null) {
         this.handlers = [];
         if (params) {
             params.forEach((logParams) => {
@@ -187,12 +186,22 @@ class Logger {
                 this.handlers.push(new loggerClass(logParams));
             });
         }
-        cleanupCallback = cleanupCallback || (() => {});
+
-        this.cleanup(cleanupCallback);
+        this.closed = false;
+        ayncExit.onSignal((signal) => {
+            this.log(LM_FATAL, `Signal ${signal} received, exiting...`);
+        });
+        ayncExit.addAfter(this.close.bind(this));
+    }
+
+    formatDate(date) {
+        return `${date.getFullYear()}-${(date.getMonth() + 1).toString().padStart(2, '0')}-${date.getDate().toString().padStart(2, '0')} ` +
+            `${date.getHours().toString().padStart(2, '0')}:${date.getMinutes().toString().padStart(2, '0')}:${date.getSeconds().toString().padStart(2, '0')}.` +
+            `${date.getMilliseconds().toString().padStart(3, '0')}`;
     }
 
     prepareMessage(msgType, message) {
-        return (new Date().toISOString()) + ` ${msgTypeToStr[msgType]}: ${message}\n`;
+        return this.formatDate(new Date()) + ` ${msgTypeToStr[msgType]}: ${message}\n`;
     }
 
     log(msgType, message) {
@@ -203,47 +212,18 @@ class Logger {
 
         const mes = this.prepareMessage(msgType, message);
 
+        if (!this.closed) {
             for (let i = 0; i < this.handlers.length; i++)
                 this.handlers[i].log(msgType, mes);
+        } else {
+            console.log(mes);
+        }
     }
 
-    close() {
+    async close() {
         for (let i = 0; i < this.handlers.length; i++)
-            this.handlers[i].close();
+            await this.handlers[i].close();
-    }
+        this.closed = true;
-
-    cleanup(callback) {
-        // attach user callback to the process event emitter
-        // if no callback, it will still exit gracefully on Ctrl-C
-        callback = callback || (() => {});
-        process.on('cleanup', callback);
-
-        // do app specific cleaning before exiting
-        process.on('exit', () => {
-            this.close();
-            process.emit('cleanup');
-        });
-
-        // catch ctrl+c event and exit normally
-        process.on('SIGINT', () => {
-            this.log(LM_FATAL, 'Ctrl-C pressed, exiting...');
-            process.exit(2);
-        });
-
-        process.on('SIGTERM', () => {
-            this.log(LM_FATAL, 'Kill signal, exiting...');
-            process.exit(2);
-        });
-
-        //catch uncaught exceptions, trace, then exit normally
-        process.on('uncaughtException', e => {
-            try {
-                this.log(LM_FATAL, e.stack);
-            } catch (e) {
-                console.log(e.stack);
-            }
-            process.exit(99);
-        });
     }
 }
 

server/core/Reader/JembaReaderStorage.js

@@ -0,0 +1,122 @@
+const _ = require('lodash');
+
+const utils = require('../utils');
+const JembaConnManager = require('../../db/JembaConnManager');//singleton
+
+let instance = null;
+
+//singleton
+class JembaReaderStorage {
+    constructor() {
+        if (!instance) {
+            this.connManager = new JembaConnManager();
+            this.db = this.connManager.db['reader-storage'];
+            this.periodicCleanCache(3*3600*1000);//1 раз в 3 часа
+
+            instance = this;
+        }
+
+        return instance;
+    }
+
+    async doAction(act) {
+        if (!_.isObject(act.items))
+            throw new Error('items is not an object');
+
+        let result = {};
+        switch (act.action) {
+            case 'check':
+                result = await this.checkItems(act.items);
+                break;
+            case 'get':
+                result = await this.getItems(act.items);
+                break;
+            case 'set':
+                result = await this.setItems(act.items, act.force);
+                break;
+            default:
+                throw new Error('Unknown action');
+        }
+
+        return result;
+    }
+
+    async checkItems(items) {
+        let result = {state: 'success', items: {}};
+
+        const db = this.db;
+
+        for (const id of Object.keys(items)) {
+            if (this.cache[id]) {
+                result.items[id] = this.cache[id];
+            } else {
+                const rows = await db.select({//SQL`SELECT rev FROM storage WHERE id = ${id}`
+                    table: 'storage',
+                    map: '(r) => ({rev: r.rev})',
+                    where: `@@id(${db.esc(id)})`
+                });
+                const rev = (rows.length && rows[0].rev ? rows[0].rev : 0);
+                result.items[id] = {rev};
+                this.cache[id] = result.items[id];
+            }
+        }
+
+        return result;
+    }
+
+    async getItems(items) {
+        let result = {state: 'success', items: {}};
+
+        const db = this.db;
+
+        for (const id of Object.keys(items)) {
+            const rows = await db.select({//SQL`SELECT rev, data FROM storage WHERE id = ${id}`);
+                table: 'storage',
+                where: `@@id(${db.esc(id)})`
+            });
+            const rev = (rows.length && rows[0].rev ? rows[0].rev : 0);
+            const data = (rows.length && rows[0].data ? rows[0].data : '');
+            result.items[id] = {rev, data};
+        }
+
+        return result;
+    }
+
+    async setItems(items, force) {
+        let check = await this.checkItems(items);
+
+        //сначала проверим совпадение ревизий
+        for (const id of Object.keys(items)) {
+            if (!_.isString(items[id].data))
+                throw new Error('items.data is not a string');
+
+            if (!force && check.items[id].rev + 1 !== items[id].rev)
+                return {state: 'reject', items: check.items};
+        }
+
+        const db = this.db;
+        const newRev = {};
+        for (const id of Object.keys(items)) {
+            await db.insert({//SQL`INSERT OR REPLACE INTO storage (id, rev, time, data) VALUES (${id}, ${items[id].rev}, strftime('%s','now'), ${items[id].data})`);
+                table: 'storage',
+                replace: true,
+                rows: [{id, rev: items[id].rev, time: utils.toUnixTime(Date.now()), data: items[id].data}],
+            });
+            newRev[id] = {rev: items[id].rev};
+        }
+        
+        Object.assign(this.cache, newRev);
+
+        return {state: 'success'};
+    }
+
+    periodicCleanCache(timeout) {
+        this.cache = {};
+
+        setTimeout(() => {
+            this.periodicCleanCache(timeout);
+        }, timeout);
+    }
+}
+
+module.exports = JembaReaderStorage;

server/core/Reader/ReaderStorage.js

@@ -1,126 +0,0 @@
-const SQL = require('sql-template-strings');
-const _ = require('lodash');
-
-const ConnManager = require('../../db/ConnManager');//singleton
-
-let instance = null;
-
-//singleton
-class ReaderStorage {
-    constructor() {
-        if (!instance) {
-            this.connManager = new ConnManager();
-            this.storagePool = this.connManager.pool.readerStorage;
-            this.periodicCleanCache(3*3600*1000);//1 раз в 3 часа
-
-            instance = this;
-        }
-
-        return instance;
-    }
-
-    async doAction(act) {
-        if (!_.isObject(act.items))
-            throw new Error('items is not an object');
-
-        let result = {};
-        switch (act.action) {
-            case 'check':
-                result = await this.checkItems(act.items);
-                break;
-            case 'get':
-                result = await this.getItems(act.items);
-                break;
-            case 'set':
-                result = await this.setItems(act.items, act.force);
-                break;
-            default:
-                throw new Error('Unknown action');
-        }
-
-        return result;
-    }
-
-    async checkItems(items) {
-        let result = {state: 'success', items: {}};
-
-        const dbh = await this.storagePool.get();
-        try {
-            for (const id of Object.keys(items)) {
-                if (this.cache[id]) {
-                    result.items[id] = this.cache[id];
-                } else {
-                    const rows = await dbh.all(SQL`SELECT rev FROM storage WHERE id = ${id}`);
-                    const rev = (rows.length && rows[0].rev ? rows[0].rev : 0);
-                    result.items[id] = {rev};
-                    this.cache[id] = result.items[id];
-                }
-            }
-        } finally {
-            dbh.ret();
-        }
-
-        return result;
-    }
-
-    async getItems(items) {
-        let result = {state: 'success', items: {}};
-
-        const dbh = await this.storagePool.get();
-        try {
-            for (const id of Object.keys(items)) {
-                const rows = await dbh.all(SQL`SELECT rev, data FROM storage WHERE id = ${id}`);                
-                const rev = (rows.length && rows[0].rev ? rows[0].rev : 0);
-                const data = (rows.length && rows[0].data ? rows[0].data : '');
-                result.items[id] = {rev, data};
-            }
-        } finally {
-            dbh.ret();
-        }
-
-        return result;
-    }
-
-    async setItems(items, force) {
-        let check = await this.checkItems(items);
-
-        //сначала проверим совпадение ревизий
-        for (const id of Object.keys(items)) {
-            if (!_.isString(items[id].data))
-                throw new Error('items.data is not a string');
-
-            if (!force && check.items[id].rev + 1 !== items[id].rev)
-                return {state: 'reject', items: check.items};
-        }
-
-        const dbh = await this.storagePool.get();
-        await dbh.run('BEGIN');
-        try {
-            const newRev = {};
-            for (const id of Object.keys(items)) {
-                await dbh.run(SQL`INSERT OR REPLACE INTO storage (id, rev, time, data) VALUES (${id}, ${items[id].rev}, strftime('%s','now'), ${items[id].data})`);
-                newRev[id] = {rev: items[id].rev};
-            }
-            await dbh.run('COMMIT');
-            
-            Object.assign(this.cache, newRev);
-        } catch (e) {
-            await dbh.run('ROLLBACK');
-            throw e;
-        } finally {
-            dbh.ret();
-        }
-
-        return {state: 'success'};
-    }
-
-    periodicCleanCache(timeout) {
-        this.cache = {};
-
-        setTimeout(() => {
-            this.periodicCleanCache(timeout);
-        }, timeout);
-    }
-}
-
-module.exports = ReaderStorage;

server/core/utils.js

@@ -38,6 +38,10 @@ function sleep(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+function toUnixTime(time) {
+    return parseInt(time/1000);
+}
+
 function randomHexString(len) {
     return crypto.randomBytes(len).toString('hex')
 }
@@ -126,6 +130,7 @@ module.exports = {
     bufferRemoveZeroes,
     getFileHash,
     sleep,
+    toUnixTime,
     randomHexString,
     touchFile,
     spawnProcess,

server/db/ConnManager.js

@@ -1,3 +1,4 @@
+//TODO: удалить модуль в 2023г
 const fs = require('fs-extra');
 
 const SqliteConnectionPool = require('./SqliteConnectionPool');

server/db/Converter.js

@@ -0,0 +1,42 @@
+//TODO: удалить модуль в 2023г
+const fs = require('fs-extra');
+const log = new (require('../core/AppLogger'))().log;//singleton
+
+class Converter {    
+    async run(config) {        
+        log('Converter start');
+
+        try {
+            const connManager = new (require('./ConnManager'))();//singleton
+            const storagePool = connManager.pool.readerStorage;
+
+            const jembaConnManager = new (require('./JembaConnManager'))();//singleton
+            const db = jembaConnManager.db['reader-storage'];
+
+            const srcDbPath = `${config.dataDir}/reader-storage.sqlite`;
+            if (!await fs.pathExists(srcDbPath)) {
+                log(LM_WARN, '  Source DB does not exist, nothing to do');
+                return;
+            }
+
+            const rows = await db.select({table: 'storage', count: true});
+            if (rows.length && rows[0].count != 0) {
+                log(LM_WARN, `  Destination table already exists (found ${rows[0].count} items), nothing to do`);
+                return;
+            }
+
+            const dbSrc = await storagePool.get();
+            try {
+                const rows = await dbSrc.all(`SELECT * FROM storage`);
+                await db.insert({table: 'storage', rows});
+                log(`  Inserted ${rows.length} items`);
+            } finally {
+                dbSrc.ret();
+            }
+        } finally {
+            log('Converter finish');
+        }
+    }
+}
+
+module.exports = Converter;

server/db/JembaConnManager.js

@@ -0,0 +1,187 @@
+const fs = require('fs-extra');
+const _ = require('lodash');
+
+const ayncExit = new (require('../core/AsyncExit'))();//singleton
+const { JembaDb, JembaDbThread } = require('jembadb');
+const log = new (require('../core/AppLogger'))().log;//singleton
+
+const jembaMigrations = require('./jembaMigrations');
+
+let instance = null;
+
+//singleton
+class JembaConnManager {
+    constructor() {
+        if (!instance) {
+            this.inited = false;
+
+            instance = this;
+        }
+
+        return instance;
+    }
+
+    async init(config, forceAutoRepair = false, migs = jembaMigrations, undoLastMigration = false) {
+        if (this.inited)
+            throw new Error('JembaConnManager initialized already');
+
+        this.config = config;
+        this._db = {};
+
+        for (const dbConfig of this.config.jembaDb) {
+            const dbPath = `${this.config.dataDir}/db/${dbConfig.dbName}`;
+
+            //бэкап
+            if (!dbConfig.noBak && await fs.pathExists(dbPath)) {
+                const bakFile = `${dbPath}.bak`;
+                await fs.remove(bakFile);
+                await fs.copy(dbPath, bakFile);
+            }
+
+            let dbConn = null;
+            if (dbConfig.thread) {
+                dbConn = new JembaDbThread();
+            } else {
+                dbConn = new JembaDb();
+            }
+
+            log(`Open "${dbConfig.dbName}" begin`);
+            await dbConn.openDb({
+                dbPath,
+                create: true,
+                cacheSize: dbConfig.cacheSize,
+                compressed: dbConfig.compressed,
+                forceFileClosing: dbConfig.forceFileClosing
+            });
+
+            if (dbConfig.openAll) {
+                try {
+                    await dbConn.openAll();
+                } catch(e) {
+                    if ((forceAutoRepair || dbConfig.autoRepair) && 
+                        (
+                            e.message.indexOf('corrupted') >= 0 
+                            || e.message.indexOf('Unexpected token') >= 0
+                            || e.message.indexOf('invalid stored block lengths') >= 0
+                        )
+                        ) {
+                        log(LM_ERR, e);
+                        log(`Open "${dbConfig.dbName}" with auto repair`);
+                        await dbConn.openAll({autoRepair: true});
+                    } else {
+                        throw e;
+                    }
+                }
+            }
+
+            log(`Open "${dbConfig.dbName}" end`);
+
+            //миграции
+            const mig = migs[dbConfig.dbName];
+            if (mig && mig.data) {
+                const applied = await this.migrate(dbConn, mig.data, mig.table, undoLastMigration);
+                if (applied.length)
+                    log(`${applied.length} migrations applied to "${dbConfig.dbName}"`);
+            }
+
+            this._db[dbConfig.dbName] = dbConn;
+        }
+
+        ayncExit.add(this.close.bind(this));
+
+        this.inited = true;
+    }
+
+    async close() {
+        if (!this.inited)
+            return;
+
+        for (const dbConfig of this.config.jembaDb) {
+            await this._db[dbConfig.dbName].closeDb();
+        }
+
+        this._db = {};
+        this.inited = false;
+    }
+
+    async migrate(db, migs, table, undoLastMigration) {
+        const migrations = _.cloneDeep(migs).sort((a, b) => a.id - b.id);
+
+        if (!migrations.length) {
+            throw new Error('No migration data');
+        }
+
+        migrations.map(migration => {
+            const data = migration.data;
+            if (!data.up || !data.down) {
+                throw new Error(`The ${migration.id}:${migration.name} does not contain 'up' or 'down' instructions`);
+            } else {
+                migration.up = data.up;
+                migration.down = data.down;
+            }
+            delete migration.data;
+        });
+
+        // Create a database table for migrations meta data if it doesn't exist
+        // id, name, up, down
+        await db.create({
+            table, 
+            quietIfExists: true,
+        });
+
+        // Get the list of already applied migrations
+        let dbMigrations = await db.select({
+            table,
+            sort: '(a, b) => a.id - b.id'
+        });
+
+        const execUpDown = async(items) => {
+            for (const item of items) {
+                const action = item[0];
+                await db[action](item[1]);
+            }
+        };
+
+        // Undo migrations that exist only in the database but not in migs,
+        // also undo the last migration if the undoLastMigration
+        const lastMigration = migrations[migrations.length - 1];
+        for (const migration of dbMigrations.slice().sort((a, b) => b.id - a.id)) {
+            if (!migrations.some(x => x.id === migration.id) ||
+                (undoLastMigration && migration.id === lastMigration.id)) {
+                    await execUpDown(migration.down);
+                    await db.delete({
+                        table, 
+                        where: `@@id(${db.esc(migration.id)})`
+                    });
+                    dbMigrations = dbMigrations.filter(x => x.id !== migration.id);
+            } else {
+                break;
+            }
+        }
+
+        // Apply pending migrations
+        let applied = [];
+        const lastMigrationId = dbMigrations.length ? dbMigrations[dbMigrations.length - 1].id : 0;
+        for (const migration of migrations) {
+            if (migration.id > lastMigrationId) {
+                await execUpDown(migration.up);
+                await db.insert({
+                    table,
+                    rows: [migration],
+                });
+                applied.push(migration.id);
+            }
+        }
+
+        return applied;
+    }
+
+    get db() {
+        if (!this.inited)
+            throw new Error('JembaConnManager not inited');
+
+        return this._db;
+    }
+}
+
+module.exports = JembaConnManager;

server/db/SqliteConnectionPool.js

@@ -1,3 +1,4 @@
+//TODO: удалить модуль в 2023г
 const sqlite3 = require('sqlite3');
 const sqlite = require('sqlite');
 

server/db/jembaMigrations/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+    //'app': require('./jembaMigrations/app'),
+    'reader-storage': require('./reader-storage'),
+};

server/db/jembaMigrations/reader-storage/001-create.js

@@ -0,0 +1,13 @@
+module.exports = {
+    up: [
+        //CREATE TABLE storage (id TEXT PRIMARY KEY, rev INTEGER, time INTEGER, data TEXT);
+        ['create', {
+            table: 'storage'
+        }],
+    ],    
+    down: [
+        ['drop', {
+            table: 'storage'
+        }],
+    ]
+};

server/db/jembaMigrations/reader-storage/index.js

@@ -0,0 +1,6 @@
+module.exports = {
+    table: 'migration1',
+    data: [
+        {id: 1, name: 'create', data: require('./001-create')}
+    ]
+}

server/index.js

@@ -7,6 +7,11 @@ const compression = require('compression');
 const http = require('http');
 const WebSocket = require ('ws');
 
+const ayncExit = new (require('./core/AsyncExit'))();
+ayncExit.init();
+
+let log = null;
+
 async function init() {
     //config
     const configManager = new (require('./config'))();//singleton
@@ -18,7 +23,7 @@ async function init() {
     //logger
     const appLogger = new (require('./core/AppLogger'))();//singleton
     await appLogger.init(config);
-    const log = appLogger.log;
+    log = appLogger.log;
 
     //dirs
     log(`${config.name} v${config.version}, Node.js ${process.version}`);
@@ -41,6 +46,13 @@ async function init() {
     //connections
     const connManager = new (require('./db/ConnManager'))();//singleton
     await connManager.init(config);
+
+    const jembaConnManager = new (require('./db/JembaConnManager'))();//singleton
+    await jembaConnManager.init(config, argv['auto-repair']);
+
+    //converter SQLITE => JembaDb
+    const converter = new  (require('./db/Converter'))();
+    await converter.run(config);
 }
 
 async function main() {
@@ -96,13 +108,15 @@ async function main() {
     }
 }
 
-
 (async() => {
     try {
         await init();
         await main();
     } catch (e) {
-        console.error(e);
+        if (log)
-        process.exit(1);
+            log(LM_FATAL, e.stack);
+        else
+            console.error(e.stack);
+        ayncExit.exit(1);
     }
 })();