Merge branch 'release/0.11.8-7'

0.11.8-6

server/config/base.js

@@ -39,6 +39,11 @@ module.exports = {
     ],
 
     jembaDb: [
+        {
+            dbName: 'app',
+            thread: true,
+            openAll: true,
+        },
         {
             dbName: 'reader-storage',
             thread: true,

server/core/Reader/ReaderWorker.js

@@ -7,11 +7,15 @@ const FileDownloader = require('../FileDownloader');
 const FileDecompressor = require('../FileDecompressor');
 const BookConverter = require('./BookConverter');
 const RemoteStorage = require('../RemoteStorage');
+const JembaConnManager = require('../../db/JembaConnManager');//singleton
+const ayncExit = new (require('../AsyncExit'))();
 
 const utils = require('../utils');
 const log = new (require('../AppLogger'))().log;//singleton
 
-const cleanDirPeriod = 30*60*1000;//раз в полчаса
+const cleanDirPeriod = 60*60*1000;//каждый час
+const remoteSendPeriod = 119*1000;//примерно раз 2 минуты
+
 const queue = new LimitedQueue(5, 100, 2*60*1000 + 15000);//2 минуты ожидание подвижек
 
 let instance = null;
@@ -33,6 +37,9 @@ class ReaderWorker {
             this.decomp = new FileDecompressor(3*config.maxUploadFileSize);
             this.bookConverter = new BookConverter(this.config);
 
+            this.connManager = new JembaConnManager();
+            this.appDb = this.connManager.db['app'];
+
             this.remoteStorage = false;
             if (config.remoteStorage) {
                 this.remoteStorage = new RemoteStorage(
@@ -40,20 +47,27 @@ class ReaderWorker {
                 );
             }
 
-            this.remoteConfig = {
+            this.dirConfigArr = [
-                '/tmp': {
+                {
                     dir: this.config.tempPublicDir,
+                    remoteDir: '/tmp',
                     maxSize: this.config.maxTempPublicDirSize,
                     moveToRemote: true,
                 },
-                '/upload': {
+                {
                     dir: this.config.uploadDir,
+                    remoteDir: '/upload',
                     maxSize: this.config.maxUploadPublicDirSize,
                     moveToRemote: true,
                 }
-            };
+            ];
+            //преобразуем в объект для большего удобства
+            this.dirConfig = {};
+            for (const configRec of this.dirConfigArr)
+                this.dirConfig[configRec.remoteDir] = configRec;
 
-            this.periodicCleanDir(this.remoteConfig);//no await
+            this.remoteFilesToSend = [];
+            this.periodicCleanDir();//no await
             
             instance = this;
         }
@@ -154,6 +168,13 @@ class ReaderWorker {
             const finishFilename = path.basename(compFilename);
             wState.finish({path: `/tmp/${finishFilename}`, size: stat.size});
 
+            //асинхронно через 30 сек добавим в очередь на отправку
+            //т.к. gzipFileIfNotExists может переупаковать файл
+            (async() => {
+                await utils.sleep(30*1000);
+                this.pushRemoteSend(compFilename, '/tmp');
+            })();
+
         } catch (e) {
             log(LM_ERR, e.stack);
             let mes = e.message.split('|FORLOG|');
@@ -194,6 +215,7 @@ class ReaderWorker {
 
         if (!await fs.pathExists(outFilename)) {
             await fs.move(file.path, outFilename);
+            this.pushRemoteSend(outFilename, '/upload');
         } else {
             await utils.touchFile(outFilename);
             await fs.remove(file.path);
@@ -208,6 +230,7 @@ class ReaderWorker {
 
         if (!await fs.pathExists(outFilename)) {
             await fs.writeFile(outFilename, buf);
+            this.pushRemoteSend(outFilename, '/upload');
         } else {
             await utils.touchFile(outFilename);
         }
@@ -225,8 +248,8 @@ class ReaderWorker {
 
     async restoreRemoteFile(filename, remoteDir) {
         let targetDir = '';
-        if (this.remoteConfig[remoteDir])
+        if (this.dirConfig[remoteDir])
-            targetDir = this.remoteConfig[remoteDir].dir;
+            targetDir = this.dirConfig[remoteDir].dir;
         else
             throw new Error(`restoreRemoteFile: unknown remoteDir value (${remoteDir})`);
 
@@ -247,13 +270,57 @@ class ReaderWorker {
         return targetName;
     }
 
-    async cleanDir(dir, remoteDir, maxSize, moveToRemote) {
+    pushRemoteSend(fileName, remoteDir) {
-        if (!this.remoteSent)
+        if (this.remoteStorage
-            this.remoteSent = {};
+            && this.dirConfig[remoteDir] 
-        if (!this.remoteSent[remoteDir])
+            && this.dirConfig[remoteDir].moveToRemote) {
-            this.remoteSent[remoteDir] = {};
+            this.remoteFilesToSend.push({fileName, remoteDir});
+        }
+    }
 
-        const sent = this.remoteSent[remoteDir];
+    async remoteSendFile(sendFileRec) {
+        const {fileName, remoteDir} = sendFileRec;
+        const sent = this.remoteSent;
+
+        if (!fileName || sent[fileName])
+            return;
+
+        log(`remoteSendFile ${remoteDir}/${path.basename(fileName)}`);
+
+        //отправляем в remoteStorage
+        await this.remoteStorage.putFile(fileName, remoteDir);
+        
+        sent[fileName] = true;
+        await this.appDb.insert({table: 'remote_sent', ignore: true, rows: [{id: fileName, remoteDir}]});
+    }
+
+    async remoteSendAll() {
+        if (!this.remoteStorage)
+            return;
+
+        const newSendQueue = [];
+        while (this.remoteFilesToSend.length) {
+            const sendFileRec = this.remoteFilesToSend.shift();
+
+            if (sendFileRec.remoteDir
+                && this.dirConfig[sendFileRec.remoteDir]
+                && this.dirConfig[sendFileRec.remoteDir].moveToRemote) {
+
+                try {
+                    await this.remoteSendFile(sendFileRec);
+                } catch (e) {
+                    newSendQueue.push(sendFileRec)
+                    log(LM_ERR, e.stack);
+                }
+            }
+        }
+
+        this.remoteFilesToSend = newSendQueue;
+    }
+
+    async cleanDir(config) {
+        const {dir, remoteDir, maxSize, moveToRemote} = config;
+        const sent = this.remoteSent;
 
         const list = await fs.readdir(dir);
 
@@ -267,24 +334,35 @@ class ReaderWorker {
                 files.push({name: filePath, stat});
             }
         }
-        log(`clean dir ${dir}, maxSize=${maxSize}, found ${files.length} files, total size=${size}`);
+
+        log(LM_WARN, `clean dir ${dir}, maxSize=${maxSize}, found ${files.length} files, total size=${size}`);
 
         files.sort((a, b) => a.stat.mtimeMs - b.stat.mtimeMs);
 
+        //удаленное хранилище
         if (moveToRemote && this.remoteStorage) {
+            const foundFiles = new Set();
             for (const file of files) {
-                if (sent[file.name])
+                foundFiles.add(file.name);
-                    continue;
 
-                //отправляем в remoteStorage
+                //отсылаем на всякий случай перед удалением, если вдруг remoteSendAll не справился
                 try {
-                    log(`remoteStorage.putFile ${remoteDir}/${path.basename(file.name)}`);
+                    await this.remoteSendFile({fileName: file.name, remoteDir});
-                    await this.remoteStorage.putFile(file.name, remoteDir);
-                    sent[file.name] = true;
                 } catch (e) {
                     log(LM_ERR, e.stack);
                 }
             }
+
+            //почистим remoteSent и БД
+            //несколько неоптимально, таскает все записи из таблицы
+            const rows = await this.appDb.select({table: 'remote_sent'});
+            for (const row of rows) {
+                if ((row.remoteDir === remoteDir && !foundFiles.has(row.id))
+                    || !this.dirConfig[row.remoteDir]) {
+                    delete sent[row.id];
+                    await this.appDb.delete({table: 'remote_sent', where: `@@id(${this.appDb.esc(row.id)})`});
+                }
+            }
         }
 
         let i = 0;
@@ -298,27 +376,61 @@ class ReaderWorker {
                 || (moveToRemote && this.remoteStorage && sent[oldFile])
                 || size > maxSize*1.5) {
                 await fs.remove(oldFile);
-                delete sent[oldFile];
                 j++;
             }
             
             size -= file.stat.size;
             i++;
         }
-        log(`removed ${j} files`);
+
+        log(LM_WARN, `removed ${j} files`);
     }
 
-    async periodicCleanDir(cleanConfig) {
+    async periodicCleanDir() {
-        while (1) {// eslint-disable-line no-constant-condition
+        try {
-            for (const [remoteDir, config] of Object.entries(cleanConfig)) {
+            if (!this.remoteSent)
-                try {
+                this.remoteSent = {};
-                    await this.cleanDir(config.dir, remoteDir, config.maxSize, config.moveToRemote);
+
-                } catch(e) {
+            //инициализация this.remoteSent
-                    log(LM_ERR, e.stack);
+            if (this.remoteStorage) {
+                const rows = await this.appDb.select({table: 'remote_sent'});
+                for (const row of rows) {
+                    this.remoteSent[row.id] = true;
                 }
             }
 
-            await utils.sleep(cleanDirPeriod);
+            let lastCleanDirTime = 0;
+            let lastRemoteSendTime = 0;
+            while (1) {// eslint-disable-line no-constant-condition
+                //отсылка в удаленное хранилище
+                if (Date.now() - lastRemoteSendTime >= remoteSendPeriod) {
+                    try {
+                        await this.remoteSendAll();
+                    } catch(e) {
+                        log(LM_ERR, e.stack);
+                    }
+
+                    lastRemoteSendTime = Date.now();
+                }
+
+                //чистка папок
+                if (Date.now() - lastCleanDirTime >= cleanDirPeriod) {
+                    for (const config of Object.values(this.dirConfig)) {
+                        try {
+                            await this.cleanDir(config);
+                        } catch(e) {
+                            log(LM_ERR, e.stack);
+                        }
+                    }
+
+                    lastCleanDirTime = Date.now();
+                }
+
+                await utils.sleep(60*1000);//интервал проверки 1 минута
+            }
+        } catch (e) {
+            log(LM_FATAL, e.message);
+            ayncExit.exit(1);
         }
     }
 

server/core/RemoteStorage.js

@@ -10,13 +10,13 @@ class RemoteStorage {
 
         this.accessToken = this.config.accessToken;
 
-        this.wsc = new WebSocketConnection(config.url);
+        this.wsc = new WebSocketConnection(config.url, 10, 30, {rejectUnauthorized: false});
     }
 
-    async wsQuery(query) {
+    async wsRequest(query) {
         const response = await this.wsc.message(
-            await this.wsc.send(Object.assign({accessToken: this.accessToken}, query), 30),
+            await this.wsc.send(Object.assign({accessToken: this.accessToken}, query), 600),
-            300
+            600
         );
         if (response.error)
             throw new Error(response.error);
@@ -24,19 +24,19 @@ class RemoteStorage {
     }
 
     async wsStat(fileName) {
-        return await this.wsQuery({action: 'get-stat', fileName});
+        return await this.wsRequest({action: 'get-stat', fileName});
     }
 
     async wsGetFile(fileName) {
-        return this.wsQuery({action: 'get-file', fileName});
+        return this.wsRequest({action: 'get-file', fileName});
     }
 
     async wsPutFile(fileName, data) {//data base64 encoded string
-        return this.wsQuery({action: 'put-file', fileName, data});
+        return this.wsRequest({action: 'put-file', fileName, data});
     }
 
     async wsDelFile(fileName) {
-        return this.wsQuery({action: 'del-file', fileName});
+        return this.wsRequest({action: 'del-file', fileName});
     }
 
     makeRemoteFileName(fileName, dir = '') {

server/core/WebSocketConnection.js

@@ -8,10 +8,13 @@ const cleanPeriod = 5*1000;//5 секунд
 
 class WebSocketConnection {
     //messageLifeTime в секундах (проверка каждый cleanPeriod интервал)
-    constructor(url, openTimeoutSecs = 10, messageLifeTimeSecs = 30) {
+    constructor(url, openTimeoutSecs = 10, messageLifeTimeSecs = 30, webSocketOptions = {}) {
         this.WebSocket = (isBrowser ? WebSocket : require('ws'));
         this.url = url;
+        this.webSocketOptions = webSocketOptions;
+
         this.ws = null;
+
         this.listeners = [];
         this.messageQueue = [];
         this.messageLifeTime = messageLifeTimeSecs*1000;
@@ -91,7 +94,7 @@ class WebSocketConnection {
                     const url = this.url || `${protocol}//${window.location.host}/ws`;
                     this.ws = new this.WebSocket(url);
                 } else {
-                    this.ws = new this.WebSocket(this.url);
+                    this.ws = new this.WebSocket(this.url, this.webSocketOptions);
                 }
 
                 const onopen = () => {

server/db/jembaMigrations/app/001-create.js

@@ -0,0 +1,12 @@
+module.exports = {
+    up: [
+        ['create', {
+            table: 'remote_sent'
+        }],
+    ],    
+    down: [
+        ['drop', {
+            table: 'remote_sent'
+        }],
+    ]
+};

server/db/jembaMigrations/app/index.js

@@ -0,0 +1,6 @@
+module.exports = {
+    table: 'migration1',
+    data: [
+        {id: 1, name: 'create', data: require('./001-create')}
+    ]
+}

server/db/jembaMigrations/index.js

@@ -1,4 +1,5 @@
 module.exports = {
+    'app': require('./app'),
     'reader-storage': require('./reader-storage'),
     'book-update-server': require('./book-update-server'),
 };

server/index.js

@@ -1,5 +1,4 @@
 require('tls').DEFAULT_MIN_VERSION = 'TLSv1';
-process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0;
 
 const fs = require('fs-extra');
 const argv = require('minimist')(process.argv.slice(2));

server/routes.js

@@ -115,7 +115,7 @@ function initStatic(app, config) {
                 }
             }
         } catch(e) {
-            log(LM_ERR, `Static.restoreRemoteFile: ${e.message}`);
+            log(LM_ERR, `static::restoreRemoteFile ${req.path} > ${e.message}`);
         }
 
         return next();