Merge branch 'release/0.11.8-5'

0.11.8-4

docs/beta/beta.liberama

@@ -6,6 +6,7 @@ server {
   ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
 
   server_name beta.liberama.top;
+  set $liberama http://127.0.0.1:34082;
 
   client_max_body_size 50m;
   proxy_read_timeout 1h;
@@ -15,15 +16,20 @@ server {
   gzip_proxied expired no-cache no-store private auth;
   gzip_types *;
 
+  location @liberama {
+    proxy_pass $liberama;
+  }
+
   location /api {
-    proxy_pass http://127.0.0.1:34082;
+    proxy_pass $liberama;
   }
 
   location /ws {
-    proxy_pass http://127.0.0.1:34082;
+    proxy_pass $liberama;
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
+    proxy_read_timeout 600s;
   }
 
   location / {
@@ -32,6 +38,11 @@ server {
     location /tmp {
       types { } default_type "application/xml; charset=utf-8";
       add_header Content-Encoding gzip;
+      try_files $uri @liberama;
+    }
+
+    location /upload {
+      try_files $uri @liberama;
     }
 
     location ~* \.(?:manifest|appcache|html)$ {
@@ -50,6 +61,7 @@ server {
 server {
   listen 80;
   server_name b.beta.liberama.top;
+  set $liberama http://127.0.0.1:34082;
 
   client_max_body_size 50m;
   proxy_read_timeout 1h;
@@ -59,15 +71,20 @@ server {
   gzip_proxied expired no-cache no-store private auth;
   gzip_types *;
 
+  location @liberama {
+    proxy_pass $liberama;
+  }
+
   location /api {
-    proxy_pass http://127.0.0.1:34082;
+    proxy_pass $liberama;
   }
 
   location /ws {
-    proxy_pass http://127.0.0.1:34082;
+    proxy_pass $liberama;
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
+    proxy_read_timeout 600s;
   }
 
   location / {
@@ -76,6 +93,11 @@ server {
     location /tmp {
       types { } default_type "application/xml; charset=utf-8";
       add_header Content-Encoding gzip;
+      try_files $uri @liberama;
+    }
+
+    location /upload {
+      try_files $uri @liberama;
     }
 
     location ~* \.(?:manifest|appcache|html)$ {

docs/beta/beta.omnireader

@@ -6,6 +6,7 @@ server {
   ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
 
   server_name beta.omnireader.ru;
+  set $liberama http://127.0.0.1:34081;
 
   client_max_body_size 50m;
   proxy_read_timeout 1h;
@@ -15,15 +16,20 @@ server {
   gzip_proxied expired no-cache no-store private auth;
   gzip_types *;
 
+  location @liberama {
+    proxy_pass $liberama;
+  }
+
   location /api {
-    proxy_pass http://127.0.0.1:34081;
+    proxy_pass $liberama;
   }
 
   location /ws {
-    proxy_pass http://127.0.0.1:34081;
+    proxy_pass $liberama;
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
+    proxy_read_timeout 600s;
   }
 
   location / {
@@ -32,6 +38,11 @@ server {
     location /tmp {
       types { } default_type "application/xml; charset=utf-8";
       add_header Content-Encoding gzip;
+      try_files $uri @liberama;
+    }
+
+    location /upload {
+      try_files $uri @liberama;
     }
 
     location ~* \.(?:manifest|appcache|html)$ {

docs/beta/beta.omnireader_http

@@ -1,6 +1,7 @@
 server {
   listen 80;
   server_name beta.omnireader.ru;
+  set $liberama http://127.0.0.1:34081;
 
   client_max_body_size 50m;
   proxy_read_timeout 1h;
@@ -10,15 +11,20 @@ server {
   gzip_proxied expired no-cache no-store private auth;
   gzip_types *;
 
+  location @liberama {
+    proxy_pass $liberama;
+  }
+
   location /api {
-    proxy_pass http://127.0.0.1:34081;
+    proxy_pass $liberama;
   }
 
   location /ws {
-    proxy_pass http://127.0.0.1:34081;
+    proxy_pass $liberama;
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
+    proxy_read_timeout 600s;
   }
 
   location / {
@@ -27,6 +33,11 @@ server {
     location /tmp {
       types { } default_type "application/xml; charset=utf-8";
       add_header Content-Encoding gzip;
+      try_files $uri @liberama;
+    }
+
+    location /upload {
+      try_files $uri @liberama;
     }
 
     location ~* \.(?:manifest|appcache|html)$ {

server/config/base.js

@@ -39,6 +39,11 @@ module.exports = {
     ],
 
     jembaDb: [
+        {
+            dbName: 'app',
+            thread: true,
+            openAll: true,
+        },
         {
             dbName: 'reader-storage',
             thread: true,
@@ -55,8 +60,8 @@ module.exports = {
         },
     ],
 
-    remoteWebDavStorage: false,
     /*
+    remoteWebDavStorage: false,
     remoteWebDavStorage: {
         url: '127.0.0.1:1900',
         username: '',
@@ -64,5 +69,12 @@ module.exports = {
     },
     */
 
+    remoteStorage: false,
+    /*
+    remoteStorage: {
+        url: 'https://127.0.0.1:11900',
+        accessToken: '',
+    },
+    */
 };
 

server/core/Reader/ReaderWorker.js

@@ -6,7 +6,9 @@ const WorkerState = require('../WorkerState');//singleton
 const FileDownloader = require('../FileDownloader');
 const FileDecompressor = require('../FileDecompressor');
 const BookConverter = require('./BookConverter');
-const RemoteWebDavStorage = require('../RemoteWebDavStorage');
+const RemoteStorage = require('../RemoteStorage');
+const JembaConnManager = require('../../db/JembaConnManager');//singleton
+const ayncExit = new (require('../AsyncExit'))();
 
 const utils = require('../utils');
 const log = new (require('../AppLogger'))().log;//singleton
@@ -33,10 +35,13 @@ class ReaderWorker {
             this.decomp = new FileDecompressor(3*config.maxUploadFileSize);
             this.bookConverter = new BookConverter(this.config);
 
-            this.remoteWebDavStorage = false;
-            if (config.remoteWebDavStorage) {
-                this.remoteWebDavStorage = new RemoteWebDavStorage(
-                    Object.assign({maxContentLength: 3*config.maxUploadFileSize}, config.remoteWebDavStorage)
+            this.connManager = new JembaConnManager();
+            this.appDb = this.connManager.db['app'];
+
+            this.remoteStorage = false;
+            if (config.remoteStorage) {
+                this.remoteStorage = new RemoteStorage(
+                    Object.assign({maxContentLength: 3*config.maxUploadFileSize}, config.remoteStorage)
                 );
             }
 
@@ -235,8 +240,8 @@ class ReaderWorker {
 
         if (!await fs.pathExists(targetName)) {
             let found = false;
-            if (this.remoteWebDavStorage) {
-                found = await this.remoteWebDavStorage.getFileSuccess(targetName, remoteDir);
+            if (this.remoteStorage) {
+                found = await this.remoteStorage.getFileSuccess(targetName, remoteDir);
             }
 
             if (!found) {
@@ -248,12 +253,7 @@ class ReaderWorker {
     }
 
     async cleanDir(dir, remoteDir, maxSize, moveToRemote) {
-        if (!this.remoteSent)
-            this.remoteSent = {};
-        if (!this.remoteSent[remoteDir])
-            this.remoteSent[remoteDir] = {};
-
-        const sent = this.remoteSent[remoteDir];
+        const sent = this.remoteSent;
 
         const list = await fs.readdir(dir);
 
@@ -267,24 +267,41 @@ class ReaderWorker {
                 files.push({name: filePath, stat});
             }
         }
+
         log(`clean dir ${dir}, maxSize=${maxSize}, found ${files.length} files, total size=${size}`);
 
         files.sort((a, b) => a.stat.mtimeMs - b.stat.mtimeMs);
 
-        if (moveToRemote && this.remoteWebDavStorage) {
+        //удаленное хранилище
+        if (moveToRemote && this.remoteStorage) {
+            const foundFiles = new Set();
             for (const file of files) {
+                foundFiles.add(file.name);
+
                 if (sent[file.name])
                     continue;
 
-                //отправляем в remoteWebDavStorage
+                //отправляем в remoteStorage
                 try {
-                    log(`remoteWebDavStorage.putFile ${remoteDir}/${path.basename(file.name)}`);
-                    await this.remoteWebDavStorage.putFile(file.name, remoteDir);
+                    log(`remoteStorage.putFile ${remoteDir}/${path.basename(file.name)}`);
+                    await this.remoteStorage.putFile(file.name, remoteDir);
+                    
                     sent[file.name] = true;
+                    await this.appDb.insert({table: 'remote_sent', ignore: true, rows: [{id: file.name, remoteDir}]});
                 } catch (e) {
                     log(LM_ERR, e.stack);
                 }
             }
+
+            //почистим remoteSent и БД
+            //несколько неоптимально, таскает все записи из БД
+            const rows = await this.appDb.select({table: 'remote_sent'});
+            for (const row of rows) {
+                if (row.remoteDir === remoteDir && !foundFiles.has(row.id)) {
+                    delete sent[row.id];
+                    await this.appDb.delete({table: 'remote_sent', where: `@@id(${this.appDb.esc(row.id)})`});
+                }
+            }
         }
 
         let i = 0;
@@ -294,7 +311,9 @@ class ReaderWorker {
             const oldFile = file.name;
 
             //реально удаляем только если сохранили в хранилище или размер dir увеличен в 1.5 раза
-            if ((moveToRemote && this.remoteWebDavStorage && sent[oldFile]) || size > maxSize*1.5) {
+            if (!(moveToRemote && this.remoteStorage)
+                || (moveToRemote && this.remoteStorage && sent[oldFile])
+                || size > maxSize*1.5) {
                 await fs.remove(oldFile);
                 j++;
             }
@@ -302,20 +321,38 @@ class ReaderWorker {
             size -= file.stat.size;
             i++;
         }
+
         log(`removed ${j} files`);
     }
 
     async periodicCleanDir(cleanConfig) {
-        while (1) {// eslint-disable-line no-constant-condition
-            for (const [remoteDir, config] of Object.entries(cleanConfig)) {
-                try {
-                    await this.cleanDir(config.dir, remoteDir, config.maxSize, config.moveToRemote);
-                } catch(e) {
-                    log(LM_ERR, e.stack);
+        try {
+            if (!this.remoteSent)
+                this.remoteSent = {};
+
+            //инициализация this.remoteSent
+            if (this.remoteStorage) {
+                const rows = await this.appDb.select({table: 'remote_sent'});
+                for (const row of rows) {
+                    this.remoteSent[row.id] = true;
                 }
             }
 
-            await utils.sleep(cleanDirPeriod);
+            while (1) {// eslint-disable-line no-constant-condition
+
+                for (const [remoteDir, config] of Object.entries(cleanConfig)) {
+                    try {
+                        await this.cleanDir(config.dir, remoteDir, config.maxSize, config.moveToRemote);
+                    } catch(e) {
+                        log(LM_ERR, e.stack);
+                    }
+                }
+
+                await utils.sleep(cleanDirPeriod);
+            }
+        } catch (e) {
+            log(LM_FATAL, e.message);
+            ayncExit.exit(1);
         }
     }
 

server/core/RemoteStorage.js

@@ -0,0 +1,98 @@
+const fs = require('fs-extra');
+const path = require('path');
+
+const WebSocketConnection = require('./WebSocketConnection');
+
+class RemoteStorage {
+    constructor(config) {
+        this.config = Object.assign({}, config);
+        this.config.maxContentLength = this.config.maxContentLength || 10*1024*1024;
+
+        this.accessToken = this.config.accessToken;
+
+        this.wsc = new WebSocketConnection(config.url);
+    }
+
+    async wsRequest(query) {
+        const response = await this.wsc.message(
+            await this.wsc.send(Object.assign({accessToken: this.accessToken}, query), 600),
+            600
+        );
+        if (response.error)
+            throw new Error(response.error);
+        return response;
+    }
+
+    async wsStat(fileName) {
+        return await this.wsRequest({action: 'get-stat', fileName});
+    }
+
+    async wsGetFile(fileName) {
+        return this.wsRequest({action: 'get-file', fileName});
+    }
+
+    async wsPutFile(fileName, data) {//data base64 encoded string
+        return this.wsRequest({action: 'put-file', fileName, data});
+    }
+
+    async wsDelFile(fileName) {
+        return this.wsRequest({action: 'del-file', fileName});
+    }
+
+    makeRemoteFileName(fileName, dir = '') {
+        const base = path.basename(fileName);
+        if (base.length > 3) {
+            return `${dir}/${base.substr(0, 3)}/${base}`;
+        } else {
+            return `${dir}/${base}`;
+        }
+    }
+
+    async putFile(fileName, dir = '') {
+        if (!await fs.pathExists(fileName)) {
+            throw new Error(`File not found: ${fileName}`);
+        }
+
+        const remoteFilename = this.makeRemoteFileName(fileName, dir);
+
+        try {
+            const localStat = await fs.stat(fileName);
+            let remoteStat = await this.wsStat(remoteFilename);
+            remoteStat = remoteStat.stat;
+
+            if (remoteStat.isFile && localStat.size == remoteStat.size) {
+                return;
+            }
+
+            await this.wsDelFile(remoteFilename);
+        } catch (e) {
+            //
+        }
+
+        const data = await fs.readFile(fileName, 'base64');
+        await this.wsPutFile(remoteFilename, data);
+    }
+
+    async getFile(fileName, dir = '') {
+        if (await fs.pathExists(fileName)) {
+            return;
+        }
+
+        const remoteFilename = this.makeRemoteFileName(fileName, dir);
+
+        const response = await this.wsGetFile(remoteFilename);
+        await fs.writeFile(fileName, response.data, 'base64');
+    }
+
+    async getFileSuccess(filename, dir = '') {
+        try {
+            await this.getFile(filename, dir);
+            return true;
+        } catch (e) {
+            //
+        }
+        return false;
+    }
+}
+
+module.exports = RemoteStorage;

server/db/jembaMigrations/app/001-create.js

@@ -0,0 +1,12 @@
+module.exports = {
+    up: [
+        ['create', {
+            table: 'remote_sent'
+        }],
+    ],    
+    down: [
+        ['drop', {
+            table: 'remote_sent'
+        }],
+    ]
+};

server/db/jembaMigrations/app/index.js

@@ -0,0 +1,6 @@
+module.exports = {
+    table: 'migration1',
+    data: [
+        {id: 1, name: 'create', data: require('./001-create')}
+    ]
+}

server/db/jembaMigrations/index.js

@@ -1,4 +1,5 @@
 module.exports = {
+    'app': require('./app'),
     'reader-storage': require('./reader-storage'),
     'book-update-server': require('./book-update-server'),
 };