Merge branch 'release/0.8.2-5'

0.8.2-4

client/api/misc.js

@@ -1,4 +1,5 @@
 import axios from 'axios';
+import wsc from './webSocketConnection';
 
 const api = axios.create({
   baseURL: '/api'
@@ -6,9 +7,20 @@ const api = axios.create({
 
 class Misc {
     async loadConfig() {
-        const response = await api.post('/config', {params: [
+
+        const query = {params: [
             'name', 'version', 'mode', 'maxUploadFileSize', 'useExternalBookConverter', 'branch',
-        ]});
+        ]};
+
+        try {
+            await wsc.open();
+            return await wsc.message(wsc.send(Object.assign({action: 'get-config'}, query)));
+        } catch (e) {
+            console.error(e);
+        }
+
+        //если с WebSocket проблема, работаем по http
+        const response = await api.post('/config', query);
         return response.data;
     }
 }

client/api/reader.js

@@ -1,6 +1,6 @@
 import axios from 'axios';
-
 import * as utils from '../share/utils';
+import wsc from './webSocketConnection';
 
 const api = axios.create({
     baseURL: '/api/reader'
@@ -11,8 +11,61 @@ const workerApi = axios.create({
 });
 
 class Reader {
+    constructor() {
+    }
+
+    async getWorkerStateFinish(workerId, callback) {
+        if (!callback) callback = () => {};
+
+        let response = {};
+
+        try {
+            await wsc.open();
+            const requestId = wsc.send({action: 'worker-get-state-finish', workerId});
+
+            while (1) {// eslint-disable-line no-constant-condition
+                response = await wsc.message(requestId);
+                callback(response);
+
+                if (response.state == 'finish' || response.state == 'error') {
+                    break;
+                }
+            }
+            return response;
+        } catch (e) {
+            console.error(e);
+        }
+
+        //если с WebSocket проблема, работаем по http
+        const refreshPause = 500;
+        let i = 0;
+        response = {};
+        while (1) {// eslint-disable-line no-constant-condition
+            const prevProgress = response.progress || 0;
+            const prevState = response.state || 0;
+            response = await workerApi.post('/get-state', {workerId});
+            response = response.data;
+            callback(response);
+
+            if (response.state == 'finish' || response.state == 'error') {
+                break;
+            }
+
+            if (i > 0)
+                await utils.sleep(refreshPause);
+
+            i++;
+            if (i > 120*1000/refreshPause) {//2 мин ждем телодвижений воркера
+                throw new Error('Слишком долгое время ожидания');
+            }
+            //проверка воркера
+            i = (prevProgress != response.progress || prevState != response.state ? 1 : i);
+        }
+
+        return response;
+    }
+
     async loadBook(opts, callback) {
-        const refreshPause = 300;
         if (!callback) callback = () => {};
 
         let response = await api.post('/load-book', opts);
@@ -22,62 +75,90 @@ class Reader {
             throw new Error('Неверный ответ api');
 
         callback({totalSteps: 4});
+        callback(response.data);
 
-        let i = 0;
-        while (1) {// eslint-disable-line no-constant-condition
-            callback(response.data);
+        response = await this.getWorkerStateFinish(workerId, callback);
 
-            if (response.data.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл
+        if (response) {
+            if (response.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл
                 callback({step: 4});
-                const book = await this.loadCachedBook(response.data.path, callback);
-                return Object.assign({}, response.data, {data: book.data});
+                const book = await this.loadCachedBook(response.path, callback, response.size);
+                return Object.assign({}, response, {data: book.data});
             }
-            if (response.data.state == 'error') {
-                let errMes = response.data.error;
+
+            if (response.state == 'error') {
+                let errMes = response.error;
                 if (errMes.indexOf('getaddrinfo') >= 0 || 
                     errMes.indexOf('ECONNRESET') >= 0 ||
                     errMes.indexOf('EINVAL') >= 0 ||
                     errMes.indexOf('404') >= 0)
-                    errMes = `Ресурс не найден по адресу: ${response.data.url}`;
+                    errMes = `Ресурс не найден по адресу: ${response.url}`;
                 throw new Error(errMes);
             }
-            if (i > 0)
-                await utils.sleep(refreshPause);
+        } else {
+            throw new Error('Пустой ответ сервера');
+        }
+    }
 
-            i++;
-            if (i > 120*1000/refreshPause) {//2 мин ждем телодвижений воркера
-                throw new Error('Слишком долгое время ожидания');
+    async checkCachedBook(url) {
+        let estSize = -1;
+        try {
+            const response = await axios.head(url, {headers: {'Cache-Control': 'no-cache'}});
+
+            if (response.headers['content-length']) {
+                estSize = response.headers['content-length'];
+            }
+        } catch (e) {
+            //восстановим при необходимости файл на сервере из удаленного облака
+            let response = null
+            
+            try {
+                await wsc.open();
+                response = await wsc.message(wsc.send({action: 'reader-restore-cached-file', path: url}));
+            } catch (e) {
+                console.error(e);
+                //если с WebSocket проблема, работаем по http
+                response = await api.post('/restore-cached-file', {path: url});
+                response = response.data;
+            }
+
+            const workerId = response.workerId;
+            if (!workerId)
+                throw new Error('Неверный ответ api');
+
+            response = await this.getWorkerStateFinish(workerId);
+            if (response.state == 'error') {
+                throw new Error(response.error);
+            }
+            if (response.size && estSize < 0) {
+                estSize = response.size;
             }
-            //проверка воркера
-            const prevProgress = response.data.progress;
-            const prevState = response.data.state;
-            response = await workerApi.post('/get-state', {workerId});
-            i = (prevProgress != response.data.progress || prevState != response.data.state ? 1 : i);
         }
+
+        return estSize;
     }
 
-    async checkUrl(url) {
-        return await axios.head(url, {headers: {'Cache-Control': 'no-cache'}});
-    }
-
-    async loadCachedBook(url, callback) {
-        const response = await axios.head(url);
-
-        let estSize = 1000000;
-        if (response.headers['content-length']) {
-            estSize = response.headers['content-length'];
-        }
+    async loadCachedBook(url, callback, estSize = -1) {
+        if (!callback) callback = () => {};
 
         callback({state: 'loading', progress: 0});
+
+        //получение размера файла
+        if (estSize && estSize < 0) {
+            estSize = await this.checkCachedBook(url);
+        }
+
+        //получение файла
+        estSize = (estSize > 0 ? estSize : 1000000);
         const options = {
-            onDownloadProgress: progress => {
+            onDownloadProgress: (progress) => {
                 while (progress.loaded > estSize) estSize *= 1.5;
 
                 if (callback)
                     callback({progress: Math.round((progress.loaded*100)/estSize)});
             }
         }
-        //загрузка
+
         return await axios.get(url, options);
     }
 
@@ -114,13 +195,22 @@ class Reader {
     }
 
     async storage(request) {
-        let response = await api.post('/storage', request);
+        let response = null;
+        try {
+            await wsc.open();
+            response = await wsc.message(wsc.send({action: 'reader-storage', body: request}));
+        } catch (e) {
+            console.error(e);
+            //если с WebSocket проблема, работаем по http
+            response = await api.post('/storage', request);
+            response = response.data;
+        }
 
-        const state = response.data.state;
+        const state = response.state;
         if (!state)
             throw new Error('Неверный ответ api');
 
-        return response.data;
+        return response;
     }
 }
 

client/api/webSocketConnection.js

@@ -0,0 +1,176 @@
+const cleanPeriod = 60*1000;//1 минута
+
+class WebSocketConnection {
+    //messageLifeTime в минутах (cleanPeriod)
+    constructor(messageLifeTime = 5) {
+        this.ws = null;
+        this.timer = null;
+        this.listeners = [];
+        this.messageQueue = [];
+        this.messageLifeTime = messageLifeTime;
+        this.requestId = 0;
+    }
+
+    addListener(listener) {
+        if (this.listeners.indexOf(listener) < 0)
+            this.listeners.push(Object.assign({regTime: Date.now()}, listener));
+    }
+
+    //рассылаем сообщение и удаляем те обработчики, которые его получили
+    emit(mes, isError) {
+        const len = this.listeners.length;
+        if (len > 0) {
+            let newListeners = [];
+            for (const listener of this.listeners) {
+                let emitted = false;
+                if (isError) {
+                    if (listener.onError)
+                        listener.onError(mes);
+                    emitted = true;
+                } else {
+                    if (listener.onMessage) {
+                        if (listener.requestId) {
+                            if (listener.requestId === mes.requestId) {
+                                listener.onMessage(mes);
+                                emitted = true;
+                            }
+                        } else {
+                            listener.onMessage(mes);
+                            emitted = true;
+                        }
+                    } else {
+                        emitted = true;
+                    }
+                }
+
+                if (!emitted)
+                    newListeners.push(listener);
+            }
+            this.listeners = newListeners;
+        }
+        
+        return this.listeners.length != len;
+    }
+
+    open(url) {
+        return new Promise((resolve, reject) => {
+            if (this.ws && this.ws.readyState == WebSocket.OPEN) {
+                resolve(this.ws);
+            } else {
+                let protocol = 'ws:';
+                if (window.location.protocol == 'https:') {
+                    protocol = 'wss:'
+                }
+
+                url = url || `${protocol}//${window.location.host}/ws`;
+                
+                this.ws = new WebSocket(url);
+
+                if (this.timer) {
+                    clearTimeout(this.timer);
+                }
+                this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod);
+
+                let resolved = false;
+                this.ws.onopen = (e) => {
+                    resolved = true;
+                    resolve(e);
+                };
+
+                this.ws.onmessage = (e) => {
+                    try {
+                        const mes = JSON.parse(e.data);
+                        this.messageQueue.push({regTime: Date.now(), mes});
+
+                        let newMessageQueue = [];
+                        for (const message of this.messageQueue) {
+                            if (!this.emit(message.mes)) {
+                                newMessageQueue.push(message);
+                            }
+                        }
+
+                        this.messageQueue = newMessageQueue;
+                    } catch (e) {
+                        this.emit(e.message, true);
+                    }
+                };
+
+                this.ws.onerror = (e) => {
+                    this.emit(e.message, true);
+                    if (!resolved)
+                        reject(e);
+                };
+            }
+        });
+    }
+
+    //timeout в минутах (cleanPeriod)
+    message(requestId, timeout = 2) {
+        return new Promise((resolve, reject) => {
+            this.addListener({
+                requestId,
+                timeout,
+                onMessage: (mes) => {
+                    if (mes.error) {
+                        reject(mes.error);
+                    } else {
+                        resolve(mes);
+                    }
+                },
+                onError: (e) => {
+                    reject(e);
+                }
+            });
+        });
+    }
+
+    send(req) {
+        if (this.ws && this.ws.readyState == WebSocket.OPEN) {
+            const requestId = ++this.requestId;
+            this.ws.send(JSON.stringify(Object.assign({requestId}, req)));
+            return requestId;
+        } else {
+            throw new Error('WebSocket connection is not ready');
+        }
+    }
+
+    close() {
+        if (this.ws && this.ws.readyState == WebSocket.OPEN) {
+            this.ws.close();
+        }
+    }
+
+    periodicClean() {
+        try {
+            this.timer = null;
+
+            const now = Date.now();
+            //чистка listeners
+            let newListeners = [];
+            for (const listener of this.listeners) {
+                if (now - listener.regTime < listener.timeout*cleanPeriod - 50) {
+                    newListeners.push(listener);
+                } else {
+                    if (listener.onError)
+                        listener.onError('Время ожидания ответа истекло');
+                }
+            }
+            this.listeners = newListeners;
+
+            //чистка messageQueue
+            let newMessageQueue = [];
+            for (const message of this.messageQueue) {
+                if (now - message.regTime < this.messageLifeTime*cleanPeriod - 50) {
+                    newMessageQueue.push(message);
+                }
+            }
+            this.messageQueue = newMessageQueue;
+        } finally {
+            if (this.ws.readyState == WebSocket.OPEN) {
+                this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod);
+            }
+        }
+    }
+}
+
+export default new WebSocketConnection();

client/components/Reader/LoaderPage/LoaderPage.vue

@@ -112,7 +112,7 @@ class LoaderPage extends Vue {
 
     submitUrl() {
         if (this.bookUrl) {
-            this.$emit('load-book', {url: this.bookUrl});
+            this.$emit('load-book', {url: this.bookUrl, force: true});
             this.bookUrl = '';
         }
     }

client/components/Reader/Reader.vue

@@ -719,15 +719,16 @@ class Reader extends Vue {
             case 'scrolling':
             case 'search':
             case 'copyText':
-            case 'recentBooks':
+            case 'refresh':
             case 'offlineMode':
+            case 'recentBooks':
             case 'settings':
-                if (this[`${button}Active`])
+                if (this.progressActive) {
+                    classResult = classDisabled;
+                } else if (this[`${button}Active`]) {
                     classResult = classActive;
+                }
                 break;
-        }
-
-        switch (button) {
             case 'undoAction':
                 if (this.actionCur <= 0)
                     classResult = classDisabled;

client/components/Reader/RecentBooksPage/RecentBooksPage.vue

@@ -272,7 +272,7 @@ class RecentBooksPage extends Vue {
 
     async downloadBook(fb2path) {
         try {
-            await readerApi.checkUrl(fb2path);
+            await readerApi.checkCachedBook(fb2path);
 
             const d = this.$refs.download;
             d.href = fb2path;

client/components/Reader/share/bookManager.js

@@ -464,7 +464,7 @@ class BookManager {
 
     addEventListener(listener) {
         if (this.eventListeners.indexOf(listener) < 0)
-            this.eventListeners.push(listener);        
+            this.eventListeners.push(listener);
     }
 
     removeEventListener(listener) {

client/components/Reader/versionHistory.js

@@ -1,4 +1,15 @@
 export const versionHistory = [
+{
+    showUntil: '2020-01-19',
+    header: '0.8.2 (2020-01-20)',
+    content:
+`
+<ul>
+    <li>внутренние оптимизации</li>
+</ul>
+`
+},
+
 {
     showUntil: '2020-01-06',
     header: '0.8.1 (2020-01-07)',

docs/omnireader/omnireader

@@ -8,6 +8,7 @@ server {
   server_name omnireader.ru;
 
   client_max_body_size 50m;
+  proxy_read_timeout 1h;
 
   gzip on;
   gzip_min_length 1024;
@@ -18,6 +19,13 @@ server {
     proxy_pass http://127.0.0.1:44081;
   }
 
+  location /ws {
+    proxy_pass http://127.0.0.1:44081;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+  }
+
   location / {
     root /home/liberama/public;
 

docs/omnireader/omnireader_http

@@ -0,0 +1,59 @@
+server {
+  listen 80;
+  server_name omnireader.ru;
+
+  client_max_body_size 50m;
+  proxy_read_timeout 1h;
+
+  gzip on;
+  gzip_min_length 1024;
+  gzip_proxied expired no-cache no-store private auth;
+  gzip_types *;
+
+  location /api {
+    proxy_pass http://127.0.0.1:44081;
+  }
+
+  location /ws {
+    proxy_pass http://127.0.0.1:44081;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+  }
+
+  location / {
+    root /home/liberama/public;
+
+    location /tmp {
+      add_header Content-Type text/xml;
+      add_header Content-Encoding gzip;
+    }
+
+    location ~* \.(?:manifest|appcache|html)$ {
+      expires -1;
+    }
+  }
+}
+
+server {
+  listen 80;
+  server_name old.omnireader.ru;
+
+  client_max_body_size 50m;
+
+  gzip on;
+  gzip_min_length 1024;
+  gzip_proxied expired no-cache no-store private auth;
+  gzip_types *;
+
+  root /home/oldreader;
+
+  index index.html;
+
+  # Обработка php файлов с помощью fpm
+  location ~ \.php$ { 
+    try_files $uri =404; 
+    include /etc/nginx/fastcgi.conf;
+    fastcgi_pass unix:/run/php/php7.2-fpm.sock;
+  }
+}

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "Liberama",
-  "version": "0.8.1",
+  "version": "0.8.2",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -1725,6 +1725,11 @@
         }
       }
     },
+    "base-64": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
+      "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs="
+    },
     "base-x": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.7.tgz",
@@ -5719,6 +5724,11 @@
         "parse-passwd": "^1.0.0"
       }
     },
+    "hot-patcher": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/hot-patcher/-/hot-patcher-0.5.0.tgz",
+      "integrity": "sha512-2Uu2W0s8+dnqXzdlg0MRsRzPoDCs1wVjOGSyMRRaMzLDX4bgHw6xDYKccsWafXPPxQpkQfEjgW6+17pwcg60bw=="
+    },
     "hsl-regex": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/hsl-regex/-/hsl-regex-1.0.0.tgz",
@@ -6902,6 +6912,11 @@
         }
       }
     },
+    "merge": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/merge/-/merge-1.2.1.tgz",
+      "integrity": "sha512-VjFo4P5Whtj4vsLzsYBu5ayHhoHJ0UqNm7ibvShmbmoz7tGi0vXaoJbGdB+GmDMLUdg8DpQXEIeVDAe8MaABvQ=="
+    },
     "merge-descriptors": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
@@ -7882,6 +7897,11 @@
       "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==",
       "dev": true
     },
+    "path-posix": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/path-posix/-/path-posix-1.0.0.tgz",
+      "integrity": "sha1-BrJhE/Vr6rBCVFojv6iAA8ysJg8="
+    },
     "path-to-regexp": {
       "version": "0.1.7",
       "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
@@ -10423,6 +10443,11 @@
       "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=",
       "dev": true
     },
+    "querystringify": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.1.tgz",
+      "integrity": "sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA=="
+    },
     "randombytes": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
@@ -10709,6 +10734,11 @@
       "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==",
       "dev": true
     },
+    "requires-port": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
+      "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
+    },
     "resize-observer-polyfill": {
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz",
@@ -12489,6 +12519,11 @@
         }
       }
     },
+    "url-join": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz",
+      "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA=="
+    },
     "url-loader": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/url-loader/-/url-loader-1.1.2.tgz",
@@ -12508,6 +12543,15 @@
         }
       }
     },
+    "url-parse": {
+      "version": "1.4.7",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz",
+      "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==",
+      "requires": {
+        "querystringify": "^2.1.1",
+        "requires-port": "^1.0.0"
+      }
+    },
     "url-parse-lax": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz",
@@ -12737,6 +12781,32 @@
         "neo-async": "^2.5.0"
       }
     },
+    "webdav": {
+      "version": "2.10.1",
+      "resolved": "https://registry.npmjs.org/webdav/-/webdav-2.10.1.tgz",
+      "integrity": "sha512-3UfnjGTAqSM9MW3Rpt1KrY1KneYK0wPCFryHTncqw1OP1pyiniT3uYhVpgmH6za/TkWOfnTnKCDKhwrLJFdzow==",
+      "requires": {
+        "axios": "^0.19.0",
+        "base-64": "^0.1.0",
+        "hot-patcher": "^0.5.0",
+        "merge": "^1.2.1",
+        "minimatch": "^3.0.4",
+        "path-posix": "^1.0.0",
+        "url-join": "^4.0.1",
+        "url-parse": "^1.4.7",
+        "xml2js": "^0.4.19"
+      },
+      "dependencies": {
+        "axios": {
+          "version": "0.19.1",
+          "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.1.tgz",
+          "integrity": "sha512-Yl+7nfreYKaLRvAvjNPkvfjnQHJM1yLBY3zhqAwcJSwR/6ETkanUgylgtIvkvz0xJ+p/vZuNw8X7Hnb7Whsbpw==",
+          "requires": {
+            "follow-redirects": "1.5.10"
+          }
+        }
+      }
+    },
     "webpack": {
       "version": "4.40.2",
       "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.40.2.tgz",
@@ -13005,6 +13075,25 @@
         "mkdirp": "^0.5.1"
       }
     },
+    "ws": {
+      "version": "7.2.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz",
+      "integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A=="
+    },
+    "xml2js": {
+      "version": "0.4.23",
+      "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
+      "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
+      "requires": {
+        "sax": ">=0.6.0",
+        "xmlbuilder": "~11.0.0"
+      }
+    },
+    "xmlbuilder": {
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
+      "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
+    },
     "xtend": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "Liberama",
-  "version": "0.8.1",
+  "version": "0.8.2",
   "author": "Book Pauk <bookpauk@gmail.com>",
   "license": "CC0-1.0",
   "repository": "bookpauk/liberama",
@@ -84,6 +84,8 @@
     "vue-router": "^3.1.3",
     "vuex": "^3.1.1",
     "vuex-persistedstate": "^2.5.4",
+    "webdav": "^2.10.1",
+    "ws": "^7.2.1",
     "zip-stream": "^2.1.2"
   }
 }

server/config/base.js

@@ -21,7 +21,7 @@ module.exports = {
     maxTempPublicDirSize: 512*1024*1024,//512Мб
     maxUploadPublicDirSize: 200*1024*1024,//100Мб
 
-    useExternalBookConverter: false,
+    useExternalBookConverter: false,    
 
     db: [
         {
@@ -45,5 +45,14 @@ module.exports = {
         },
     ],
 
+    remoteWebDavStorage: false,
+    /*
+    remoteWebDavStorage: {
+        url: '127.0.0.1:1900',
+        username: '',
+        password: '',
+    },
+    */
+
 };
 

server/config/index.js

@@ -10,6 +10,7 @@ const propsToSave = [
     'useExternalBookConverter',
     
     'servers',
+    'remoteWebDavStorage',
 ];
 
 let instance = null;

server/controllers/ReaderController.js

@@ -35,9 +35,9 @@ class ReaderController extends BaseController {
         const request = req.body;
         let error = '';
         try {
-            if (!request.action) 
+            if (!request.action)
                 throw new Error(`key 'action' is empty`);
-            if (!request.items || Array.isArray(request.data)) 
+            if (!request.items || Array.isArray(request.data))
                 throw new Error(`key 'items' is empty`);
 
             return await this.readerStorage.doAction(request);
@@ -62,6 +62,24 @@ class ReaderController extends BaseController {
         res.status(400).send({error});
         return false;
     }
+
+    async restoreCachedFile(req, res) {
+        const request = req.body;
+        let error = '';
+        try {
+            if (!request.path) 
+                throw new Error(`key 'path' is empty`);
+
+            const workerId = this.readerWorker.restoreCachedFile(request.path);
+            const state = this.workerState.getState(workerId);
+            return (state ? state : {});
+        } catch (e) {
+            error = e.message;
+        }
+        //bad request
+        res.status(400).send({error});
+        return false;
+    }
 }
 
 module.exports = ReaderController;

server/controllers/WebSocketController.js

@@ -0,0 +1,164 @@
+const WebSocket = require ('ws');
+const _ = require('lodash');
+
+const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton
+const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton
+const WorkerState = require('../core/WorkerState');//singleton
+const log = new (require('../core/AppLogger'))().log;//singleton
+const utils = require('../core/utils');
+
+const cleanPeriod = 1*60*1000;//1 минута
+const closeSocketOnIdle = 5*60*1000;//5 минут
+
+class WebSocketController {
+    constructor(wss, config) {
+        this.config = config;
+        this.isDevelopment = (config.branch == 'development');
+
+        this.readerStorage = new ReaderStorage();
+        this.readerWorker = new ReaderWorker(config);
+        this.workerState = new WorkerState();
+
+        this.wss = wss;
+
+        wss.on('connection', (ws) => {
+            ws.on('message', (message) => {
+                this.onMessage(ws, message);
+            });
+        });
+
+        setTimeout(() => { this.periodicClean(); }, cleanPeriod);
+    }
+
+    periodicClean() {
+        try {
+            const now = Date.now();
+            this.wss.clients.forEach((ws) => {
+                if (!ws.lastActivity || now - ws.lastActivity > closeSocketOnIdle - 50) {
+                    ws.terminate();
+                }
+            });
+        } finally {
+            setTimeout(() => { this.periodicClean(); }, cleanPeriod);
+        }
+    }
+
+    async onMessage(ws, message) {
+        let req = {};
+        try {
+            if (this.isDevelopment) {
+                log(`WebSocket-IN:  ${message.substr(0, 4000)}`);
+            }
+
+            ws.lastActivity = Date.now();
+            req = JSON.parse(message);
+            switch (req.action) {
+                case 'test':
+                    await this.test(req, ws); break;
+                case 'get-config':
+                    await this.getConfig(req, ws); break;
+                case 'worker-get-state':
+                    await this.workerGetState(req, ws); break;
+                case 'worker-get-state-finish':
+                    await this.workerGetStateFinish(req, ws); break;
+                case 'reader-restore-cached-file':
+                    await this.readerRestoreCachedFile(req, ws); break;
+                case 'reader-storage':
+                    await this.readerStorageDo(req, ws); break;
+
+                default:
+                    throw new Error(`Action not found: ${req.action}`);
+            }
+        } catch (e) {
+            this.send({error: e.message}, req, ws);
+        }
+    }
+
+    send(res, req, ws) {
+        if (ws.readyState == WebSocket.OPEN) {
+            ws.lastActivity = Date.now();
+            let r = res;
+            if (req.requestId)
+                r = Object.assign({requestId: req.requestId}, r);
+
+            const message = JSON.stringify(r);
+            ws.send(message);
+
+            if (this.isDevelopment) {
+                log(`WebSocket-OUT: ${message.substr(0, 4000)}`);
+            }
+
+        }
+    }
+
+    //Actions ------------------------------------------------------------------
+    async test(req, ws) {
+        this.send({message: 'Liberama project is awesome'}, req, ws);
+    }
+
+    async getConfig(req, ws) {
+        if (Array.isArray(req.params)) {
+            this.send(_.pick(this.config, req.params), req, ws);
+        } else {
+            throw new Error('params is not an array');
+        }
+    }
+
+    async workerGetState(req, ws) {
+        if (!req.workerId)
+            throw new Error(`key 'workerId' is wrong`);
+
+        const state = this.workerState.getState(req.workerId);
+        this.send((state ? state : {}), req, ws);
+    }
+
+    async workerGetStateFinish(req, ws) {
+        if (!req.workerId)
+            throw new Error(`key 'workerId' is wrong`);
+
+        const refreshPause = 200;
+        let i = 0;
+        let state = {};
+        while (1) {// eslint-disable-line no-constant-condition
+            const prevProgress = state.progress || -1;
+            const prevState = state.state || '';
+            state = this.workerState.getState(req.workerId);
+
+            this.send((state ? state : {}), req, ws);
+            if (!state) break;
+
+            if (state.state != 'finish' && state.state != 'error')
+                await utils.sleep(refreshPause);
+            else
+                break;
+
+            i++;
+            if (i > 2*60*1000/refreshPause) {//2 мин ждем телодвижений воркера
+                this.send({state: 'error', error: 'Время ожидания процесса истекло'}, req, ws);
+            }
+            i = (prevProgress != state.progress || prevState != state.state ? 1 : i);
+        }        
+    }
+
+    async readerRestoreCachedFile(req, ws) {
+        if (!req.path)
+            throw new Error(`key 'path' is empty`);
+
+        const workerId = this.readerWorker.restoreCachedFile(req.path);
+        const state = this.workerState.getState(workerId);
+        this.send((state ? state : {}), req, ws);
+    }
+
+    async readerStorageDo(req, ws) {
+        if (!req.body)
+            throw new Error(`key 'body' is empty`);
+        if (!req.body.action)
+            throw new Error(`key 'action' is empty`);
+        if (!req.body.items || Array.isArray(req.body.data))
+            throw new Error(`key 'items' is empty`);
+
+        this.send(await this.readerStorage.doAction(req.body), req, ws);
+    }
+}
+
+module.exports = WebSocketController;

server/controllers/WorkerController.js

@@ -1,5 +1,6 @@
 const BaseController = require('./BaseController');
 const WorkerState = require('../core/WorkerState');//singleton
+const utils = require('../core/utils');
 
 class WorkerController extends BaseController {
     constructor(config) {
@@ -15,6 +16,7 @@ class WorkerController extends BaseController {
                 throw new Error(`key 'workerId' is wrong`);
 
             const state = this.workerState.getState(request.workerId);
+
             return (state ? state : {});
         } catch (e) {
             error = e.message;
@@ -23,6 +25,60 @@ class WorkerController extends BaseController {
         res.status(400).send({error});
         return false;
     }
+
+    //TODO: удалить бесполезную getStateFinish
+    async getStateFinish(req, res) {
+        const request = req.body;
+        let error = '';
+        try {
+            if (!request.workerId)
+                throw new Error(`key 'workerId' is wrong`);
+
+            res.writeHead(200, {
+                'Content-Type': 'text/json; charset=utf-8',
+            });
+
+            const splitter = '-- aod2t5hDXU32bUFyqlFE next status --';            
+            const refreshPause = 200;
+            let i = 0;
+            let prevProgress = -1;
+            let prevState = '';
+            let state;
+            while (1) {// eslint-disable-line no-constant-condition
+                state = this.workerState.getState(request.workerId);
+                if (!state) break;
+
+                res.write(splitter + JSON.stringify(state));
+                res.flush();
+
+                if (state.state != 'finish' && state.state != 'error')
+                    await utils.sleep(refreshPause);
+                else
+                    break;
+
+                i++;
+                if (i > 2*60*1000/refreshPause) {//2 мин ждем телодвижений воркера
+                    res.write(splitter + JSON.stringify({state: 'error', error: 'Слишком долгое время ожидания'}));
+                    break;
+                }
+                i = (prevProgress != state.progress || prevState != state.state ? 1 : i);
+                prevProgress = state.progress;
+                prevState = state.state;
+            }
+            
+            if (!state) {
+                res.write(splitter + JSON.stringify({}));
+            }
+
+            res.end();
+            return false;
+        } catch (e) {
+            error = e.message;
+        }
+        //bad request
+        res.status(400).send({error});
+        return false;
+    }
 }
 
 module.exports = WorkerController;

server/controllers/index.js

@@ -2,4 +2,5 @@ module.exports = {
     MiscController: require('./MiscController'),
     ReaderController: require('./ReaderController'),
     WorkerController: require('./WorkerController'),
+    WebSocketController: require('./WebSocketController'),
 }

server/core/FileDecompressor.js

@@ -5,6 +5,7 @@ const unbzip2Stream = require('unbzip2-stream');
 const tar = require('tar-fs');
 const ZipStreamer = require('./ZipStreamer');
 
+const appLogger = new (require('./AppLogger'))();//singleton
 const utils = require('./utils');
 const FileDetector = require('./FileDetector');
 
@@ -189,9 +190,9 @@ class FileDecompressor {
         });
     }
 
-    async gzipFile(inputFile, outputFile) {
+    async gzipFile(inputFile, outputFile, level = 1) {
         return new Promise((resolve, reject) => {
-            const gzip = zlib.createGzip({level: 1});
+            const gzip = zlib.createGzip({level});
             const input = fs.createReadStream(inputFile);
             const output = fs.createWriteStream(outputFile);
 
@@ -208,7 +209,21 @@ class FileDecompressor {
         const outFilename = `${outDir}/${hash}`;
 
         if (!await fs.pathExists(outFilename)) {
-            await this.gzipFile(filename, outFilename);
+            await this.gzipFile(filename, outFilename, 1);
+
+            // переупакуем через некоторое время на максималках
+            const filenameCopy = `${filename}.copy`;
+            await fs.copy(filename, filenameCopy);
+
+            (async() => {
+                await utils.sleep(5000);
+                const filenameGZ = `${filename}.gz`;
+                await this.gzipFile(filenameCopy, filenameGZ, 9);
+
+                await fs.move(filenameGZ, outFilename, {overwrite: true});
+
+                await fs.remove(filenameCopy);
+            })().catch((e) => { if (appLogger.inited) appLogger.log(LM_ERR, `FileDecompressor.gzipFileIfNotExists: ${e.message}`) });
         } else {
             await utils.touchFile(outFilename);
         }

server/core/Reader/ReaderWorker.js

@@ -5,10 +5,13 @@ const WorkerState = require('../WorkerState');//singleton
 const FileDownloader = require('../FileDownloader');
 const FileDecompressor = require('../FileDecompressor');
 const BookConverter = require('./BookConverter');
+const RemoteWebDavStorage = require('../RemoteWebDavStorage');
 
 const utils = require('../utils');
 const log = new (require('../AppLogger'))().log;//singleton
 
+const cleanDirPeriod = 60*60*1000;//1 раз в час
+
 let instance = null;
 
 //singleton
@@ -28,8 +31,15 @@ class ReaderWorker {
             this.decomp = new FileDecompressor();
             this.bookConverter = new BookConverter(this.config);
 
-            this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час
-            this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, 60*60*1000);//1 раз в час
+            this.remoteWebDavStorage = false;
+            if (config.remoteWebDavStorage) {
+                this.remoteWebDavStorage = new RemoteWebDavStorage(
+                    Object.assign({maxContentLength: config.maxUploadFileSize}, config.remoteWebDavStorage)
+                );
+            }
+
+            this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, cleanDirPeriod);
+            this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, cleanDirPeriod);
             
             instance = this;
         }
@@ -39,7 +49,6 @@ class ReaderWorker {
 
     async loadBook(opts, wState) {
         const url = opts.url;
-        let errMes = '';
         let decompDir = '';
         let downloadedFilename = '';
         let isUploaded = false;
@@ -87,17 +96,31 @@ class ReaderWorker {
             });
 
             //сжимаем файл в tmp, если там уже нет с тем же именем-sha256
-            const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, `${this.config.tempPublicDir}`);
+            const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, this.config.tempPublicDir);
+            const stat = await fs.stat(compFilename);
 
             wState.set({progress: 100});
 
             //finish
             const finishFilename = path.basename(compFilename);
-            wState.finish({path: `/tmp/${finishFilename}`});
+            wState.finish({path: `/tmp/${finishFilename}`, size: stat.size});
+
+            //лениво сохраним compFilename в удаленном хранилище
+            if (this.remoteWebDavStorage) {
+                (async() => {
+                    await utils.sleep(20*1000);
+                    try {
+                        //log(`remoteWebDavStorage.putFile ${path.basename(compFilename)}`);
+                        await this.remoteWebDavStorage.putFile(compFilename);
+                    } catch (e) {
+                        log(LM_ERR, e.stack);
+                    }
+                })();
+            }
 
         } catch (e) {
             log(LM_ERR, e.stack);
-            wState.set({state: 'error', error: (errMes ? errMes : e.message)});
+            wState.set({state: 'error', error: e.message});
         } finally {
             //clean
             if (decompDir)
@@ -133,6 +156,41 @@ class ReaderWorker {
         return `file://${hash}`;
     }
 
+    restoreCachedFile(filename) {
+        const workerId = this.workerState.generateWorkerId();
+        const wState = this.workerState.getControl(workerId);
+        wState.set({state: 'start'});
+
+        (async() => {
+            try {
+                wState.set({state: 'download', step: 1, totalSteps: 1, path: filename, progress: 0});
+
+                const basename = path.basename(filename);
+                const targetName = `${this.config.tempPublicDir}/${basename}`;
+
+                if (!await fs.pathExists(targetName)) {
+                    let found = false;
+                    if (this.remoteWebDavStorage) {
+                        found = await this.remoteWebDavStorage.getFileSuccess(targetName);
+                    } 
+
+                    if (!found) {
+                        throw new Error('404 Файл не найден');
+                    }
+                }
+
+                const stat = await fs.stat(targetName);
+                wState.finish({path: `/tmp/${basename}`, size: stat.size, progress: 100});
+            } catch (e) {
+                if (e.message.indexOf('404') < 0)
+                    log(LM_ERR, e.stack);
+                wState.set({state: 'error', error: e.message});
+            }
+        })();
+
+        return workerId;
+    }
+
     async periodicCleanDir(dir, maxSize, timeout) {
         try {
             const list = await fs.readdir(dir);
@@ -153,7 +211,19 @@ class ReaderWorker {
             let i = 0;
             while (i < files.length && size > maxSize) {
                 const file = files[i];
-                await fs.remove(`${dir}/${file.name}`);
+                const oldFile = `${dir}/${file.name}`;
+
+                //отправляем только this.config.tempPublicDir
+                //TODO: убрать в будущем, т.к. уже делается ленивое сохранение compFilename в удаленном хранилище
+                if (this.remoteWebDavStorage && dir === this.config.tempPublicDir) {
+                    try {
+                        //log(`remoteWebDavStorage.putFile ${path.basename(oldFile)}`);
+                        await this.remoteWebDavStorage.putFile(oldFile);
+                    } catch (e) {
+                        log(LM_ERR, e.stack);
+                    }
+                }
+                await fs.remove(oldFile);
                 size -= file.stat.size;
                 i++;
             }

server/core/RemoteWebDavStorage.js

@@ -0,0 +1,107 @@
+const fs = require('fs-extra');
+const path = require('path');
+
+const { createClient } = require('webdav');
+
+class RemoteWebDavStorage {
+    constructor(config) {
+        this.config = Object.assign({}, config);
+        this.config.maxContentLength = this.config.maxContentLength || 10*1024*1024;
+        this.wdc = createClient(config.url, this.config);
+    }
+
+    _convertStat(data) {
+        return {
+            isDirectory: function() {
+                return data.type === "directory";
+            },
+            isFile: function() {
+                return data.type === "file";
+            },
+            mtime: (new Date(data.lastmod)).getTime(),
+            name: data.basename,
+            size: data.size || 0
+        };
+    }
+
+    async stat(filename) {
+        const stat = await this.wdc.stat(filename);
+        return this._convertStat(stat);
+    }
+
+    async writeFile(filename, data) {
+        return await this.wdc.putFileContents(filename, data, { maxContentLength: this.config.maxContentLength })
+    }
+
+    async unlink(filename) {
+        return await this.wdc.deleteFile(filename);
+    }
+
+    async readFile(filename) {
+        return await this.wdc.getFileContents(filename, { maxContentLength: this.config.maxContentLength })
+    }
+
+    async mkdir(dirname) {
+        return await this.wdc.createDirectory(dirname);
+    }
+
+    async putFile(filename) {
+        if (!await fs.pathExists(filename)) {
+            throw new Error(`File not found: ${filename}`);
+        }
+
+        const base = path.basename(filename);
+        let remoteFilename = `/${base}`;
+        
+        if (base.length > 3) {
+            const remoteDir = `/${base.substr(0, 3)}`;
+            try {
+                await this.mkdir(remoteDir);
+            } catch (e) {
+                //
+            }
+            remoteFilename = `${remoteDir}/${base}`;
+        }
+
+        try {
+            const localStat = await fs.stat(filename);
+            const remoteStat = await this.stat(remoteFilename);
+            if (remoteStat.isFile && localStat.size == remoteStat.size) {
+                return;
+            }
+            await this.unlink(remoteFilename);
+        } catch (e) {
+            //
+        }
+
+        const data = await fs.readFile(filename);
+        await this.writeFile(remoteFilename, data);
+    }
+
+    async getFile(filename) {
+        if (await fs.pathExists(filename)) {
+            return;
+        }
+
+        const base = path.basename(filename);
+        let remoteFilename = `/${base}`;        
+        if (base.length > 3) {
+            remoteFilename = `/${base.substr(0, 3)}/${base}`;
+        }
+
+        const data = await this.readFile(remoteFilename);
+        await fs.writeFile(filename, data);
+    }
+
+    async getFileSuccess(filename) {
+        try {
+            await this.getFile(filename);
+            return true;
+        } catch (e) {
+            //
+        }
+        return false;
+    }
+}
+
+module.exports = RemoteWebDavStorage;

server/index.js

@@ -4,6 +4,8 @@ const path = require('path');
 const argv = require('minimist')(process.argv.slice(2));
 const express = require('express');
 const compression = require('compression');
+const http = require('http');
+const WebSocket = require ('ws');
 
 async function init() {
     //config
@@ -46,10 +48,13 @@ async function main() {
     const config = new (require('./config'))().config;//singleton
 
     //servers
-    for (let server of config.servers) {
-        if (server.mode !== 'none') {
+    for (let serverCfg of config.servers) {
+        if (serverCfg.mode !== 'none') {
             const app = express();
-            const serverConfig = Object.assign({}, config, server);
+            const server = http.createServer(app);
+            const wss = new WebSocket.Server({ server, maxPayload: 10*1024*1024 });
+
+            const serverConfig = Object.assign({}, config, serverCfg);
 
             let devModule = undefined;
             if (serverConfig.branch == 'development') {
@@ -73,7 +78,7 @@ async function main() {
                 }               
             }));
 
-            require('./routes').initRoutes(app, serverConfig);
+            require('./routes').initRoutes(app, wss, serverConfig);
 
             if (devModule) {
                 devModule.logErrors(app);
@@ -84,7 +89,7 @@ async function main() {
                 });
             }
 
-            app.listen(serverConfig.port, serverConfig.ip, function() {
+            server.listen(serverConfig.port, serverConfig.ip, function() {
                 log(`Server-${serverConfig.serverName} is ready on ${serverConfig.ip}:${serverConfig.port}, mode: ${serverConfig.mode}`);
             });
         }

server/routes.js

@@ -2,10 +2,11 @@ const c = require('./controllers');
 const utils = require('./core/utils');
 const multer = require('multer');
 
-function initRoutes(app, config) {
+function initRoutes(app, wss, config) {
     const misc = new c.MiscController(config);
     const reader = new c.ReaderController(config);
     const worker = new c.WorkerController(config);
+    new c.WebSocketController(wss, config);
 
     //access
     const [aAll, aNormal, aSite, aReader, aOmnireader] = // eslint-disable-line no-unused-vars
@@ -28,7 +29,9 @@ function initRoutes(app, config) {
         ['POST', '/api/reader/load-book', reader.loadBook.bind(reader), [aAll], {}],
         ['POST', '/api/reader/storage', reader.storage.bind(reader), [aAll], {}],
         ['POST', '/api/reader/upload-file', [upload.single('file'), reader.uploadFile.bind(reader)], [aAll], {}],
+        ['POST', '/api/reader/restore-cached-file', reader.restoreCachedFile.bind(reader), [aAll], {}],        
         ['POST', '/api/worker/get-state', worker.getState.bind(worker), [aAll], {}],
+        ['POST', '/api/worker/get-state-finish', worker.getStateFinish.bind(worker), [aAll], {}],
     ];
 
     //to app