Работа над RemoteLib

client/components/Search/Search.vue

@@ -694,9 +694,13 @@ class Search {
                 || makeValidFilenameOrEmpty(at[0])
                 || makeValidFilenameOrEmpty(at[1])
                 || downFileName;
-            downFileName = `${downFileName.substring(0, 100)}.${book.ext}`;
+            downFileName = downFileName.substring(0, 100);
 
-            const bookPath = `${book.folder}/${book.file}.${book.ext}`;
+            const ext = `.${book.ext}`;
+            if (downFileName.substring(downFileName.length - ext.length) != ext)
+                downFileName += ext;
+
+            const bookPath = `${book.folder}/${book.file}${ext}`;
             //подготовка
             const response = await this.api.getBookLink({bookPath, downFileName});
             

server/config/base.js

@@ -14,6 +14,7 @@ module.exports = {
     bookReadLink: '',
     loggingEnabled: true,
 
+    maxPayloadSize: 500,//in MB
     maxFilesDirSize: 1024*1024*1024,//1Gb
     queryCacheEnabled: true,
     cacheCleanInterval: 60,//minutes

server/core/FileDownloader.js

@@ -0,0 +1,125 @@
+const https = require('https');
+const axios = require('axios');
+const utils = require('./utils');
+
+const userAgent = 'Mozilla/5.0 (X11; HasCodingOs 1.0; Linux x64) AppleWebKit/637.36 (KHTML, like Gecko) Chrome/70.0.3112.101 Safari/637.36 HasBrowser/5.0';
+
+class FileDownloader {
+    constructor(limitDownloadSize = 0) {
+        this.limitDownloadSize = limitDownloadSize;
+    }
+
+    async load(url, callback, abort) {
+        let errMes = '';
+
+        const options = {
+            headers: {
+                'user-agent': userAgent,
+                timeout: 300*1000,
+            },
+            httpsAgent: new https.Agent({
+                rejectUnauthorized: false // решение проблемы 'unable to verify the first certificate' для некоторых сайтов с валидным сертификатом
+            }),
+            responseType: 'stream',
+        };
+
+        try {
+            const res = await axios.get(url, options);
+
+            let estSize = 0;
+            if (res.headers['content-length']) {
+                estSize = res.headers['content-length'];
+            }
+
+            if (this.limitDownloadSize && estSize > this.limitDownloadSize) {
+                throw new Error('Файл слишком большой');
+            }
+
+            let prevProg = 0;
+            let transferred = 0;
+
+            const download = this.streamToBuffer(res.data, (chunk) => {
+                transferred += chunk.length;
+                if (this.limitDownloadSize) {
+                    if (transferred > this.limitDownloadSize) {
+                        errMes = 'Файл слишком большой';
+                        res.request.abort();
+                    }
+                }
+
+                let prog = 0;
+                if (estSize)
+                    prog = Math.round(transferred/estSize*100);
+                else
+                    prog = Math.round(transferred/(transferred + 200000)*100);
+
+                if (prog != prevProg && callback)
+                    callback(prog);
+                prevProg = prog;
+
+                if (abort && abort()) {
+                    errMes = 'abort';
+                    res.request.abort();
+                }
+            });
+
+            return await download;
+        } catch (error) {
+            errMes = (errMes ? errMes : error.message);
+            throw new Error(errMes);
+        }
+    }
+
+    async head(url) {
+        const options = {
+            headers: {
+                'user-agent': userAgent,
+                timeout: 10*1000,
+            },
+        };
+
+        const res = await axios.head(url, options);
+        return res.headers;
+    }
+
+    streamToBuffer(stream, progress, timeout = 30*1000) {
+        return new Promise((resolve, reject) => {
+            
+            if (!progress)
+                progress = () => {};
+
+            const _buf = [];
+            let resolved = false;
+            let timer = 0;
+
+            stream.on('data', (chunk) => {
+                timer = 0;
+                _buf.push(chunk);
+                progress(chunk);
+            });
+            stream.on('end', () => {
+                resolved = true;
+                timer = timeout;
+                resolve(Buffer.concat(_buf));
+            });
+            stream.on('error', (err) => {
+                reject(err);
+            });
+            stream.on('aborted', () => {
+                reject(new Error('aborted'));
+            });
+
+            //бодяга с timer и timeout, чтобы гарантировать отсутствие зависания по каким-либо причинам
+            (async() => {
+                while (timer < timeout) {
+                    await utils.sleep(1000);
+                    timer += 1000;
+                }
+                if (!resolved)
+                    reject(new Error('FileDownloader: timed out'))
+            })();
+        });
+    }
+}
+
+module.exports = FileDownloader;

server/core/RemoteLib.js

@@ -1,7 +1,10 @@
 const fs = require('fs-extra');
+const path = require('path');
 const utils = require('./utils');
 
+const FileDownloader = require('./FileDownloader');
 const WebSocketConnection = require('./WebSocketConnection');
+const log = new (require('./AppLogger'))().log;//singleton
 
 //singleton
 let instance = null;
@@ -15,9 +18,13 @@ class RemoteLib {
             if (config.remoteLib.accessPassword)
                 this.accessToken = utils.getBufHash(config.remoteLib.accessPassword, 'sha256', 'hex');
 
+            this.remoteHost = config.remoteLib.url.replace(/^ws:\/\//, 'http://').replace(/^wss:\/\//, 'https://');
+
             this.inpxFile = `${config.tempDir}/${utils.randomHexString(20)}`;
             this.lastUpdateTime = 0;
 
+            this.down = new FileDownloader(config.maxPayloadSize*1024*1024);
+
             instance = this;
         }
 
@@ -29,8 +36,8 @@ class RemoteLib {
             query.accessToken = this.accessToken;
 
         const response = await this.wsc.message(
-            await this.wsc.send(query, 60),
+            await this.wsc.send(query),
-            60
+            120
         );
 
         if (response.error)
@@ -39,7 +46,7 @@ class RemoteLib {
         return response;
     }
 
-    async getInpxFile(getPeriod = 0) {
+    async downloadInpxFile(getPeriod = 0) {
         if (getPeriod && Date.now() - this.lastUpdateTime < getPeriod)
             return this.inpxFile;
 
@@ -51,6 +58,23 @@ class RemoteLib {
 
         return this.inpxFile;
     }
+
+    async downloadBook(bookPath, downFileName) {
+        try {
+            const response = await await this.wsRequest({action: 'get-book-link', bookPath, downFileName});
+            const link = response.link;
+
+            const buf = await this.down.load(`${this.remoteHost}${link}`);
+
+            const publicPath = `${this.config.publicDir}${link}`;
+            await fs.writeFile(publicPath, buf);
+
+            return path.basename(link);
+        } catch (e) {
+            log(LM_ERR, `RemoteLib.downloadBook: ${e.message}`);
+            throw new Error('502 Bad Gateway');
+        }
+    }
 }
 
 module.exports = RemoteLib;

server/core/WebWorker.js

@@ -38,6 +38,11 @@ class WebWorker {
         if (!instance) {
             this.config = config;
             this.workerState = new WorkerState();
+
+            this.remoteLib = null;
+            if (config.remoteLib) {
+                this.remoteLib = new RemoteLib(config);
+            }
             
             this.wState = this.workerState.getControl('server_state');
             this.myState = '';
@@ -314,9 +319,16 @@ class WebWorker {
     async restoreBook(bookPath, downFileName) {
         const db = this.db;
 
-        const extractedFile = await this.extractBook(bookPath);
+        let extractedFile = '';
+        let hash = '';
+
+        if (!this.remoteLib) {
+            extractedFile = await this.extractBook(bookPath);
+            hash = await utils.getFileHash(extractedFile, 'sha256', 'hex');
+        } else {
+            hash = await this.remoteLib.downloadBook(bookPath, downFileName);
+        }
 
-        const hash = await utils.getFileHash(extractedFile, 'sha256', 'hex');
         const link = `/files/${hash}`;
         const publicPath = `${this.config.publicDir}${link}`;
 
@@ -328,7 +340,8 @@ class WebWorker {
             await fs.remove(extractedFile);
             await fs.move(tmpFile, publicPath, {overwrite: true});
         } else {
-            await fs.remove(extractedFile);
+            if (extractedFile)
+                await fs.remove(extractedFile);
             await utils.touchFile(publicPath);
         }
 
@@ -506,9 +519,8 @@ class WebWorker {
                 while (this.myState != ssNormal)
                     await utils.sleep(1000);
 
-                if (this.config.remoteLib) {
+                if (this.remoteLib) {
-                    const remoteLib = new RemoteLib(this.config);
+                    await this.remoteLib.downloadInpxFile(60*1000);
-                    await remoteLib.getInpxFile(60*1000);
                 }
 
                 const newInpxHash = await inpxHashCreator.getHash();

server/index.js

@@ -6,13 +6,10 @@ const compression = require('compression');
 const http = require('http');
 const WebSocket = require ('ws');
 
-const RemoteLib = require('./core/RemoteLib');//singleton
 const utils = require('./core/utils');
 
 const ayncExit = new (require('./core/AsyncExit'))();
 
-const maxPayloadSize = 50;//in MB
-
 let log;
 let config;
 let argv;
@@ -111,8 +108,9 @@ async function init() {
             }
         }
     } else {
+        const RemoteLib = require('./core/RemoteLib');//singleton
         const remoteLib = new RemoteLib(config);
-        config.inpxFile = await remoteLib.getInpxFile();
+        config.inpxFile = await remoteLib.downloadInpxFile();
     }
 
     config.recreateDb = argv.recreate || false;
@@ -127,7 +125,7 @@ async function main() {
     const app = express();
 
     const server = http.createServer(app);
-    const wss = new WebSocket.Server({ server, maxPayload: maxPayloadSize*1024*1024 });
+    const wss = new WebSocket.Server({ server, maxPayload: config.maxPayloadSize*1024*1024 });
 
     let devModule = undefined;
     if (branch == 'development') {
@@ -137,7 +135,7 @@ async function main() {
     }
 
     app.use(compression({ level: 1 }));
-    //app.use(express.json({limit: `${maxPayloadSize}mb`}));
+    //app.use(express.json({limit: `${config.maxPayloadSize}mb`}));
     if (devModule)
         devModule.logQueries(app);