Работа над RemoteLib

2022-10-07 19:06:00 +07:00
parent e6dfe4d221
commit a82392197f
6 changed files with 181 additions and 17 deletions
--- a/client/components/Search/Search.vue
+++ b/client/components/Search/Search.vue
@@ -694,9 +694,13 @@ class Search {
                || makeValidFilenameOrEmpty(at[0])
                || makeValidFilenameOrEmpty(at[1])
                || downFileName;
-            downFileName = `${downFileName.substring(0, 100)}.${book.ext}`;
+            downFileName = downFileName.substring(0, 100);

-            const bookPath = `${book.folder}/${book.file}.${book.ext}`;
+            const ext = `.${book.ext}`;
+            if (downFileName.substring(downFileName.length - ext.length) != ext)
+                downFileName += ext;
+
+            const bookPath = `${book.folder}/${book.file}${ext}`;
            //подготовка
            const response = await this.api.getBookLink({bookPath, downFileName});
            
--- a/server/config/base.js
+++ b/server/config/base.js
@@ -14,6 +14,7 @@ module.exports = {
    bookReadLink: '',
    loggingEnabled: true,

+    maxPayloadSize: 500,//in MB
    maxFilesDirSize: 1024*1024*1024,//1Gb
    queryCacheEnabled: true,
    cacheCleanInterval: 60,//minutes
--- a/server/core/FileDownloader.js
+++ b/server/core/FileDownloader.js
@@ -0,0 +1,125 @@
+const https = require('https');
+const axios = require('axios');
+const utils = require('./utils');
+
+const userAgent = 'Mozilla/5.0 (X11; HasCodingOs 1.0; Linux x64) AppleWebKit/637.36 (KHTML, like Gecko) Chrome/70.0.3112.101 Safari/637.36 HasBrowser/5.0';
+
+class FileDownloader {
+    constructor(limitDownloadSize = 0) {
+        this.limitDownloadSize = limitDownloadSize;
+    }
+
+    async load(url, callback, abort) {
+        let errMes = '';
+
+        const options = {
+            headers: {
+                'user-agent': userAgent,
+                timeout: 300*1000,
+            },
+            httpsAgent: new https.Agent({
+                rejectUnauthorized: false // решение проблемы 'unable to verify the first certificate' для некоторых сайтов с валидным сертификатом
+            }),
+            responseType: 'stream',
+        };
+
+        try {
+            const res = await axios.get(url, options);
+
+            let estSize = 0;
+            if (res.headers['content-length']) {
+                estSize = res.headers['content-length'];
+            }
+
+            if (this.limitDownloadSize && estSize > this.limitDownloadSize) {
+                throw new Error('Файл слишком большой');
+            }
+
+            let prevProg = 0;
+            let transferred = 0;
+
+            const download = this.streamToBuffer(res.data, (chunk) => {
+                transferred += chunk.length;
+                if (this.limitDownloadSize) {
+                    if (transferred > this.limitDownloadSize) {
+                        errMes = 'Файл слишком большой';
+                        res.request.abort();
+                    }
+                }
+
+                let prog = 0;
+                if (estSize)
+                    prog = Math.round(transferred/estSize*100);
+                else
+                    prog = Math.round(transferred/(transferred + 200000)*100);
+
+                if (prog != prevProg && callback)
+                    callback(prog);
+                prevProg = prog;
+
+                if (abort && abort()) {
+                    errMes = 'abort';
+                    res.request.abort();
+                }
+            });
+
+            return await download;
+        } catch (error) {
+            errMes = (errMes ? errMes : error.message);
+            throw new Error(errMes);
+        }
+    }
+
+    async head(url) {
+        const options = {
+            headers: {
+                'user-agent': userAgent,
+                timeout: 10*1000,
+            },
+        };
+
+        const res = await axios.head(url, options);
+        return res.headers;
+    }
+
+    streamToBuffer(stream, progress, timeout = 30*1000) {
+        return new Promise((resolve, reject) => {
+            
+            if (!progress)
+                progress = () => {};
+
+            const _buf = [];
+            let resolved = false;
+            let timer = 0;
+
+            stream.on('data', (chunk) => {
+                timer = 0;
+                _buf.push(chunk);
+                progress(chunk);
+            });
+            stream.on('end', () => {
+                resolved = true;
+                timer = timeout;
+                resolve(Buffer.concat(_buf));
+            });
+            stream.on('error', (err) => {
+                reject(err);
+            });
+            stream.on('aborted', () => {
+                reject(new Error('aborted'));
+            });
+
+            //бодяга с timer и timeout, чтобы гарантировать отсутствие зависания по каким-либо причинам
+            (async() => {
+                while (timer < timeout) {
+                    await utils.sleep(1000);
+                    timer += 1000;
+                }
+                if (!resolved)
+                    reject(new Error('FileDownloader: timed out'))
+            })();
+        });
+    }
+}
+
+module.exports = FileDownloader;
--- a/server/core/RemoteLib.js
+++ b/server/core/RemoteLib.js
@@ -1,7 +1,10 @@
 const fs = require('fs-extra');
+const path = require('path');
 const utils = require('./utils');

+const FileDownloader = require('./FileDownloader');
 const WebSocketConnection = require('./WebSocketConnection');
+const log = new (require('./AppLogger'))().log;//singleton

 //singleton
 let instance = null;
@@ -15,9 +18,13 @@ class RemoteLib {
            if (config.remoteLib.accessPassword)
                this.accessToken = utils.getBufHash(config.remoteLib.accessPassword, 'sha256', 'hex');

+            this.remoteHost = config.remoteLib.url.replace(/^ws:\/\//, 'http://').replace(/^wss:\/\//, 'https://');
+
            this.inpxFile = `${config.tempDir}/${utils.randomHexString(20)}`;
            this.lastUpdateTime = 0;

+            this.down = new FileDownloader(config.maxPayloadSize*1024*1024);
+
            instance = this;
        }

@@ -29,8 +36,8 @@ class RemoteLib {
            query.accessToken = this.accessToken;

        const response = await this.wsc.message(
-            await this.wsc.send(query, 60),
-            60
+            await this.wsc.send(query),
+            120
        );

        if (response.error)
@@ -39,7 +46,7 @@ class RemoteLib {
        return response;
    }

-    async getInpxFile(getPeriod = 0) {
+    async downloadInpxFile(getPeriod = 0) {
        if (getPeriod && Date.now() - this.lastUpdateTime < getPeriod)
            return this.inpxFile;

@@ -51,6 +58,23 @@ class RemoteLib {

        return this.inpxFile;
    }
+
+    async downloadBook(bookPath, downFileName) {
+        try {
+            const response = await await this.wsRequest({action: 'get-book-link', bookPath, downFileName});
+            const link = response.link;
+
+            const buf = await this.down.load(`${this.remoteHost}${link}`);
+
+            const publicPath = `${this.config.publicDir}${link}`;
+            await fs.writeFile(publicPath, buf);
+
+            return path.basename(link);
+        } catch (e) {
+            log(LM_ERR, `RemoteLib.downloadBook: ${e.message}`);
+            throw new Error('502 Bad Gateway');
+        }
+    }
 }

 module.exports = RemoteLib;
--- a/server/core/WebWorker.js
+++ b/server/core/WebWorker.js
@@ -39,6 +39,11 @@ class WebWorker {
            this.config = config;
            this.workerState = new WorkerState();

+            this.remoteLib = null;
+            if (config.remoteLib) {
+                this.remoteLib = new RemoteLib(config);
+            }
+            
            this.wState = this.workerState.getControl('server_state');
            this.myState = '';
            this.db = null;
@@ -314,9 +319,16 @@ class WebWorker {
    async restoreBook(bookPath, downFileName) {
        const db = this.db;

-        const extractedFile = await this.extractBook(bookPath);
+        let extractedFile = '';
+        let hash = '';
+
+        if (!this.remoteLib) {
+            extractedFile = await this.extractBook(bookPath);
+            hash = await utils.getFileHash(extractedFile, 'sha256', 'hex');
+        } else {
+            hash = await this.remoteLib.downloadBook(bookPath, downFileName);
+        }

-        const hash = await utils.getFileHash(extractedFile, 'sha256', 'hex');
        const link = `/files/${hash}`;
        const publicPath = `${this.config.publicDir}${link}`;

@@ -328,6 +340,7 @@ class WebWorker {
            await fs.remove(extractedFile);
            await fs.move(tmpFile, publicPath, {overwrite: true});
        } else {
+            if (extractedFile)
                await fs.remove(extractedFile);
            await utils.touchFile(publicPath);
        }
@@ -506,9 +519,8 @@ class WebWorker {
                while (this.myState != ssNormal)
                    await utils.sleep(1000);

-                if (this.config.remoteLib) {
-                    const remoteLib = new RemoteLib(this.config);
-                    await remoteLib.getInpxFile(60*1000);
+                if (this.remoteLib) {
+                    await this.remoteLib.downloadInpxFile(60*1000);
                }

                const newInpxHash = await inpxHashCreator.getHash();
--- a/server/index.js
+++ b/server/index.js
@@ -6,13 +6,10 @@ const compression = require('compression');
 const http = require('http');
 const WebSocket = require ('ws');

-const RemoteLib = require('./core/RemoteLib');//singleton
 const utils = require('./core/utils');

 const ayncExit = new (require('./core/AsyncExit'))();

-const maxPayloadSize = 50;//in MB
-
 let log;
 let config;
 let argv;
@@ -111,8 +108,9 @@ async function init() {
            }
        }
    } else {
+        const RemoteLib = require('./core/RemoteLib');//singleton
        const remoteLib = new RemoteLib(config);
-        config.inpxFile = await remoteLib.getInpxFile();
+        config.inpxFile = await remoteLib.downloadInpxFile();
    }

    config.recreateDb = argv.recreate || false;
@@ -127,7 +125,7 @@ async function main() {
    const app = express();

    const server = http.createServer(app);
-    const wss = new WebSocket.Server({ server, maxPayload: maxPayloadSize*1024*1024 });
+    const wss = new WebSocket.Server({ server, maxPayload: config.maxPayloadSize*1024*1024 });

    let devModule = undefined;
    if (branch == 'development') {
@@ -137,7 +135,7 @@ async function main() {
    }

    app.use(compression({ level: 1 }));
-    //app.use(express.json({limit: `${maxPayloadSize}mb`}));
+    //app.use(express.json({limit: `${config.maxPayloadSize}mb`}));
    if (devModule)
        devModule.logQueries(app);