From 81629fab7a31189c88d3f586085680a3c6702594 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 13:54:21 +0700 Subject: [PATCH 1/4] =?UTF-8?q?=D0=97=D0=B0=D0=BC=D0=B5=D0=BD=D0=B0=20webd?= =?UTF-8?q?av-fs=20=D0=BD=D0=B0=20webdav?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 10 +---- package.json | 2 +- server/core/Reader/ReaderWorker.js | 4 +- server/core/RemoteWebDavStorage.js | 72 ++++++++++++------------------ 4 files changed, 34 insertions(+), 54 deletions(-) diff --git a/package-lock.json b/package-lock.json index b6dcfe3e..f4d149a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.8.1", + "version": "0.8.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -12807,14 +12807,6 @@ } } }, - "webdav-fs": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/webdav-fs/-/webdav-fs-2.0.0.tgz", - "integrity": "sha512-TjqQKNnf1NuPiMEFJVrmWUYpIEgUxWRktddu5JhQsyxFIOyAJT0cpLyaoOdAp/yrG1yXgDsa6ZL9z+h4Z71zWA==", - "requires": { - "webdav": "^2.0.0" - } - }, "webpack": { "version": "4.40.2", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.40.2.tgz", diff --git a/package.json b/package.json index 99e0e533..3be87510 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "vue-router": "^3.1.3", "vuex": "^3.1.1", "vuex-persistedstate": "^2.5.4", - "webdav-fs": "^2.0.0", + "webdav": "^2.10.1", "zip-stream": "^2.1.2" } } diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index d2b6a92f..014a2e76 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -31,7 +31,9 @@ class ReaderWorker { this.remoteWebDavStorage = false; if (config.remoteWebDavStorage) { - this.remoteWebDavStorage = new RemoteWebDavStorage(config.remoteWebDavStorage); + this.remoteWebDavStorage = new RemoteWebDavStorage( + Object.assign({maxContentLength: config.maxUploadFileSize}, config.remoteWebDavStorage) + ); } this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час diff --git a/server/core/RemoteWebDavStorage.js b/server/core/RemoteWebDavStorage.js index e7d0ad83..21d0c107 100644 --- a/server/core/RemoteWebDavStorage.js +++ b/server/core/RemoteWebDavStorage.js @@ -1,62 +1,48 @@ const fs = require('fs-extra'); const path = require('path'); -const WebDavFS = require('webdav-fs'); +const { createClient } = require('webdav'); class RemoteWebDavStorage { constructor(config) { - const opts = Object.assign({}, config); - this.wfs = WebDavFS(config.url, opts); + this.config = Object.assign({}, config); + this.config.maxContentLength = this.config.maxContentLength || 10*1024*1024; + this.wdc = createClient(config.url, this.config); } - stat(filename) { - return new Promise((resolve, reject) => { - this.wfs.stat(filename, function(err, fileStat) { - if (err) - reject(err); - resolve(fileStat); - }); - }); + _convertStat(data) { + return { + isDirectory: function() { + return data.type === "directory"; + }, + isFile: function() { + return data.type === "file"; + }, + mtime: (new Date(data.lastmod)).getTime(), + name: data.basename, + size: data.size || 0 + }; } - writeFile(filename, data) { - return new Promise((resolve, reject) => { - this.wfs.writeFile(filename, data, 'binary', function(err) { - if (err) - reject(err); - resolve(); - }); - }); + async stat(filename) { + const stat = await this.wdc.stat(filename); + return this._convertStat(stat); } - unlink(filename) { - return new Promise((resolve, reject) => { - this.wfs.unlink(filename, function(err) { - if (err) - reject(err); - resolve(); - }); - }); + async writeFile(filename, data) { + return await this.wdc.putFileContents(filename, data, { maxContentLength: this.config.maxContentLength }) } - readFile(filename) { - return new Promise((resolve, reject) => { - this.wfs.readFile(filename, 'binary', function(err, data) { - if (err) - reject(err); - resolve(data); - }); - }); + async unlink(filename) { + return await this.wdc.deleteFile(filename); } - mkdir(dirname) { - return new Promise((resolve, reject) => { - this.wfs.mkdir(dirname, function(err) { - if (err) - reject(err); - resolve(); - }); - }); + async readFile(filename) { + return await this.wdc.getFileContents(filename, { maxContentLength: this.config.maxContentLength }) + } + + async mkdir(dirname) { + return await this.wdc.createDirectory(dirname); } async putFile(filename) { From 5647e8219d49fcb01d21f01d58e5c7e8b33f358b Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 14:58:42 +0700 Subject: [PATCH 2/4] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B8=D0=B9=20?= =?UTF-8?q?=D1=80=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE=D1=80=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/ReaderWorker.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 014a2e76..bfffb716 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -10,6 +10,8 @@ const RemoteWebDavStorage = require('../RemoteWebDavStorage'); const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton +const cleanDirPeriod = 60*60*1000;//1 раз в час + let instance = null; //singleton @@ -36,8 +38,8 @@ class ReaderWorker { ); } - this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час - this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, 60*60*1000);//1 раз в час + this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, cleanDirPeriod); + this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, cleanDirPeriod); instance = this; } From 7cc63fe849ee66669cbdf6050cb93a4768a7f1d6 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 15:53:23 +0700 Subject: [PATCH 3/4] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=B0=D0=B2=D1=82=D0=BE=D0=BC=D0=B0=D1=82=D0=B8?= =?UTF-8?q?=D1=87=D0=B5=D1=81=D0=BA=D0=B0=D1=8F=20=D0=BE=D1=82=D0=BF=D1=80?= =?UTF-8?q?=D0=B0=D0=B2=D0=BA=D0=B0=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83=D0=B6?= =?UTF-8?q?=D0=B5=D0=BD=D0=BD=D0=BE=D0=B9=20=D0=BA=D0=BD=D0=B8=D0=B3=D0=B8?= =?UTF-8?q?=20=D1=83=D0=B4=D0=B0=D0=BB=D0=B5=D0=BD=D0=BD=D0=BE=D0=B5=20?= =?UTF-8?q?=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/ReaderWorker.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index bfffb716..ef24a707 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -105,6 +105,19 @@ class ReaderWorker { const finishFilename = path.basename(compFilename); wState.finish({path: `/tmp/${finishFilename}`, size: stat.size}); + //лениво сохраним compFilename в удаленном хранилище + if (this.remoteWebDavStorage) { + (async() => { + await utils.sleep(20*1000); + try { + //log(`remoteWebDavStorage.putFile ${path.basename(compFilename)}`); + await this.remoteWebDavStorage.putFile(compFilename); + } catch (e) { + log(LM_ERR, e.stack); + } + })(); + } + } catch (e) { log(LM_ERR, e.stack); wState.set({state: 'error', error: e.message}); From 9c7a6c64b0d2c13c9625b2b86f37ee7215abba64 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 16:13:38 +0700 Subject: [PATCH 4/4] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C=D1=88?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/controllers/WorkerController.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/controllers/WorkerController.js b/server/controllers/WorkerController.js index 54d229dc..86554ab4 100644 --- a/server/controllers/WorkerController.js +++ b/server/controllers/WorkerController.js @@ -38,7 +38,7 @@ class WorkerController extends BaseController { }); const splitter = '-- aod2t5hDXU32bUFyqlFE next status --'; - const refreshPause = 300; + const refreshPause = 200; let i = 0; let prevProgress = -1; let prevState = ''; @@ -47,6 +47,10 @@ class WorkerController extends BaseController { state = this.workerState.getState(request.workerId); if (!state) break; + if (i == 0) { + state = Object.assign({dummy: '0'.repeat(1024)}, state); + } + res.write(splitter + JSON.stringify(state)); res.flush();