From 000f8dde82716c6ed0318e6f5fc23e65df8d9c6b Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 17 Jul 2022 15:43:12 +0700 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D1=85=D0=BE=D0=B4=20?= =?UTF-8?q?=D0=BD=D0=B0=20RemoteStorage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/config/base.js | 9 ++- server/core/Reader/ReaderWorker.js | 26 ++++---- server/core/RemoteStorage.js | 97 ++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 13 deletions(-) create mode 100644 server/core/RemoteStorage.js diff --git a/server/config/base.js b/server/config/base.js index 86457c11..7760a3ce 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -55,8 +55,8 @@ module.exports = { }, ], - remoteWebDavStorage: false, /* + remoteWebDavStorage: false, remoteWebDavStorage: { url: '127.0.0.1:1900', username: '', @@ -64,5 +64,12 @@ module.exports = { }, */ + remoteStorage: false, + /* + remoteStorage: { + url: 'https://127.0.0.1:11900', + accessToken: '', + }, + */ }; diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 1ece6d92..16c79a89 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -6,7 +6,7 @@ const WorkerState = require('../WorkerState');//singleton const FileDownloader = require('../FileDownloader'); const FileDecompressor = require('../FileDecompressor'); const BookConverter = require('./BookConverter'); -const RemoteWebDavStorage = require('../RemoteWebDavStorage'); +const RemoteStorage = require('../RemoteStorage'); const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton @@ -33,10 +33,10 @@ class ReaderWorker { this.decomp = new FileDecompressor(3*config.maxUploadFileSize); this.bookConverter = new BookConverter(this.config); - this.remoteWebDavStorage = false; - if (config.remoteWebDavStorage) { - this.remoteWebDavStorage = new RemoteWebDavStorage( - Object.assign({maxContentLength: 3*config.maxUploadFileSize}, config.remoteWebDavStorage) + this.remoteStorage = false; + if (config.remoteStorage) { + this.remoteStorage = new RemoteStorage( + Object.assign({maxContentLength: 3*config.maxUploadFileSize}, config.remoteStorage) ); } @@ -235,8 +235,8 @@ class ReaderWorker { if (!await fs.pathExists(targetName)) { let found = false; - if (this.remoteWebDavStorage) { - found = await this.remoteWebDavStorage.getFileSuccess(targetName, remoteDir); + if (this.remoteStorage) { + found = await this.remoteStorage.getFileSuccess(targetName, remoteDir); } if (!found) { @@ -271,15 +271,15 @@ class ReaderWorker { files.sort((a, b) => a.stat.mtimeMs - b.stat.mtimeMs); - if (moveToRemote && this.remoteWebDavStorage) { + if (moveToRemote && this.remoteStorage) { for (const file of files) { if (sent[file.name]) continue; - //отправляем в remoteWebDavStorage + //отправляем в remoteStorage try { - log(`remoteWebDavStorage.putFile ${remoteDir}/${path.basename(file.name)}`); - await this.remoteWebDavStorage.putFile(file.name, remoteDir); + log(`remoteStorage.putFile ${remoteDir}/${path.basename(file.name)}`); + await this.remoteStorage.putFile(file.name, remoteDir); sent[file.name] = true; } catch (e) { log(LM_ERR, e.stack); @@ -294,7 +294,9 @@ class ReaderWorker { const oldFile = file.name; //реально удаляем только если сохранили в хранилище или размер dir увеличен в 1.5 раза - if ((moveToRemote && this.remoteWebDavStorage && sent[oldFile]) || size > maxSize*1.5) { + if (!(moveToRemote && this.remoteStorage) + || (moveToRemote && this.remoteStorage && sent[oldFile]) + || size > maxSize*1.5) { await fs.remove(oldFile); delete sent[oldFile]; j++; diff --git a/server/core/RemoteStorage.js b/server/core/RemoteStorage.js new file mode 100644 index 00000000..d3a6f4bd --- /dev/null +++ b/server/core/RemoteStorage.js @@ -0,0 +1,97 @@ +const fs = require('fs-extra'); +const path = require('path'); + +const WebSocketConnection = require('./WebSocketConnection'); + +class RemoteStorage { + constructor(config) { + this.config = Object.assign({}, config); + this.config.maxContentLength = this.config.maxContentLength || 10*1024*1024; + + this.accessToken = this.config.accessToken; + + this.wsc = new WebSocketConnection(config.url); + } + + async wsQuery(query) { + const response = await this.wsc.message( + await this.wsc.send(Object.assign({accessToken: this.accessToken}, query)) + ); + if (response.error) + throw new Error(response.error); + return response; + } + + async wsStat(fileName) { + return await this.wsQuery({action: 'get-stat', fileName}); + } + + async wsGetFile(fileName) { + return this.wsQuery({action: 'get-file', fileName}); + } + + async wsPutFile(fileName, data) {//data base64 encoded string + return this.wsQuery({action: 'put-file', fileName, data}); + } + + async wsDelFile(fileName) { + return this.wsQuery({action: 'del-file', fileName}); + } + + makeRemoteFileName(fileName, dir = '') { + const base = path.basename(fileName); + if (base.length > 3) { + return `${dir}/${base.substr(0, 3)}/${base}`; + } else { + return `${dir}/${base}`; + } + } + + async putFile(fileName, dir = '') { + if (!await fs.pathExists(fileName)) { + throw new Error(`File not found: ${fileName}`); + } + + const remoteFilename = this.makeRemoteFileName(fileName, dir); + + try { + const localStat = await fs.stat(fileName); + let remoteStat = await this.wsStat(remoteFilename); + remoteStat = remoteStat.stat; + + if (remoteStat.isFile && localStat.size == remoteStat.size) { + return; + } + + await this.wsDelFile(remoteFilename); + } catch (e) { + // + } + + const data = await fs.readFile(fileName, 'base64'); + await this.wsPutFile(remoteFilename, data); + } + + async getFile(fileName, dir = '') { + if (await fs.pathExists(fileName)) { + return; + } + + const remoteFilename = this.makeRemoteFileName(fileName, dir); + + const response = await this.wsGetFile(remoteFilename); + await fs.writeFile(fileName, response.data, 'base64'); + } + + async getFileSuccess(filename, dir = '') { + try { + await this.getFile(filename, dir); + return true; + } catch (e) { + // + } + return false; + } +} + +module.exports = RemoteStorage; \ No newline at end of file