diff --git a/docs/beta/beta.liberama b/docs/beta/beta.liberama index a1e4027f..9268bb9f 100644 --- a/docs/beta/beta.liberama +++ b/docs/beta/beta.liberama @@ -6,6 +6,7 @@ server { ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot server_name beta.liberama.top; + set $liberama http://127.0.0.1:34082; client_max_body_size 50m; proxy_read_timeout 1h; @@ -15,15 +16,20 @@ server { gzip_proxied expired no-cache no-store private auth; gzip_types *; + location @liberama { + proxy_pass $liberama; + } + location /api { - proxy_pass http://127.0.0.1:34082; + proxy_pass $liberama; } location /ws { - proxy_pass http://127.0.0.1:34082; + proxy_pass $liberama; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; + proxy_read_timeout 600s; } location / { @@ -32,6 +38,11 @@ server { location /tmp { types { } default_type "application/xml; charset=utf-8"; add_header Content-Encoding gzip; + try_files $uri @liberama; + } + + location /upload { + try_files $uri @liberama; } location ~* \.(?:manifest|appcache|html)$ { @@ -50,6 +61,7 @@ server { server { listen 80; server_name b.beta.liberama.top; + set $liberama http://127.0.0.1:34082; client_max_body_size 50m; proxy_read_timeout 1h; @@ -59,15 +71,20 @@ server { gzip_proxied expired no-cache no-store private auth; gzip_types *; + location @liberama { + proxy_pass $liberama; + } + location /api { - proxy_pass http://127.0.0.1:34082; + proxy_pass $liberama; } location /ws { - proxy_pass http://127.0.0.1:34082; + proxy_pass $liberama; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; + proxy_read_timeout 600s; } location / { @@ -76,6 +93,11 @@ server { location /tmp { types { } default_type "application/xml; charset=utf-8"; add_header Content-Encoding gzip; + try_files $uri @liberama; + } + + location /upload { + try_files $uri @liberama; } location ~* \.(?:manifest|appcache|html)$ { diff --git a/docs/beta/beta.omnireader b/docs/beta/beta.omnireader index 97976294..81f02114 100644 --- a/docs/beta/beta.omnireader +++ b/docs/beta/beta.omnireader @@ -6,6 +6,7 @@ server { ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot server_name beta.omnireader.ru; + set $liberama http://127.0.0.1:34081; client_max_body_size 50m; proxy_read_timeout 1h; @@ -15,15 +16,20 @@ server { gzip_proxied expired no-cache no-store private auth; gzip_types *; + location @liberama { + proxy_pass $liberama; + } + location /api { - proxy_pass http://127.0.0.1:34081; + proxy_pass $liberama; } location /ws { - proxy_pass http://127.0.0.1:34081; + proxy_pass $liberama; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; + proxy_read_timeout 600s; } location / { @@ -32,6 +38,11 @@ server { location /tmp { types { } default_type "application/xml; charset=utf-8"; add_header Content-Encoding gzip; + try_files $uri @liberama; + } + + location /upload { + try_files $uri @liberama; } location ~* \.(?:manifest|appcache|html)$ { diff --git a/docs/beta/beta.omnireader_http b/docs/beta/beta.omnireader_http index dd467283..df27997b 100644 --- a/docs/beta/beta.omnireader_http +++ b/docs/beta/beta.omnireader_http @@ -1,6 +1,7 @@ server { listen 80; server_name beta.omnireader.ru; + set $liberama http://127.0.0.1:34081; client_max_body_size 50m; proxy_read_timeout 1h; @@ -10,15 +11,20 @@ server { gzip_proxied expired no-cache no-store private auth; gzip_types *; + location @liberama { + proxy_pass $liberama; + } + location /api { - proxy_pass http://127.0.0.1:34081; + proxy_pass $liberama; } location /ws { - proxy_pass http://127.0.0.1:34081; + proxy_pass $liberama; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; + proxy_read_timeout 600s; } location / { @@ -27,6 +33,11 @@ server { location /tmp { types { } default_type "application/xml; charset=utf-8"; add_header Content-Encoding gzip; + try_files $uri @liberama; + } + + location /upload { + try_files $uri @liberama; } location ~* \.(?:manifest|appcache|html)$ { diff --git a/server/config/base.js b/server/config/base.js index 86457c11..7760a3ce 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -55,8 +55,8 @@ module.exports = { }, ], - remoteWebDavStorage: false, /* + remoteWebDavStorage: false, remoteWebDavStorage: { url: '127.0.0.1:1900', username: '', @@ -64,5 +64,12 @@ module.exports = { }, */ + remoteStorage: false, + /* + remoteStorage: { + url: 'https://127.0.0.1:11900', + accessToken: '', + }, + */ }; diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 6672e6ce..16c79a89 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -6,7 +6,7 @@ const WorkerState = require('../WorkerState');//singleton const FileDownloader = require('../FileDownloader'); const FileDecompressor = require('../FileDecompressor'); const BookConverter = require('./BookConverter'); -const RemoteWebDavStorage = require('../RemoteWebDavStorage'); +const RemoteStorage = require('../RemoteStorage'); const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton @@ -33,10 +33,10 @@ class ReaderWorker { this.decomp = new FileDecompressor(3*config.maxUploadFileSize); this.bookConverter = new BookConverter(this.config); - this.remoteWebDavStorage = false; - if (config.remoteWebDavStorage) { - this.remoteWebDavStorage = new RemoteWebDavStorage( - Object.assign({maxContentLength: 3*config.maxUploadFileSize}, config.remoteWebDavStorage) + this.remoteStorage = false; + if (config.remoteStorage) { + this.remoteStorage = new RemoteStorage( + Object.assign({maxContentLength: 3*config.maxUploadFileSize}, config.remoteStorage) ); } @@ -235,8 +235,8 @@ class ReaderWorker { if (!await fs.pathExists(targetName)) { let found = false; - if (this.remoteWebDavStorage) { - found = await this.remoteWebDavStorage.getFileSuccess(targetName, remoteDir); + if (this.remoteStorage) { + found = await this.remoteStorage.getFileSuccess(targetName, remoteDir); } if (!found) { @@ -271,15 +271,15 @@ class ReaderWorker { files.sort((a, b) => a.stat.mtimeMs - b.stat.mtimeMs); - if (moveToRemote && this.remoteWebDavStorage) { + if (moveToRemote && this.remoteStorage) { for (const file of files) { if (sent[file.name]) continue; - //отправляем в remoteWebDavStorage + //отправляем в remoteStorage try { - log(`remoteWebDavStorage.putFile ${remoteDir}/${path.basename(file.name)}`); - await this.remoteWebDavStorage.putFile(file.name, remoteDir); + log(`remoteStorage.putFile ${remoteDir}/${path.basename(file.name)}`); + await this.remoteStorage.putFile(file.name, remoteDir); sent[file.name] = true; } catch (e) { log(LM_ERR, e.stack); @@ -294,8 +294,11 @@ class ReaderWorker { const oldFile = file.name; //реально удаляем только если сохранили в хранилище или размер dir увеличен в 1.5 раза - if ((moveToRemote && this.remoteWebDavStorage && sent[oldFile]) || size > maxSize*1.5) { + if (!(moveToRemote && this.remoteStorage) + || (moveToRemote && this.remoteStorage && sent[oldFile]) + || size > maxSize*1.5) { await fs.remove(oldFile); + delete sent[oldFile]; j++; } diff --git a/server/core/RemoteStorage.js b/server/core/RemoteStorage.js new file mode 100644 index 00000000..d3a6f4bd --- /dev/null +++ b/server/core/RemoteStorage.js @@ -0,0 +1,97 @@ +const fs = require('fs-extra'); +const path = require('path'); + +const WebSocketConnection = require('./WebSocketConnection'); + +class RemoteStorage { + constructor(config) { + this.config = Object.assign({}, config); + this.config.maxContentLength = this.config.maxContentLength || 10*1024*1024; + + this.accessToken = this.config.accessToken; + + this.wsc = new WebSocketConnection(config.url); + } + + async wsQuery(query) { + const response = await this.wsc.message( + await this.wsc.send(Object.assign({accessToken: this.accessToken}, query)) + ); + if (response.error) + throw new Error(response.error); + return response; + } + + async wsStat(fileName) { + return await this.wsQuery({action: 'get-stat', fileName}); + } + + async wsGetFile(fileName) { + return this.wsQuery({action: 'get-file', fileName}); + } + + async wsPutFile(fileName, data) {//data base64 encoded string + return this.wsQuery({action: 'put-file', fileName, data}); + } + + async wsDelFile(fileName) { + return this.wsQuery({action: 'del-file', fileName}); + } + + makeRemoteFileName(fileName, dir = '') { + const base = path.basename(fileName); + if (base.length > 3) { + return `${dir}/${base.substr(0, 3)}/${base}`; + } else { + return `${dir}/${base}`; + } + } + + async putFile(fileName, dir = '') { + if (!await fs.pathExists(fileName)) { + throw new Error(`File not found: ${fileName}`); + } + + const remoteFilename = this.makeRemoteFileName(fileName, dir); + + try { + const localStat = await fs.stat(fileName); + let remoteStat = await this.wsStat(remoteFilename); + remoteStat = remoteStat.stat; + + if (remoteStat.isFile && localStat.size == remoteStat.size) { + return; + } + + await this.wsDelFile(remoteFilename); + } catch (e) { + // + } + + const data = await fs.readFile(fileName, 'base64'); + await this.wsPutFile(remoteFilename, data); + } + + async getFile(fileName, dir = '') { + if (await fs.pathExists(fileName)) { + return; + } + + const remoteFilename = this.makeRemoteFileName(fileName, dir); + + const response = await this.wsGetFile(remoteFilename); + await fs.writeFile(fileName, response.data, 'base64'); + } + + async getFileSuccess(filename, dir = '') { + try { + await this.getFile(filename, dir); + return true; + } catch (e) { + // + } + return false; + } +} + +module.exports = RemoteStorage; \ No newline at end of file