diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue index 00f8ffc..3fcbf81 100644 --- a/client/components/Search/Search.vue +++ b/client/components/Search/Search.vue @@ -694,9 +694,13 @@ class Search { || makeValidFilenameOrEmpty(at[0]) || makeValidFilenameOrEmpty(at[1]) || downFileName; - downFileName = `${downFileName.substring(0, 100)}.${book.ext}`; + downFileName = downFileName.substring(0, 100); - const bookPath = `${book.folder}/${book.file}.${book.ext}`; + const ext = `.${book.ext}`; + if (downFileName.substring(downFileName.length - ext.length) != ext) + downFileName += ext; + + const bookPath = `${book.folder}/${book.file}${ext}`; //подготовка const response = await this.api.getBookLink({bookPath, downFileName}); diff --git a/server/config/base.js b/server/config/base.js index 59a4744..3882f40 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -14,6 +14,7 @@ module.exports = { bookReadLink: '', loggingEnabled: true, + maxPayloadSize: 500,//in MB maxFilesDirSize: 1024*1024*1024,//1Gb queryCacheEnabled: true, cacheCleanInterval: 60,//minutes diff --git a/server/core/FileDownloader.js b/server/core/FileDownloader.js new file mode 100644 index 0000000..5f86e5a --- /dev/null +++ b/server/core/FileDownloader.js @@ -0,0 +1,125 @@ +const https = require('https'); +const axios = require('axios'); +const utils = require('./utils'); + +const userAgent = 'Mozilla/5.0 (X11; HasCodingOs 1.0; Linux x64) AppleWebKit/637.36 (KHTML, like Gecko) Chrome/70.0.3112.101 Safari/637.36 HasBrowser/5.0'; + +class FileDownloader { + constructor(limitDownloadSize = 0) { + this.limitDownloadSize = limitDownloadSize; + } + + async load(url, callback, abort) { + let errMes = ''; + + const options = { + headers: { + 'user-agent': userAgent, + timeout: 300*1000, + }, + httpsAgent: new https.Agent({ + rejectUnauthorized: false // решение проблемы 'unable to verify the first certificate' для некоторых сайтов с валидным сертификатом + }), + responseType: 'stream', + }; + + try { + const res = await axios.get(url, options); + + let estSize = 0; + if (res.headers['content-length']) { + estSize = res.headers['content-length']; + } + + if (this.limitDownloadSize && estSize > this.limitDownloadSize) { + throw new Error('Файл слишком большой'); + } + + let prevProg = 0; + let transferred = 0; + + const download = this.streamToBuffer(res.data, (chunk) => { + transferred += chunk.length; + if (this.limitDownloadSize) { + if (transferred > this.limitDownloadSize) { + errMes = 'Файл слишком большой'; + res.request.abort(); + } + } + + let prog = 0; + if (estSize) + prog = Math.round(transferred/estSize*100); + else + prog = Math.round(transferred/(transferred + 200000)*100); + + if (prog != prevProg && callback) + callback(prog); + prevProg = prog; + + if (abort && abort()) { + errMes = 'abort'; + res.request.abort(); + } + }); + + return await download; + } catch (error) { + errMes = (errMes ? errMes : error.message); + throw new Error(errMes); + } + } + + async head(url) { + const options = { + headers: { + 'user-agent': userAgent, + timeout: 10*1000, + }, + }; + + const res = await axios.head(url, options); + return res.headers; + } + + streamToBuffer(stream, progress, timeout = 30*1000) { + return new Promise((resolve, reject) => { + + if (!progress) + progress = () => {}; + + const _buf = []; + let resolved = false; + let timer = 0; + + stream.on('data', (chunk) => { + timer = 0; + _buf.push(chunk); + progress(chunk); + }); + stream.on('end', () => { + resolved = true; + timer = timeout; + resolve(Buffer.concat(_buf)); + }); + stream.on('error', (err) => { + reject(err); + }); + stream.on('aborted', () => { + reject(new Error('aborted')); + }); + + //бодяга с timer и timeout, чтобы гарантировать отсутствие зависания по каким-либо причинам + (async() => { + while (timer < timeout) { + await utils.sleep(1000); + timer += 1000; + } + if (!resolved) + reject(new Error('FileDownloader: timed out')) + })(); + }); + } +} + +module.exports = FileDownloader; diff --git a/server/core/RemoteLib.js b/server/core/RemoteLib.js index 6675b45..6369d37 100644 --- a/server/core/RemoteLib.js +++ b/server/core/RemoteLib.js @@ -1,7 +1,10 @@ const fs = require('fs-extra'); +const path = require('path'); const utils = require('./utils'); +const FileDownloader = require('./FileDownloader'); const WebSocketConnection = require('./WebSocketConnection'); +const log = new (require('./AppLogger'))().log;//singleton //singleton let instance = null; @@ -15,9 +18,13 @@ class RemoteLib { if (config.remoteLib.accessPassword) this.accessToken = utils.getBufHash(config.remoteLib.accessPassword, 'sha256', 'hex'); + this.remoteHost = config.remoteLib.url.replace(/^ws:\/\//, 'http://').replace(/^wss:\/\//, 'https://'); + this.inpxFile = `${config.tempDir}/${utils.randomHexString(20)}`; this.lastUpdateTime = 0; + this.down = new FileDownloader(config.maxPayloadSize*1024*1024); + instance = this; } @@ -29,8 +36,8 @@ class RemoteLib { query.accessToken = this.accessToken; const response = await this.wsc.message( - await this.wsc.send(query, 60), - 60 + await this.wsc.send(query), + 120 ); if (response.error) @@ -39,7 +46,7 @@ class RemoteLib { return response; } - async getInpxFile(getPeriod = 0) { + async downloadInpxFile(getPeriod = 0) { if (getPeriod && Date.now() - this.lastUpdateTime < getPeriod) return this.inpxFile; @@ -51,6 +58,23 @@ class RemoteLib { return this.inpxFile; } + + async downloadBook(bookPath, downFileName) { + try { + const response = await await this.wsRequest({action: 'get-book-link', bookPath, downFileName}); + const link = response.link; + + const buf = await this.down.load(`${this.remoteHost}${link}`); + + const publicPath = `${this.config.publicDir}${link}`; + await fs.writeFile(publicPath, buf); + + return path.basename(link); + } catch (e) { + log(LM_ERR, `RemoteLib.downloadBook: ${e.message}`); + throw new Error('502 Bad Gateway'); + } + } } module.exports = RemoteLib; \ No newline at end of file diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index 3ece346..2cb8b4c 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -38,6 +38,11 @@ class WebWorker { if (!instance) { this.config = config; this.workerState = new WorkerState(); + + this.remoteLib = null; + if (config.remoteLib) { + this.remoteLib = new RemoteLib(config); + } this.wState = this.workerState.getControl('server_state'); this.myState = ''; @@ -314,9 +319,16 @@ class WebWorker { async restoreBook(bookPath, downFileName) { const db = this.db; - const extractedFile = await this.extractBook(bookPath); + let extractedFile = ''; + let hash = ''; + + if (!this.remoteLib) { + extractedFile = await this.extractBook(bookPath); + hash = await utils.getFileHash(extractedFile, 'sha256', 'hex'); + } else { + hash = await this.remoteLib.downloadBook(bookPath, downFileName); + } - const hash = await utils.getFileHash(extractedFile, 'sha256', 'hex'); const link = `/files/${hash}`; const publicPath = `${this.config.publicDir}${link}`; @@ -328,7 +340,8 @@ class WebWorker { await fs.remove(extractedFile); await fs.move(tmpFile, publicPath, {overwrite: true}); } else { - await fs.remove(extractedFile); + if (extractedFile) + await fs.remove(extractedFile); await utils.touchFile(publicPath); } @@ -506,9 +519,8 @@ class WebWorker { while (this.myState != ssNormal) await utils.sleep(1000); - if (this.config.remoteLib) { - const remoteLib = new RemoteLib(this.config); - await remoteLib.getInpxFile(60*1000); + if (this.remoteLib) { + await this.remoteLib.downloadInpxFile(60*1000); } const newInpxHash = await inpxHashCreator.getHash(); diff --git a/server/index.js b/server/index.js index 5f03468..3d83d63 100644 --- a/server/index.js +++ b/server/index.js @@ -6,13 +6,10 @@ const compression = require('compression'); const http = require('http'); const WebSocket = require ('ws'); -const RemoteLib = require('./core/RemoteLib');//singleton const utils = require('./core/utils'); const ayncExit = new (require('./core/AsyncExit'))(); -const maxPayloadSize = 50;//in MB - let log; let config; let argv; @@ -111,8 +108,9 @@ async function init() { } } } else { + const RemoteLib = require('./core/RemoteLib');//singleton const remoteLib = new RemoteLib(config); - config.inpxFile = await remoteLib.getInpxFile(); + config.inpxFile = await remoteLib.downloadInpxFile(); } config.recreateDb = argv.recreate || false; @@ -127,7 +125,7 @@ async function main() { const app = express(); const server = http.createServer(app); - const wss = new WebSocket.Server({ server, maxPayload: maxPayloadSize*1024*1024 }); + const wss = new WebSocket.Server({ server, maxPayload: config.maxPayloadSize*1024*1024 }); let devModule = undefined; if (branch == 'development') { @@ -137,7 +135,7 @@ async function main() { } app.use(compression({ level: 1 })); - //app.use(express.json({limit: `${maxPayloadSize}mb`})); + //app.use(express.json({limit: `${config.maxPayloadSize}mb`})); if (devModule) devModule.logQueries(app);