From dbb9bd1282933c45cde44b0c4fc1c282874d4aed Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 3 Mar 2020 18:06:46 +0700 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=BE=20=D1=81=D0=BE=D1=85=D1=80=D0=B0=D0=BD=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=B2=D0=B0=D0=BB=D0=B8=D0=B4=D0=BD=D1=8B=D1=85?= =?UTF-8?q?=20uploaded-=D1=84=D0=B0=D0=B9=D0=BB=D0=BE=D0=B2=20=D0=B2=20?= =?UTF-8?q?=D1=83=D0=B4=D0=B0=D0=BB=D0=B5=D0=BD=D0=BD=D0=BE=D0=BC=20=D1=85?= =?UTF-8?q?=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/FileDecompressor.js | 26 ++++++------ server/core/Reader/ReaderWorker.js | 64 ++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/server/core/FileDecompressor.js b/server/core/FileDecompressor.js index 8a601744..382d4105 100644 --- a/server/core/FileDecompressor.js +++ b/server/core/FileDecompressor.js @@ -241,27 +241,29 @@ class FileDecompressor { }); } - async gzipFileIfNotExists(filename, outDir) { + async gzipFileIfNotExists(filename, outDir, isMaxCompression) { const hash = await utils.getFileHash(filename, 'sha256', 'hex'); const outFilename = `${outDir}/${hash}`; if (!await fs.pathExists(outFilename)) { - await this.gzipFile(filename, outFilename, 1); + await this.gzipFile(filename, outFilename, (isMaxCompression ? 9 : 1)); - // переупакуем через некоторое время на максималках - const filenameCopy = `${filename}.copy`; - await fs.copy(filename, filenameCopy); + // переупакуем через некоторое время на максималках, если упаковали плохо + if (!isMaxCompression) { + const filenameCopy = `${filename}.copy`; + await fs.copy(filename, filenameCopy); - (async() => { - await utils.sleep(5000); - const filenameGZ = `${filename}.gz`; - await this.gzipFile(filenameCopy, filenameGZ, 9); + (async() => { + await utils.sleep(5000); + const filenameGZ = `${filename}.gz`; + await this.gzipFile(filenameCopy, filenameGZ, 9); - await fs.move(filenameGZ, outFilename, {overwrite: true}); + await fs.move(filenameGZ, outFilename, {overwrite: true}); - await fs.remove(filenameCopy); - })().catch((e) => { if (appLogger.inited) appLogger.log(LM_ERR, `FileDecompressor.gzipFileIfNotExists: ${e.message}`) }); + await fs.remove(filenameCopy); + })().catch((e) => { if (appLogger.inited) appLogger.log(LM_ERR, `FileDecompressor.gzipFileIfNotExists: ${e.message}`) }); + } } else { await utils.touchFile(outFilename); } diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index dafc628a..32f77807 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -54,6 +54,7 @@ class ReaderWorker { let decompDir = ''; let downloadedFilename = ''; let isUploaded = false; + let isRestored = false; let convertFilename = ''; const overLoadMes = 'Слишком большая очередь загрузки. Пожалуйста, попробуйте позже.'; @@ -88,9 +89,17 @@ class ReaderWorker { downloadedFilename = `${this.config.tempDownloadDir}/${tempFilename}`; await fs.writeFile(downloadedFilename, downdata); } else {//uploaded file - downloadedFilename = `${this.config.uploadDir}/${url.substr(7)}`; - if (!await fs.pathExists(downloadedFilename)) - throw new Error('Файл не найден на сервере (возможно был удален как устаревший). Пожалуйста, загрузите файл с диска на сервер заново.'); + const fileHash = url.substr(7); + downloadedFilename = `${this.config.uploadDir}/${fileHash}`; + if (!await fs.pathExists(downloadedFilename)) { + //если удалено из upload, попробуем восстановить из удаленного хранилища + try { + downloadedFilename = await this.restoreRemoteFile(fileHash); + isRestored = true; + } catch(e) { + throw new Error('Файл не найден на сервере (возможно был удален как устаревший). Пожалуйста, загрузите файл с диска на сервер заново.'); + } + } await utils.touchFile(downloadedFilename); isUploaded = true; } @@ -146,6 +155,20 @@ class ReaderWorker { })(); } + //лениво сохраним downloadedFilename в tmp и в удаленном хранилище в случае isUploaded + if (this.remoteWebDavStorage && isUploaded && !isRestored) { + (async() => { + await utils.sleep(30*1000); + try { + //сжимаем файл в tmp, если там уже нет с тем же именем-sha256 + const compDownloadedFilename = await this.decomp.gzipFileIfNotExists(downloadedFilename, this.config.tempPublicDir, true); + await this.remoteWebDavStorage.putFile(compDownloadedFilename); + } catch (e) { + log(LM_ERR, e.stack); + } + })(); + } + } catch (e) { log(LM_ERR, e.stack); if (e.message == 'abort') @@ -188,6 +211,24 @@ class ReaderWorker { return `file://${hash}`; } + async restoreRemoteFile(filename) { + const basename = path.basename(filename); + const targetName = `${this.config.tempPublicDir}/${basename}`; + + if (!await fs.pathExists(targetName)) { + let found = false; + if (this.remoteWebDavStorage) { + found = await this.remoteWebDavStorage.getFileSuccess(targetName); + } + + if (!found) { + throw new Error('404 Файл не найден'); + } + } + + return targetName; + } + restoreCachedFile(filename) { const workerId = this.workerState.generateWorkerId(); const wState = this.workerState.getControl(workerId); @@ -197,21 +238,10 @@ class ReaderWorker { try { wState.set({state: 'download', step: 1, totalSteps: 1, path: filename, progress: 0}); - const basename = path.basename(filename); - const targetName = `${this.config.tempPublicDir}/${basename}`; - - if (!await fs.pathExists(targetName)) { - let found = false; - if (this.remoteWebDavStorage) { - found = await this.remoteWebDavStorage.getFileSuccess(targetName); - } - - if (!found) { - throw new Error('404 Файл не найден'); - } - } - + const targetName = await this.restoreRemoteFile(filename); const stat = await fs.stat(targetName); + + const basename = path.basename(filename); wState.finish({path: `/tmp/${basename}`, size: stat.size, progress: 100}); } catch (e) { if (e.message.indexOf('404') < 0)