diff --git a/server/controllers/ReaderController.js b/server/controllers/ReaderController.js index c2288ec7..09596225 100644 --- a/server/controllers/ReaderController.js +++ b/server/controllers/ReaderController.js @@ -62,6 +62,24 @@ class ReaderController extends BaseController { res.status(400).send({error}); return false; } + + async restoreCachedFile(req, res) { + const request = req.body; + let error = ''; + try { + if (!request.path) + throw new Error(`key 'path' is empty`); + + const workerId = this.readerWorker.restoreCachedFile(request.path); + const state = this.workerState.getState(workerId); + return (state ? state : {}); + } catch (e) { + error = e.message; + } + //bad request + res.status(400).send({error}); + return false; + } } module.exports = ReaderController; diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 78dbe0ba..d2b6a92f 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -5,6 +5,7 @@ const WorkerState = require('../WorkerState');//singleton const FileDownloader = require('../FileDownloader'); const FileDecompressor = require('../FileDecompressor'); const BookConverter = require('./BookConverter'); +const RemoteWebDavStorage = require('../RemoteWebDavStorage'); const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton @@ -28,6 +29,11 @@ class ReaderWorker { this.decomp = new FileDecompressor(); this.bookConverter = new BookConverter(this.config); + this.remoteWebDavStorage = false; + if (config.remoteWebDavStorage) { + this.remoteWebDavStorage = new RemoteWebDavStorage(config.remoteWebDavStorage); + } + this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, 60*60*1000);//1 раз в час @@ -39,7 +45,6 @@ class ReaderWorker { async loadBook(opts, wState) { const url = opts.url; - let errMes = ''; let decompDir = ''; let downloadedFilename = ''; let isUploaded = false; @@ -88,16 +93,17 @@ class ReaderWorker { //сжимаем файл в tmp, если там уже нет с тем же именем-sha256 const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, this.config.tempPublicDir); + const stat = await fs.stat(compFilename); wState.set({progress: 100}); //finish const finishFilename = path.basename(compFilename); - wState.finish({path: `/tmp/${finishFilename}`}); + wState.finish({path: `/tmp/${finishFilename}`, size: stat.size}); } catch (e) { log(LM_ERR, e.stack); - wState.set({state: 'error', error: (errMes ? errMes : e.message)}); + wState.set({state: 'error', error: e.message}); } finally { //clean if (decompDir) @@ -133,6 +139,41 @@ class ReaderWorker { return `file://${hash}`; } + restoreCachedFile(filename) { + const workerId = this.workerState.generateWorkerId(); + const wState = this.workerState.getControl(workerId); + wState.set({state: 'start'}); + + (async() => { + try { + wState.set({state: 'download', step: 1, totalSteps: 1, path: filename, progress: 0}); + + const basename = path.basename(filename); + const targetName = `${this.config.tempPublicDir}/${basename}`; + + if (!await fs.pathExists(targetName)) { + let found = false; + if (this.remoteWebDavStorage) { + found = await this.remoteWebDavStorage.getFileSuccess(targetName); + } + + if (!found) { + throw new Error('404 Файл не найден'); + } + } + + const stat = await fs.stat(targetName); + wState.finish({path: `/tmp/${basename}`, size: stat.size, progress: 100}); + } catch (e) { + if (e.message.indexOf('404') < 0) + log(LM_ERR, e.stack); + wState.set({state: 'error', error: e.message}); + } + })(); + + return workerId; + } + async periodicCleanDir(dir, maxSize, timeout) { try { const list = await fs.readdir(dir); @@ -153,7 +194,16 @@ class ReaderWorker { let i = 0; while (i < files.length && size > maxSize) { const file = files[i]; - await fs.remove(`${dir}/${file.name}`); + const oldFile = `${dir}/${file.name}`; + if (this.remoteWebDavStorage) { + try { + //log(`remoteWebDavStorage.putFile ${path.basename(oldFile)}`); + await this.remoteWebDavStorage.putFile(oldFile); + } catch (e) { + log(LM_ERR, e.stack); + } + } + await fs.remove(oldFile); size -= file.stat.size; i++; } diff --git a/server/core/RemoteWebDavStorage.js b/server/core/RemoteWebDavStorage.js new file mode 100644 index 00000000..e7d0ad83 --- /dev/null +++ b/server/core/RemoteWebDavStorage.js @@ -0,0 +1,121 @@ +const fs = require('fs-extra'); +const path = require('path'); + +const WebDavFS = require('webdav-fs'); + +class RemoteWebDavStorage { + constructor(config) { + const opts = Object.assign({}, config); + this.wfs = WebDavFS(config.url, opts); + } + + stat(filename) { + return new Promise((resolve, reject) => { + this.wfs.stat(filename, function(err, fileStat) { + if (err) + reject(err); + resolve(fileStat); + }); + }); + } + + writeFile(filename, data) { + return new Promise((resolve, reject) => { + this.wfs.writeFile(filename, data, 'binary', function(err) { + if (err) + reject(err); + resolve(); + }); + }); + } + + unlink(filename) { + return new Promise((resolve, reject) => { + this.wfs.unlink(filename, function(err) { + if (err) + reject(err); + resolve(); + }); + }); + } + + readFile(filename) { + return new Promise((resolve, reject) => { + this.wfs.readFile(filename, 'binary', function(err, data) { + if (err) + reject(err); + resolve(data); + }); + }); + } + + mkdir(dirname) { + return new Promise((resolve, reject) => { + this.wfs.mkdir(dirname, function(err) { + if (err) + reject(err); + resolve(); + }); + }); + } + + async putFile(filename) { + if (!await fs.pathExists(filename)) { + throw new Error(`File not found: ${filename}`); + } + + const base = path.basename(filename); + let remoteFilename = `/${base}`; + + if (base.length > 3) { + const remoteDir = `/${base.substr(0, 3)}`; + try { + await this.mkdir(remoteDir); + } catch (e) { + // + } + remoteFilename = `${remoteDir}/${base}`; + } + + try { + const localStat = await fs.stat(filename); + const remoteStat = await this.stat(remoteFilename); + if (remoteStat.isFile && localStat.size == remoteStat.size) { + return; + } + await this.unlink(remoteFilename); + } catch (e) { + // + } + + const data = await fs.readFile(filename); + await this.writeFile(remoteFilename, data); + } + + async getFile(filename) { + if (await fs.pathExists(filename)) { + return; + } + + const base = path.basename(filename); + let remoteFilename = `/${base}`; + if (base.length > 3) { + remoteFilename = `/${base.substr(0, 3)}/${base}`; + } + + const data = await this.readFile(remoteFilename); + await fs.writeFile(filename, data); + } + + async getFileSuccess(filename) { + try { + await this.getFile(filename); + return true; + } catch (e) { + // + } + return false; + } +} + +module.exports = RemoteWebDavStorage; \ No newline at end of file diff --git a/server/routes.js b/server/routes.js index 08d20585..09b96239 100644 --- a/server/routes.js +++ b/server/routes.js @@ -28,6 +28,7 @@ function initRoutes(app, config) { ['POST', '/api/reader/load-book', reader.loadBook.bind(reader), [aAll], {}], ['POST', '/api/reader/storage', reader.storage.bind(reader), [aAll], {}], ['POST', '/api/reader/upload-file', [upload.single('file'), reader.uploadFile.bind(reader)], [aAll], {}], + ['POST', '/api/reader/restore-cached-file', reader.restoreCachedFile.bind(reader), [aAll], {}], ['POST', '/api/worker/get-state', worker.getState.bind(worker), [aAll], {}], ['POST', '/api/worker/get-state-finish', worker.getStateFinish.bind(worker), [aAll], {}], ];