From f8481413c937449ba9760a7fac0c9ba963202576 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 12 Jan 2020 17:03:34 +0700 Subject: [PATCH 01/58] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B8=D0=B9=20?= =?UTF-8?q?=D1=80=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE=D1=80=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/ReaderWorker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 2b8d1180..78dbe0ba 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -87,7 +87,7 @@ class ReaderWorker { }); //сжимаем файл в tmp, если там уже нет с тем же именем-sha256 - const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, `${this.config.tempPublicDir}`); + const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, this.config.tempPublicDir); wState.set({progress: 100}); From 6faa7b2efe1b78a0ffffaa9223ee92e02cec7d9c Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 12 Jan 2020 18:51:12 +0700 Subject: [PATCH 02/58] =?UTF-8?q?=D0=A3=D0=BC=D0=B5=D0=BD=D1=8C=D1=88?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B7=D0=B0=D0=BF=D1=80=D0=BE=D1=81?= =?UTF-8?q?=D0=BE=D0=B2=20get-state=20=D0=BA=20api,=20=D0=B4=D0=BE=D0=B1?= =?UTF-8?q?=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=20=D0=BC=D0=B5=D1=82=D0=BE=D0=B4?= =?UTF-8?q?=20get-state-finish?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 62 +++++++++++++++----------- server/controllers/WorkerController.js | 55 +++++++++++++++++++++++ server/routes.js | 1 + 3 files changed, 93 insertions(+), 25 deletions(-) diff --git a/client/api/reader.js b/client/api/reader.js index b3e2da87..4487119d 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -1,7 +1,5 @@ import axios from 'axios'; -import * as utils from '../share/utils'; - const api = axios.create({ baseURL: '/api/reader' }); @@ -12,7 +10,6 @@ const workerApi = axios.create({ class Reader { async loadBook(opts, callback) { - const refreshPause = 300; if (!callback) callback = () => {}; let response = await api.post('/load-book', opts); @@ -22,37 +19,52 @@ class Reader { throw new Error('Неверный ответ api'); callback({totalSteps: 4}); + callback(response.data); - let i = 0; - while (1) {// eslint-disable-line no-constant-condition - callback(response.data); + //присылается текст, состоящий из json-объектов state каждые 300ms, с разделителем splitter между ними + const splitter = '-- aod2t5hDXU32bUFyqlFE next status --'; + let lastIndex = 0; + response = await workerApi.post('/get-state-finish', {workerId}, { + onDownloadProgress: progress => { + //небольая оптимизация, вместо простого responseText.split + const xhr = progress.target; + let currIndex = xhr.responseText.length; + if (lastIndex == currIndex) + return; + const last = xhr.responseText.substring(lastIndex, currIndex); + lastIndex = currIndex; - if (response.data.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл - callback({step: 4}); - const book = await this.loadCachedBook(response.data.path, callback); - return Object.assign({}, response.data, {data: book.data}); + //быстрее будет last.split + const res = last.split(splitter).pop(); + if (res) { + callback(JSON.parse(res)); + } } - if (response.data.state == 'error') { - let errMes = response.data.error; + }); + + //берем последний state + response = response.data.split(splitter).pop(); + + if (response) { + response = JSON.parse(response); + + if (response.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл + callback({step: 4}); + const book = await this.loadCachedBook(response.path, callback); + return Object.assign({}, response, {data: book.data}); + } + + if (response.state == 'error') { + let errMes = response.error; if (errMes.indexOf('getaddrinfo') >= 0 || errMes.indexOf('ECONNRESET') >= 0 || errMes.indexOf('EINVAL') >= 0 || errMes.indexOf('404') >= 0) - errMes = `Ресурс не найден по адресу: ${response.data.url}`; + errMes = `Ресурс не найден по адресу: ${response.url}`; throw new Error(errMes); } - if (i > 0) - await utils.sleep(refreshPause); - - i++; - if (i > 120*1000/refreshPause) {//2 мин ждем телодвижений воркера - throw new Error('Слишком долгое время ожидания'); - } - //проверка воркера - const prevProgress = response.data.progress; - const prevState = response.data.state; - response = await workerApi.post('/get-state', {workerId}); - i = (prevProgress != response.data.progress || prevState != response.data.state ? 1 : i); + } else { + throw new Error('Пустой ответ сервера'); } } diff --git a/server/controllers/WorkerController.js b/server/controllers/WorkerController.js index 69bd4982..7fa2647f 100644 --- a/server/controllers/WorkerController.js +++ b/server/controllers/WorkerController.js @@ -1,5 +1,6 @@ const BaseController = require('./BaseController'); const WorkerState = require('../core/WorkerState');//singleton +const utils = require('../core/utils'); class WorkerController extends BaseController { constructor(config) { @@ -15,6 +16,7 @@ class WorkerController extends BaseController { throw new Error(`key 'workerId' is wrong`); const state = this.workerState.getState(request.workerId); + return (state ? state : {}); } catch (e) { error = e.message; @@ -23,6 +25,59 @@ class WorkerController extends BaseController { res.status(400).send({error}); return false; } + + async getStateFinish(req, res) { + const request = req.body; + let error = ''; + try { + if (!request.workerId) + throw new Error(`key 'workerId' is wrong`); + + res.writeHead(200, { + 'Content-Type': 'text/json; charset=utf-8', + }); + + const splitter = '-- aod2t5hDXU32bUFyqlFE next status --'; + const refreshPause = 300; + let i = 0; + let prevProgress = -1; + let prevState = ''; + let state; + while (1) {// eslint-disable-line no-constant-condition + state = this.workerState.getState(request.workerId); + if (!state) break; + + res.write(splitter + JSON.stringify(state)); + res.flush(); + + if (state.state != 'finish') + await utils.sleep(refreshPause); + else + break; + + i++; + if (i > 2*60*1000/refreshPause) {//2 мин ждем телодвижений воркера + res.write(splitter + JSON.stringify({state: 'error', error: 'Слишком долгое время ожидания'})); + break; + } + i = (prevProgress != state.progress || prevState != state.state ? 1 : i); + prevProgress = state.progress; + prevState = state.state; + } + + if (!state) { + res.write(splitter + JSON.stringify({})); + } + + res.end(); + return false; + } catch (e) { + error = e.message; + } + //bad request + res.status(400).send({error}); + return false; + } } module.exports = WorkerController; diff --git a/server/routes.js b/server/routes.js index ffdd92c8..08d20585 100644 --- a/server/routes.js +++ b/server/routes.js @@ -29,6 +29,7 @@ function initRoutes(app, config) { ['POST', '/api/reader/storage', reader.storage.bind(reader), [aAll], {}], ['POST', '/api/reader/upload-file', [upload.single('file'), reader.uploadFile.bind(reader)], [aAll], {}], ['POST', '/api/worker/get-state', worker.getState.bind(worker), [aAll], {}], + ['POST', '/api/worker/get-state-finish', worker.getStateFinish.bind(worker), [aAll], {}], ]; //to app From 52a32cfdd1f09da8efed3941012cb253a8d180b4 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 12 Jan 2020 20:06:50 +0700 Subject: [PATCH 03/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=BE=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BE=D0=BA=20JSON.pars?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/client/api/reader.js b/client/api/reader.js index 4487119d..0e94c4b1 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -37,7 +37,11 @@ class Reader { //быстрее будет last.split const res = last.split(splitter).pop(); if (res) { - callback(JSON.parse(res)); + try { + callback(JSON.parse(res)); + } catch (e) { + // + } } } }); @@ -46,8 +50,14 @@ class Reader { response = response.data.split(splitter).pop(); if (response) { - response = JSON.parse(response); + try { + response = JSON.parse(response); + } catch (e) { + response = false; + } + } + if (response) { if (response.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл callback({step: 4}); const book = await this.loadCachedBook(response.path, callback); From 02aee3e625f96f02791932afa020ab47c09fdbe2 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 15 Jan 2020 15:49:45 +0700 Subject: [PATCH 04/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=BF=D0=B5=D1=80=D0=B5=D1=83=D0=BF=D0=B0?= =?UTF-8?q?=D0=BA=D0=BE=D0=B2=D0=BA=D0=B0=20=D1=84=D0=B0=D0=B9=D0=BB=D0=B0?= =?UTF-8?q?=20=D0=BA=D0=BD=D0=B8=D0=B3=D0=B8=20=D0=BF=D0=BE=20=D0=BC=D0=B0?= =?UTF-8?q?=D0=BA=D1=81=D0=B8=D0=BC=D1=83=D0=BC=D1=83=20=D1=87=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D0=B7=205=20=D1=81=D0=B5=D0=BA=20=D0=BF=D0=BE=D1=81?= =?UTF-8?q?=D0=BB=D0=B5=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B8?= =?UTF-8?q?=20=D0=B8=20=D0=BA=D0=BE=D0=BD=D0=B2=D0=B5=D1=80=D1=82=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/FileDecompressor.js | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/server/core/FileDecompressor.js b/server/core/FileDecompressor.js index 7b7c68fc..f1353142 100644 --- a/server/core/FileDecompressor.js +++ b/server/core/FileDecompressor.js @@ -5,6 +5,7 @@ const unbzip2Stream = require('unbzip2-stream'); const tar = require('tar-fs'); const ZipStreamer = require('./ZipStreamer'); +const log = new (require('./AppLogger'))().log;//singleton const utils = require('./utils'); const FileDetector = require('./FileDetector'); @@ -189,9 +190,9 @@ class FileDecompressor { }); } - async gzipFile(inputFile, outputFile) { + async gzipFile(inputFile, outputFile, level = 1) { return new Promise((resolve, reject) => { - const gzip = zlib.createGzip({level: 1}); + const gzip = zlib.createGzip({level}); const input = fs.createReadStream(inputFile); const output = fs.createWriteStream(outputFile); @@ -208,7 +209,21 @@ class FileDecompressor { const outFilename = `${outDir}/${hash}`; if (!await fs.pathExists(outFilename)) { - await this.gzipFile(filename, outFilename); + await this.gzipFile(filename, outFilename, 1); + + // переупакуем через некоторое время на максималках + const filenameCopy = `${filename}.copy`; + await fs.copy(filename, filenameCopy); + + (async() => { + await utils.sleep(5000); + const filenameGZ = `${filename}.gz`; + await this.gzipFile(filenameCopy, filenameGZ, 9); + + await fs.move(filenameGZ, outFilename, {overwrite: true}); + + await fs.remove(filenameCopy); + })().catch((e) => { log(LM_ERR, e.message) }); } else { await utils.touchFile(outFilename); } From f814c42fdd43a7d63847c16ec04b6bfc673f6a07 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 15 Jan 2020 16:06:28 +0700 Subject: [PATCH 05/58] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=20=D0=B1=D0=B0=D0=B3=20=D0=B2=20getStateFini?= =?UTF-8?q?sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/controllers/WorkerController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/controllers/WorkerController.js b/server/controllers/WorkerController.js index 7fa2647f..54d229dc 100644 --- a/server/controllers/WorkerController.js +++ b/server/controllers/WorkerController.js @@ -50,7 +50,7 @@ class WorkerController extends BaseController { res.write(splitter + JSON.stringify(state)); res.flush(); - if (state.state != 'finish') + if (state.state != 'finish' && state.state != 'error') await utils.sleep(refreshPause); else break; From 4ab0c337f17af6cf4fe0d6fb5025085228448a73 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 15 Jan 2020 16:20:46 +0700 Subject: [PATCH 06/58] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/client/api/reader.js b/client/api/reader.js index 0e94c4b1..b57f0a79 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -9,22 +9,14 @@ const workerApi = axios.create({ }); class Reader { - async loadBook(opts, callback) { + + async getStateFinish(workerId, callback) { if (!callback) callback = () => {}; - let response = await api.post('/load-book', opts); - - const workerId = response.data.workerId; - if (!workerId) - throw new Error('Неверный ответ api'); - - callback({totalSteps: 4}); - callback(response.data); - //присылается текст, состоящий из json-объектов state каждые 300ms, с разделителем splitter между ними const splitter = '-- aod2t5hDXU32bUFyqlFE next status --'; let lastIndex = 0; - response = await workerApi.post('/get-state-finish', {workerId}, { + let response = await workerApi.post('/get-state-finish', {workerId}, { onDownloadProgress: progress => { //небольая оптимизация, вместо простого responseText.split const xhr = progress.target; @@ -57,6 +49,23 @@ class Reader { } } + return response; + } + + async loadBook(opts, callback) { + if (!callback) callback = () => {}; + + let response = await api.post('/load-book', opts); + + const workerId = response.data.workerId; + if (!workerId) + throw new Error('Неверный ответ api'); + + callback({totalSteps: 4}); + callback(response.data); + + response = await this.getStateFinish(workerId, callback); + if (response) { if (response.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл callback({step: 4}); @@ -83,7 +92,7 @@ class Reader { } async loadCachedBook(url, callback) { - const response = await axios.head(url); + const response = await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); let estSize = 1000000; if (response.headers['content-length']) { From 010ac9aa7ce4ab81591cd3d515e4312ac050882d Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 20 Jan 2020 21:21:13 +0700 Subject: [PATCH 07/58] =?UTF-8?q?=D0=94=D0=BE=D1=80=D0=B0=D0=B1=D0=BE?= =?UTF-8?q?=D1=82=D0=BA=D0=B0=20api,=20=D0=B2=D0=BE=D1=81=D1=81=D1=82?= =?UTF-8?q?=D0=B0=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BA?= =?UTF-8?q?=D1=8D=D1=88=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=BD=D0=BE?= =?UTF-8?q?=D0=B3=D0=BE=20=D1=84=D0=B0=D0=B9=D0=BB=D0=B0=20=D0=B8=D0=B7=20?= =?UTF-8?q?=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 76 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 11 deletions(-) diff --git a/client/api/reader.js b/client/api/reader.js index b57f0a79..62d1ab23 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -69,7 +69,7 @@ class Reader { if (response) { if (response.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл callback({step: 4}); - const book = await this.loadCachedBook(response.path, callback); + const book = await this.loadCachedBook(response.path, callback, false, (response.size ? response.size : -1)); return Object.assign({}, response, {data: book.data}); } @@ -88,18 +88,72 @@ class Reader { } async checkUrl(url) { - return await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); - } - - async loadCachedBook(url, callback) { - const response = await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); - - let estSize = 1000000; - if (response.headers['content-length']) { - estSize = response.headers['content-length']; + let fileExists = false; + try { + await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); + fileExists = true; + } catch (e) { + // } + //восстановим при необходимости файл на сервере из удаленного облака + if (!fileExists) { + let response = await api.post('/restore-cached-file', {path: url}); + + const workerId = response.data.workerId; + if (!workerId) + throw new Error('Неверный ответ api'); + + response = await this.getStateFinish(workerId); + if (response.state == 'error') { + throw new Error(response.error); + } + } + + return true; + } + + async loadCachedBook(url, callback, restore = true, estSize = -1) { + if (!callback) callback = () => {}; + let response = null; + callback({state: 'loading', progress: 0}); + + //получение размера файла + let fileExists = false; + if (estSize < 0) { + try { + response = await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); + + if (response.headers['content-length']) { + estSize = response.headers['content-length']; + } + fileExists = true; + } catch (e) { + // + } + } + + //восстановим при необходимости файл на сервере из удаленного облака + if (restore && !fileExists) { + response = await api.post('/restore-cached-file', {path: url}); + + const workerId = response.data.workerId; + if (!workerId) + throw new Error('Неверный ответ api'); + + response = await this.getStateFinish(workerId); + if (response.state == 'error') { + throw new Error(response.error); + } + + if (response.size && estSize < 0) { + estSize = response.size; + } + } + + //получение файла + estSize = (estSize > 0 ? estSize : 1000000); const options = { onDownloadProgress: progress => { while (progress.loaded > estSize) estSize *= 1.5; @@ -108,7 +162,7 @@ class Reader { callback({progress: Math.round((progress.loaded*100)/estSize)}); } } - //загрузка + return await axios.get(url, options); } From c926b869260b59691e76977104e58905dc35b418 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 20 Jan 2020 21:22:27 +0700 Subject: [PATCH 08/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20=D0=BF=D0=B0=D0=BA=D0=B5=D1=82=20webdav-fs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 92 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 93 insertions(+) diff --git a/package-lock.json b/package-lock.json index 58e7c28b..b6dcfe3e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1725,6 +1725,11 @@ } } }, + "base-64": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", + "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs=" + }, "base-x": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.7.tgz", @@ -5719,6 +5724,11 @@ "parse-passwd": "^1.0.0" } }, + "hot-patcher": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/hot-patcher/-/hot-patcher-0.5.0.tgz", + "integrity": "sha512-2Uu2W0s8+dnqXzdlg0MRsRzPoDCs1wVjOGSyMRRaMzLDX4bgHw6xDYKccsWafXPPxQpkQfEjgW6+17pwcg60bw==" + }, "hsl-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/hsl-regex/-/hsl-regex-1.0.0.tgz", @@ -6902,6 +6912,11 @@ } } }, + "merge": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/merge/-/merge-1.2.1.tgz", + "integrity": "sha512-VjFo4P5Whtj4vsLzsYBu5ayHhoHJ0UqNm7ibvShmbmoz7tGi0vXaoJbGdB+GmDMLUdg8DpQXEIeVDAe8MaABvQ==" + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -7882,6 +7897,11 @@ "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", "dev": true }, + "path-posix": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/path-posix/-/path-posix-1.0.0.tgz", + "integrity": "sha1-BrJhE/Vr6rBCVFojv6iAA8ysJg8=" + }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", @@ -10423,6 +10443,11 @@ "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=", "dev": true }, + "querystringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.1.tgz", + "integrity": "sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA==" + }, "randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -10709,6 +10734,11 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "dev": true }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" + }, "resize-observer-polyfill": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz", @@ -12489,6 +12519,11 @@ } } }, + "url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==" + }, "url-loader": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/url-loader/-/url-loader-1.1.2.tgz", @@ -12508,6 +12543,15 @@ } } }, + "url-parse": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", + "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "url-parse-lax": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", @@ -12737,6 +12781,40 @@ "neo-async": "^2.5.0" } }, + "webdav": { + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/webdav/-/webdav-2.10.1.tgz", + "integrity": "sha512-3UfnjGTAqSM9MW3Rpt1KrY1KneYK0wPCFryHTncqw1OP1pyiniT3uYhVpgmH6za/TkWOfnTnKCDKhwrLJFdzow==", + "requires": { + "axios": "^0.19.0", + "base-64": "^0.1.0", + "hot-patcher": "^0.5.0", + "merge": "^1.2.1", + "minimatch": "^3.0.4", + "path-posix": "^1.0.0", + "url-join": "^4.0.1", + "url-parse": "^1.4.7", + "xml2js": "^0.4.19" + }, + "dependencies": { + "axios": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.1.tgz", + "integrity": "sha512-Yl+7nfreYKaLRvAvjNPkvfjnQHJM1yLBY3zhqAwcJSwR/6ETkanUgylgtIvkvz0xJ+p/vZuNw8X7Hnb7Whsbpw==", + "requires": { + "follow-redirects": "1.5.10" + } + } + } + }, + "webdav-fs": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/webdav-fs/-/webdav-fs-2.0.0.tgz", + "integrity": "sha512-TjqQKNnf1NuPiMEFJVrmWUYpIEgUxWRktddu5JhQsyxFIOyAJT0cpLyaoOdAp/yrG1yXgDsa6ZL9z+h4Z71zWA==", + "requires": { + "webdav": "^2.0.0" + } + }, "webpack": { "version": "4.40.2", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.40.2.tgz", @@ -13005,6 +13083,20 @@ "mkdirp": "^0.5.1" } }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 7a8de918..bfc0c133 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "vue-router": "^3.1.3", "vuex": "^3.1.1", "vuex-persistedstate": "^2.5.4", + "webdav-fs": "^2.0.0", "zip-stream": "^2.1.2" } } From 8747a00de67cdddddf908c4f384f2cdf5c7f8738 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 20 Jan 2020 21:36:44 +0700 Subject: [PATCH 09/58] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=20=D0=B1=D0=B0=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/FileDecompressor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/core/FileDecompressor.js b/server/core/FileDecompressor.js index f1353142..6460cf46 100644 --- a/server/core/FileDecompressor.js +++ b/server/core/FileDecompressor.js @@ -5,7 +5,7 @@ const unbzip2Stream = require('unbzip2-stream'); const tar = require('tar-fs'); const ZipStreamer = require('./ZipStreamer'); -const log = new (require('./AppLogger'))().log;//singleton +const appLogger = new (require('./AppLogger'))();//singleton const utils = require('./utils'); const FileDetector = require('./FileDetector'); @@ -223,7 +223,7 @@ class FileDecompressor { await fs.move(filenameGZ, outFilename, {overwrite: true}); await fs.remove(filenameCopy); - })().catch((e) => { log(LM_ERR, e.message) }); + })().catch((e) => { if (appLogger.inited) appLogger.log(LM_ERR, `FileDecompressor.gzipFileIfNotExists: ${e.message}`) }); } else { await utils.touchFile(outFilename); } From 5535bd91c879d00a4021643ff859ed52bcfc86a1 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 20 Jan 2020 21:37:31 +0700 Subject: [PATCH 10/58] =?UTF-8?q?=D0=92=20=D0=BA=D0=BE=D0=BD=D1=84=D0=B8?= =?UTF-8?q?=D0=B3=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B0?= =?UTF-8?q?=20=D0=BE=D0=BF=D1=86=D0=B8=D1=8F=20remoteWebDavStorage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/config/base.js | 11 ++++++++++- server/config/index.js | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/server/config/base.js b/server/config/base.js index 33007a57..7370cec7 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -21,7 +21,7 @@ module.exports = { maxTempPublicDirSize: 512*1024*1024,//512Мб maxUploadPublicDirSize: 200*1024*1024,//100Мб - useExternalBookConverter: false, + useExternalBookConverter: false, db: [ { @@ -45,5 +45,14 @@ module.exports = { }, ], + remoteWebDavStorage: false, + /* + remoteWebDavStorage: { + url: '127.0.0.1:1900', + username: '', + password: '', + }, + */ + }; diff --git a/server/config/index.js b/server/config/index.js index bd17f7b3..7ffa99f1 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -10,6 +10,7 @@ const propsToSave = [ 'useExternalBookConverter', 'servers', + 'remoteWebDavStorage', ]; let instance = null; From 47e46f13c366ce4322e4789542181d68255fa88e Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 20 Jan 2020 21:39:55 +0700 Subject: [PATCH 11/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20=D1=81=20?= =?UTF-8?q?RemoteWebDavStorage,=20=D0=B2=20=D1=82.=D1=87.=20=D1=87=D0=B5?= =?UTF-8?q?=D1=80=D0=B5=D0=B7=20api?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/controllers/ReaderController.js | 18 ++++ server/core/Reader/ReaderWorker.js | 58 +++++++++++- server/core/RemoteWebDavStorage.js | 121 +++++++++++++++++++++++++ server/routes.js | 1 + 4 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 server/core/RemoteWebDavStorage.js diff --git a/server/controllers/ReaderController.js b/server/controllers/ReaderController.js index c2288ec7..09596225 100644 --- a/server/controllers/ReaderController.js +++ b/server/controllers/ReaderController.js @@ -62,6 +62,24 @@ class ReaderController extends BaseController { res.status(400).send({error}); return false; } + + async restoreCachedFile(req, res) { + const request = req.body; + let error = ''; + try { + if (!request.path) + throw new Error(`key 'path' is empty`); + + const workerId = this.readerWorker.restoreCachedFile(request.path); + const state = this.workerState.getState(workerId); + return (state ? state : {}); + } catch (e) { + error = e.message; + } + //bad request + res.status(400).send({error}); + return false; + } } module.exports = ReaderController; diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 78dbe0ba..d2b6a92f 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -5,6 +5,7 @@ const WorkerState = require('../WorkerState');//singleton const FileDownloader = require('../FileDownloader'); const FileDecompressor = require('../FileDecompressor'); const BookConverter = require('./BookConverter'); +const RemoteWebDavStorage = require('../RemoteWebDavStorage'); const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton @@ -28,6 +29,11 @@ class ReaderWorker { this.decomp = new FileDecompressor(); this.bookConverter = new BookConverter(this.config); + this.remoteWebDavStorage = false; + if (config.remoteWebDavStorage) { + this.remoteWebDavStorage = new RemoteWebDavStorage(config.remoteWebDavStorage); + } + this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, 60*60*1000);//1 раз в час @@ -39,7 +45,6 @@ class ReaderWorker { async loadBook(opts, wState) { const url = opts.url; - let errMes = ''; let decompDir = ''; let downloadedFilename = ''; let isUploaded = false; @@ -88,16 +93,17 @@ class ReaderWorker { //сжимаем файл в tmp, если там уже нет с тем же именем-sha256 const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, this.config.tempPublicDir); + const stat = await fs.stat(compFilename); wState.set({progress: 100}); //finish const finishFilename = path.basename(compFilename); - wState.finish({path: `/tmp/${finishFilename}`}); + wState.finish({path: `/tmp/${finishFilename}`, size: stat.size}); } catch (e) { log(LM_ERR, e.stack); - wState.set({state: 'error', error: (errMes ? errMes : e.message)}); + wState.set({state: 'error', error: e.message}); } finally { //clean if (decompDir) @@ -133,6 +139,41 @@ class ReaderWorker { return `file://${hash}`; } + restoreCachedFile(filename) { + const workerId = this.workerState.generateWorkerId(); + const wState = this.workerState.getControl(workerId); + wState.set({state: 'start'}); + + (async() => { + try { + wState.set({state: 'download', step: 1, totalSteps: 1, path: filename, progress: 0}); + + const basename = path.basename(filename); + const targetName = `${this.config.tempPublicDir}/${basename}`; + + if (!await fs.pathExists(targetName)) { + let found = false; + if (this.remoteWebDavStorage) { + found = await this.remoteWebDavStorage.getFileSuccess(targetName); + } + + if (!found) { + throw new Error('404 Файл не найден'); + } + } + + const stat = await fs.stat(targetName); + wState.finish({path: `/tmp/${basename}`, size: stat.size, progress: 100}); + } catch (e) { + if (e.message.indexOf('404') < 0) + log(LM_ERR, e.stack); + wState.set({state: 'error', error: e.message}); + } + })(); + + return workerId; + } + async periodicCleanDir(dir, maxSize, timeout) { try { const list = await fs.readdir(dir); @@ -153,7 +194,16 @@ class ReaderWorker { let i = 0; while (i < files.length && size > maxSize) { const file = files[i]; - await fs.remove(`${dir}/${file.name}`); + const oldFile = `${dir}/${file.name}`; + if (this.remoteWebDavStorage) { + try { + //log(`remoteWebDavStorage.putFile ${path.basename(oldFile)}`); + await this.remoteWebDavStorage.putFile(oldFile); + } catch (e) { + log(LM_ERR, e.stack); + } + } + await fs.remove(oldFile); size -= file.stat.size; i++; } diff --git a/server/core/RemoteWebDavStorage.js b/server/core/RemoteWebDavStorage.js new file mode 100644 index 00000000..e7d0ad83 --- /dev/null +++ b/server/core/RemoteWebDavStorage.js @@ -0,0 +1,121 @@ +const fs = require('fs-extra'); +const path = require('path'); + +const WebDavFS = require('webdav-fs'); + +class RemoteWebDavStorage { + constructor(config) { + const opts = Object.assign({}, config); + this.wfs = WebDavFS(config.url, opts); + } + + stat(filename) { + return new Promise((resolve, reject) => { + this.wfs.stat(filename, function(err, fileStat) { + if (err) + reject(err); + resolve(fileStat); + }); + }); + } + + writeFile(filename, data) { + return new Promise((resolve, reject) => { + this.wfs.writeFile(filename, data, 'binary', function(err) { + if (err) + reject(err); + resolve(); + }); + }); + } + + unlink(filename) { + return new Promise((resolve, reject) => { + this.wfs.unlink(filename, function(err) { + if (err) + reject(err); + resolve(); + }); + }); + } + + readFile(filename) { + return new Promise((resolve, reject) => { + this.wfs.readFile(filename, 'binary', function(err, data) { + if (err) + reject(err); + resolve(data); + }); + }); + } + + mkdir(dirname) { + return new Promise((resolve, reject) => { + this.wfs.mkdir(dirname, function(err) { + if (err) + reject(err); + resolve(); + }); + }); + } + + async putFile(filename) { + if (!await fs.pathExists(filename)) { + throw new Error(`File not found: ${filename}`); + } + + const base = path.basename(filename); + let remoteFilename = `/${base}`; + + if (base.length > 3) { + const remoteDir = `/${base.substr(0, 3)}`; + try { + await this.mkdir(remoteDir); + } catch (e) { + // + } + remoteFilename = `${remoteDir}/${base}`; + } + + try { + const localStat = await fs.stat(filename); + const remoteStat = await this.stat(remoteFilename); + if (remoteStat.isFile && localStat.size == remoteStat.size) { + return; + } + await this.unlink(remoteFilename); + } catch (e) { + // + } + + const data = await fs.readFile(filename); + await this.writeFile(remoteFilename, data); + } + + async getFile(filename) { + if (await fs.pathExists(filename)) { + return; + } + + const base = path.basename(filename); + let remoteFilename = `/${base}`; + if (base.length > 3) { + remoteFilename = `/${base.substr(0, 3)}/${base}`; + } + + const data = await this.readFile(remoteFilename); + await fs.writeFile(filename, data); + } + + async getFileSuccess(filename) { + try { + await this.getFile(filename); + return true; + } catch (e) { + // + } + return false; + } +} + +module.exports = RemoteWebDavStorage; \ No newline at end of file diff --git a/server/routes.js b/server/routes.js index 08d20585..09b96239 100644 --- a/server/routes.js +++ b/server/routes.js @@ -28,6 +28,7 @@ function initRoutes(app, config) { ['POST', '/api/reader/load-book', reader.loadBook.bind(reader), [aAll], {}], ['POST', '/api/reader/storage', reader.storage.bind(reader), [aAll], {}], ['POST', '/api/reader/upload-file', [upload.single('file'), reader.uploadFile.bind(reader)], [aAll], {}], + ['POST', '/api/reader/restore-cached-file', reader.restoreCachedFile.bind(reader), [aAll], {}], ['POST', '/api/worker/get-state', worker.getState.bind(worker), [aAll], {}], ['POST', '/api/worker/get-state-finish', worker.getStateFinish.bind(worker), [aAll], {}], ]; From a04161ac7cffe95a1381154c5dcb2fa907414860 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 20 Jan 2020 21:44:09 +0700 Subject: [PATCH 12/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D0=BB=20=D0=BF=D1=80=D0=B8=D0=BD=D1=83=D0=B4=D0=B8=D1=82=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D0=BD=D1=83=D1=8E=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83?= =?UTF-8?q?=D0=B7=D0=BA=D1=83=20=D0=BA=D0=BD=D0=B8=D0=B3=D0=B8=20=D0=B2=20?= =?UTF-8?q?=D0=BE=D0=B1=D1=85=D0=BE=D0=B4=20=D0=BA=D1=8D=D1=88=D0=B0,=20?= =?UTF-8?q?=D0=B5=D1=81=D0=BB=D0=B8=20=D1=83=D0=BA=D0=B0=D0=B7=D0=B0=D0=BD?= =?UTF-8?q?=20URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/LoaderPage/LoaderPage.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/components/Reader/LoaderPage/LoaderPage.vue b/client/components/Reader/LoaderPage/LoaderPage.vue index 530a9695..9eee7120 100644 --- a/client/components/Reader/LoaderPage/LoaderPage.vue +++ b/client/components/Reader/LoaderPage/LoaderPage.vue @@ -112,7 +112,7 @@ class LoaderPage extends Vue { submitUrl() { if (this.bookUrl) { - this.$emit('load-book', {url: this.bookUrl}); + this.$emit('load-book', {url: this.bookUrl, force: true}); this.bookUrl = ''; } } From 57a44c59528735cbb71f6b906e8244e44f0f55b5 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 20 Jan 2020 21:48:31 +0700 Subject: [PATCH 13/58] =?UTF-8?q?=D0=92=D0=B5=D1=80=D1=81=D0=B8=D1=8F=200.?= =?UTF-8?q?8.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/versionHistory.js | 11 +++++++++++ package.json | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/client/components/Reader/versionHistory.js b/client/components/Reader/versionHistory.js index 7d7f0afb..b05fa59d 100644 --- a/client/components/Reader/versionHistory.js +++ b/client/components/Reader/versionHistory.js @@ -1,4 +1,15 @@ export const versionHistory = [ +{ + showUntil: '2020-01-19', + header: '0.8.2 (2020-01-20)', + content: +` +
    +
  • внутренние оптимизации
    • +
+` +}, + { showUntil: '2020-01-06', header: '0.8.1 (2020-01-07)', diff --git a/package.json b/package.json index bfc0c133..99e0e533 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.8.1", + "version": "0.8.2", "author": "Book Pauk ", "license": "CC0-1.0", "repository": "bookpauk/liberama", From 81629fab7a31189c88d3f586085680a3c6702594 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 13:54:21 +0700 Subject: [PATCH 14/58] =?UTF-8?q?=D0=97=D0=B0=D0=BC=D0=B5=D0=BD=D0=B0=20we?= =?UTF-8?q?bdav-fs=20=D0=BD=D0=B0=20webdav?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 10 +---- package.json | 2 +- server/core/Reader/ReaderWorker.js | 4 +- server/core/RemoteWebDavStorage.js | 72 ++++++++++++------------------ 4 files changed, 34 insertions(+), 54 deletions(-) diff --git a/package-lock.json b/package-lock.json index b6dcfe3e..f4d149a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.8.1", + "version": "0.8.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -12807,14 +12807,6 @@ } } }, - "webdav-fs": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/webdav-fs/-/webdav-fs-2.0.0.tgz", - "integrity": "sha512-TjqQKNnf1NuPiMEFJVrmWUYpIEgUxWRktddu5JhQsyxFIOyAJT0cpLyaoOdAp/yrG1yXgDsa6ZL9z+h4Z71zWA==", - "requires": { - "webdav": "^2.0.0" - } - }, "webpack": { "version": "4.40.2", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.40.2.tgz", diff --git a/package.json b/package.json index 99e0e533..3be87510 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "vue-router": "^3.1.3", "vuex": "^3.1.1", "vuex-persistedstate": "^2.5.4", - "webdav-fs": "^2.0.0", + "webdav": "^2.10.1", "zip-stream": "^2.1.2" } } diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index d2b6a92f..014a2e76 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -31,7 +31,9 @@ class ReaderWorker { this.remoteWebDavStorage = false; if (config.remoteWebDavStorage) { - this.remoteWebDavStorage = new RemoteWebDavStorage(config.remoteWebDavStorage); + this.remoteWebDavStorage = new RemoteWebDavStorage( + Object.assign({maxContentLength: config.maxUploadFileSize}, config.remoteWebDavStorage) + ); } this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час diff --git a/server/core/RemoteWebDavStorage.js b/server/core/RemoteWebDavStorage.js index e7d0ad83..21d0c107 100644 --- a/server/core/RemoteWebDavStorage.js +++ b/server/core/RemoteWebDavStorage.js @@ -1,62 +1,48 @@ const fs = require('fs-extra'); const path = require('path'); -const WebDavFS = require('webdav-fs'); +const { createClient } = require('webdav'); class RemoteWebDavStorage { constructor(config) { - const opts = Object.assign({}, config); - this.wfs = WebDavFS(config.url, opts); + this.config = Object.assign({}, config); + this.config.maxContentLength = this.config.maxContentLength || 10*1024*1024; + this.wdc = createClient(config.url, this.config); } - stat(filename) { - return new Promise((resolve, reject) => { - this.wfs.stat(filename, function(err, fileStat) { - if (err) - reject(err); - resolve(fileStat); - }); - }); + _convertStat(data) { + return { + isDirectory: function() { + return data.type === "directory"; + }, + isFile: function() { + return data.type === "file"; + }, + mtime: (new Date(data.lastmod)).getTime(), + name: data.basename, + size: data.size || 0 + }; } - writeFile(filename, data) { - return new Promise((resolve, reject) => { - this.wfs.writeFile(filename, data, 'binary', function(err) { - if (err) - reject(err); - resolve(); - }); - }); + async stat(filename) { + const stat = await this.wdc.stat(filename); + return this._convertStat(stat); } - unlink(filename) { - return new Promise((resolve, reject) => { - this.wfs.unlink(filename, function(err) { - if (err) - reject(err); - resolve(); - }); - }); + async writeFile(filename, data) { + return await this.wdc.putFileContents(filename, data, { maxContentLength: this.config.maxContentLength }) } - readFile(filename) { - return new Promise((resolve, reject) => { - this.wfs.readFile(filename, 'binary', function(err, data) { - if (err) - reject(err); - resolve(data); - }); - }); + async unlink(filename) { + return await this.wdc.deleteFile(filename); } - mkdir(dirname) { - return new Promise((resolve, reject) => { - this.wfs.mkdir(dirname, function(err) { - if (err) - reject(err); - resolve(); - }); - }); + async readFile(filename) { + return await this.wdc.getFileContents(filename, { maxContentLength: this.config.maxContentLength }) + } + + async mkdir(dirname) { + return await this.wdc.createDirectory(dirname); } async putFile(filename) { From 5647e8219d49fcb01d21f01d58e5c7e8b33f358b Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 14:58:42 +0700 Subject: [PATCH 15/58] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B8=D0=B9=20?= =?UTF-8?q?=D1=80=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE=D1=80=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/ReaderWorker.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 014a2e76..bfffb716 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -10,6 +10,8 @@ const RemoteWebDavStorage = require('../RemoteWebDavStorage'); const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton +const cleanDirPeriod = 60*60*1000;//1 раз в час + let instance = null; //singleton @@ -36,8 +38,8 @@ class ReaderWorker { ); } - this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час - this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, 60*60*1000);//1 раз в час + this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, cleanDirPeriod); + this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, cleanDirPeriod); instance = this; } From 7cc63fe849ee66669cbdf6050cb93a4768a7f1d6 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 15:53:23 +0700 Subject: [PATCH 16/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=B0=D0=B2=D1=82=D0=BE=D0=BC=D0=B0=D1=82?= =?UTF-8?q?=D0=B8=D1=87=D0=B5=D1=81=D0=BA=D0=B0=D1=8F=20=D0=BE=D1=82=D0=BF?= =?UTF-8?q?=D1=80=D0=B0=D0=B2=D0=BA=D0=B0=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83?= =?UTF-8?q?=D0=B6=D0=B5=D0=BD=D0=BD=D0=BE=D0=B9=20=D0=BA=D0=BD=D0=B8=D0=B3?= =?UTF-8?q?=D0=B8=20=D1=83=D0=B4=D0=B0=D0=BB=D0=B5=D0=BD=D0=BD=D0=BE=D0=B5?= =?UTF-8?q?=20=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/ReaderWorker.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index bfffb716..ef24a707 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -105,6 +105,19 @@ class ReaderWorker { const finishFilename = path.basename(compFilename); wState.finish({path: `/tmp/${finishFilename}`, size: stat.size}); + //лениво сохраним compFilename в удаленном хранилище + if (this.remoteWebDavStorage) { + (async() => { + await utils.sleep(20*1000); + try { + //log(`remoteWebDavStorage.putFile ${path.basename(compFilename)}`); + await this.remoteWebDavStorage.putFile(compFilename); + } catch (e) { + log(LM_ERR, e.stack); + } + })(); + } + } catch (e) { log(LM_ERR, e.stack); wState.set({state: 'error', error: e.message}); From 9c7a6c64b0d2c13c9625b2b86f37ee7215abba64 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 16:13:38 +0700 Subject: [PATCH 17/58] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D1=88=D0=B8=D0=B5=20=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA?= =?UTF-8?q?=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/controllers/WorkerController.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/controllers/WorkerController.js b/server/controllers/WorkerController.js index 54d229dc..86554ab4 100644 --- a/server/controllers/WorkerController.js +++ b/server/controllers/WorkerController.js @@ -38,7 +38,7 @@ class WorkerController extends BaseController { }); const splitter = '-- aod2t5hDXU32bUFyqlFE next status --'; - const refreshPause = 300; + const refreshPause = 200; let i = 0; let prevProgress = -1; let prevState = ''; @@ -47,6 +47,10 @@ class WorkerController extends BaseController { state = this.workerState.getState(request.workerId); if (!state) break; + if (i == 0) { + state = Object.assign({dummy: '0'.repeat(1024)}, state); + } + res.write(splitter + JSON.stringify(state)); res.flush(); From 185fb57b8c93c41162d9c8c7e29acd46f22febf8 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 16:25:30 +0700 Subject: [PATCH 18/58] =?UTF-8?q?=D0=A3=D0=B4=D0=B0=D0=BB=D0=B5=D0=BD=20?= =?UTF-8?q?=D0=BD=D0=B5=D1=80=D0=B0=D0=B1=D0=BE=D1=87=D0=B8=D0=B9=20=D0=BA?= =?UTF-8?q?=D0=BE=D0=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/controllers/WorkerController.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/controllers/WorkerController.js b/server/controllers/WorkerController.js index 86554ab4..00742c0e 100644 --- a/server/controllers/WorkerController.js +++ b/server/controllers/WorkerController.js @@ -47,10 +47,6 @@ class WorkerController extends BaseController { state = this.workerState.getState(request.workerId); if (!state) break; - if (i == 0) { - state = Object.assign({dummy: '0'.repeat(1024)}, state); - } - res.write(splitter + JSON.stringify(state)); res.flush(); From 67feee9aa1c1b1a73b5efd62fd210d253c4f3fec Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 16:53:34 +0700 Subject: [PATCH 19/58] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=20=D0=B1=D0=B0=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/ReaderWorker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index ef24a707..91fc911b 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -212,7 +212,7 @@ class ReaderWorker { while (i < files.length && size > maxSize) { const file = files[i]; const oldFile = `${dir}/${file.name}`; - if (this.remoteWebDavStorage) { + if (this.remoteWebDavStorage && dir === this.config.tempPublicDir) { try { //log(`remoteWebDavStorage.putFile ${path.basename(oldFile)}`); await this.remoteWebDavStorage.putFile(oldFile); From 15b0f05a059e784d1c5176117fa4174537126c2e Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 21 Jan 2020 16:55:41 +0700 Subject: [PATCH 20/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D0=BB=20=D0=BA=D0=BE=D0=BC=D0=BC=D0=B5=D0=BD=D1=82=D0=B0=D1=80?= =?UTF-8?q?=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/ReaderWorker.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 91fc911b..ab407406 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -212,6 +212,9 @@ class ReaderWorker { while (i < files.length && size > maxSize) { const file = files[i]; const oldFile = `${dir}/${file.name}`; + + //отправляем только this.config.tempPublicDir + //TODO: убрать в будущем, т.к. уже делается ленивое сохранение compFilename в удаленном хранилище if (this.remoteWebDavStorage && dir === this.config.tempPublicDir) { try { //log(`remoteWebDavStorage.putFile ${path.basename(oldFile)}`); From 59539e7e90ea388266b3fb98a598d4ade43e02ee Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 19:32:11 +0700 Subject: [PATCH 21/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20=D0=BA=D0=BB=D0=B0=D1=81=D1=81=20WebSocketConnec?= =?UTF-8?q?tion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/WebSocketConnection.js | 185 ++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 client/api/WebSocketConnection.js diff --git a/client/api/WebSocketConnection.js b/client/api/WebSocketConnection.js new file mode 100644 index 00000000..e39bd790 --- /dev/null +++ b/client/api/WebSocketConnection.js @@ -0,0 +1,185 @@ +const cleanPeriod = 60*1000;//1 минута + +class WebSocketConnection { + //messageLifeTime в минутах (cleanPeriod) + constructor(messageLifeTime = 5) { + this.ws = null; + this.timer = null; + this.listeners = []; + this.messageQueue = []; + this.messageLifeTime = messageLifeTime; + this.requestId = 0; + } + + addListener(listener) { + if (this.listeners.indexOf(listener) < 0) + this.listeners.push(Object.assign({regTime: Date.now()}, listener)); + } + + //рассылаем сообщение и удаляем те обработчики, которые его получили + emit(mes, isError) { + const len = this.listeners.length; + if (len > 0) { + let newListeners = []; + for (const listener of this.listeners) { + let emitted = false; + if (isError) { + if (listener.onError) + listener.onError(mes); + emitted = true; + } else { + if (listener.onMessage) { + if (listener.requestId) { + if (listener.requestId === mes.requestId) { + listener.onMessage(mes); + emitted = true; + } + } else { + listener.onMessage(mes); + emitted = true; + } + } else { + emitted = true; + } + } + + if (!emitted) + newListeners.push(listener); + } + this.listeners = newListeners; + } + + return this.listeners.length != len; + } + + open() { + return new Promise((resolve, reject) => { + if (this.ws && this.ws.readyState == WebSocket.OPEN) { + resolve(this.ws); + } else { + this.ws = new WebSocket(`ws://${window.location.host}`); + + if (this.timer) { + clearTimeout(this.timer); + } + this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod); + + let resolved = false; + this.ws.onopen = (e) => { + resolved = true; + resolve(e); + }; + + this.ws.onmessage = (e) => { + try { + const mes = JSON.parse(e.data); + this.messageQueue.push({regTime: Date.now(), mes}); + + let newMessageQueue = []; + for (const message of this.messageQueue) { + if (!this.emit(message.mes)) { + newMessageQueue.push(message); + } + } + + this.messageQueue = newMessageQueue; + } catch (e) { + this.emit(e.message, true); + } + }; + + this.ws.onerror = (e) => { + this.emit(e.message, true); + if (!resolved) + reject(e); + }; + } + }); + } + + //timeout в минутах (cleanPeriod) + message(timeout = 2) { + return new Promise((resolve, reject) => { + this.addListener({ + timeout, + onMessage: (mes) => { + resolve(mes); + }, + onError: (e) => { + reject(e); + } + }); + }); + } + + //timeout в минутах (cleanPeriod) + messageId(requestId, timeout = 2) { + return new Promise((resolve, reject) => { + this.addListener({ + requestId, + timeout, + onMessage: (mes) => { + resolve(mes); + }, + onError: (e) => { + reject(e); + } + }); + }); + } + + send(req) { + if (this.ws && this.ws.readyState == WebSocket.OPEN) { + this.ws.send(JSON.stringify(req)); + } else { + throw new Error('WebSocket connection is not ready'); + } + } + + sendId(req) { + if (this.ws && this.ws.readyState == WebSocket.OPEN) { + const requestId = ++this.requestId; + this.ws.send(Object.assign({requestId}, JSON.stringify(req))); + return requestId; + } else { + throw new Error('WebSocket connection is not ready'); + } + } + + close() { + if (this.ws && this.ws.readyState == WebSocket.OPEN) { + this.ws.close(); + } + } + + periodicClean() { + try { + this.timer = null; + + const now = Date.now(); + //чистка listeners + let newListeners = []; + for (const listener of this.listeners) { + if (now - listener.regTime < listener.timeout*cleanPeriod - 50) { + newListeners.push(listener); + } + } + this.listeners = newListeners; + + //чистка messageQueue + let newMessageQueue = []; + for (const message of this.messageQueue) { + if (now - message.regTime < this.messageLifeTime*cleanPeriod - 50) { + newMessageQueue.push(message); + } + } + this.messageQueue = newMessageQueue; + } finally { + if (this.ws.readyState == WebSocket.OPEN) { + this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod); + } + } + } +} + +export default WebSocketConnection; \ No newline at end of file From b6a000a001a22ed1162e71d05ab8ba031898d8e4 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 20:00:52 +0700 Subject: [PATCH 22/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20=D0=BF=D0=B0=D0=BA=D0=B5=D1=82=20ws?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 5 +++++ package.json | 1 + 2 files changed, 6 insertions(+) diff --git a/package-lock.json b/package-lock.json index f4d149a7..8131a63d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13075,6 +13075,11 @@ "mkdirp": "^0.5.1" } }, + "ws": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz", + "integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A==" + }, "xml2js": { "version": "0.4.23", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", diff --git a/package.json b/package.json index 3be87510..359d8191 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "vuex": "^3.1.1", "vuex-persistedstate": "^2.5.4", "webdav": "^2.10.1", + "ws": "^7.2.1", "zip-stream": "^2.1.2" } } From 10469bae7ba6aedc9536d0ad69e48c8f6f39d119 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 20:01:21 +0700 Subject: [PATCH 23/58] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B0=D1=8F=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/share/bookManager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/components/Reader/share/bookManager.js b/client/components/Reader/share/bookManager.js index 3ef65532..71c77381 100644 --- a/client/components/Reader/share/bookManager.js +++ b/client/components/Reader/share/bookManager.js @@ -464,7 +464,7 @@ class BookManager { addEventListener(listener) { if (this.eventListeners.indexOf(listener) < 0) - this.eventListeners.push(listener); + this.eventListeners.push(listener); } removeEventListener(listener) { From fbd50bad1d9f7a74ae07811c92df182a747f3250 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 20:02:05 +0700 Subject: [PATCH 24/58] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B1=D0=B0=D0=B3=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/WebSocketConnection.js | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/client/api/WebSocketConnection.js b/client/api/WebSocketConnection.js index e39bd790..69e54561 100644 --- a/client/api/WebSocketConnection.js +++ b/client/api/WebSocketConnection.js @@ -98,22 +98,7 @@ class WebSocketConnection { } //timeout в минутах (cleanPeriod) - message(timeout = 2) { - return new Promise((resolve, reject) => { - this.addListener({ - timeout, - onMessage: (mes) => { - resolve(mes); - }, - onError: (e) => { - reject(e); - } - }); - }); - } - - //timeout в минутах (cleanPeriod) - messageId(requestId, timeout = 2) { + message(requestId, timeout = 2) { return new Promise((resolve, reject) => { this.addListener({ requestId, @@ -129,17 +114,9 @@ class WebSocketConnection { } send(req) { - if (this.ws && this.ws.readyState == WebSocket.OPEN) { - this.ws.send(JSON.stringify(req)); - } else { - throw new Error('WebSocket connection is not ready'); - } - } - - sendId(req) { if (this.ws && this.ws.readyState == WebSocket.OPEN) { const requestId = ++this.requestId; - this.ws.send(Object.assign({requestId}, JSON.stringify(req))); + this.ws.send(JSON.stringify(Object.assign({requestId}, req))); return requestId; } else { throw new Error('WebSocket connection is not ready'); @@ -162,6 +139,9 @@ class WebSocketConnection { for (const listener of this.listeners) { if (now - listener.regTime < listener.timeout*cleanPeriod - 50) { newListeners.push(listener); + } else { + if (listener.onError) + listener.onError('Время ожидания ответа истекло'); } } this.listeners = newListeners; From 7dad47b3c8f96d41119788dd6f0ec9d74394751a Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 20:02:42 +0700 Subject: [PATCH 25/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=BE=20=D0=B8=D1=81=D0=BF=D0=BE=D0=BB=D1=8C=D0=B7?= =?UTF-8?q?=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20WebSocketConnection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 71 +++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/client/api/reader.js b/client/api/reader.js index 62d1ab23..7531b798 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -1,4 +1,6 @@ import axios from 'axios'; +import * as utils from '../share/utils'; +import WebSocketConnection from './WebSocketConnection'; const api = axios.create({ baseURL: '/api/reader' @@ -9,44 +11,57 @@ const workerApi = axios.create({ }); class Reader { + constructor() { + this.wsc = new WebSocketConnection(); + } async getStateFinish(workerId, callback) { if (!callback) callback = () => {}; - //присылается текст, состоящий из json-объектов state каждые 300ms, с разделителем splitter между ними - const splitter = '-- aod2t5hDXU32bUFyqlFE next status --'; - let lastIndex = 0; - let response = await workerApi.post('/get-state-finish', {workerId}, { - onDownloadProgress: progress => { - //небольая оптимизация, вместо простого responseText.split - const xhr = progress.target; - let currIndex = xhr.responseText.length; - if (lastIndex == currIndex) - return; - const last = xhr.responseText.substring(lastIndex, currIndex); - lastIndex = currIndex; + let response = {}; - //быстрее будет last.split - const res = last.split(splitter).pop(); - if (res) { - try { - callback(JSON.parse(res)); - } catch (e) { - // - } + try { + const wsc = this.wsc; + await wsc.open(); + const requestId = wsc.send({action: 'worker-get-state-finish', workerId}); + + while (1) {// eslint-disable-line no-constant-condition + response = await wsc.message(requestId); + callback(response); + + if (response.state == 'finish' || response.state == 'error') { + break; } } - }); + return response; + } catch (e) { + // + } - //берем последний state - response = response.data.split(splitter).pop(); + //с WebSocket проблема, проверяем по http + const refreshPause = 500; + let i = 0; + response = {}; + while (1) {// eslint-disable-line no-constant-condition + const prevProgress = response.progress || 0; + const prevState = response.state || 0; + response = await workerApi.post('/get-state', {workerId}); + response = response.data; + callback(response); - if (response) { - try { - response = JSON.parse(response); - } catch (e) { - response = false; + if (response.state == 'finish' || response.state == 'error') { + break; } + + if (i > 0) + await utils.sleep(refreshPause); + + i++; + if (i > 120*1000/refreshPause) {//2 мин ждем телодвижений воркера + throw new Error('Слишком долгое время ожидания'); + } + //проверка воркера + i = (prevProgress != response.progress || prevState != response.state ? 1 : i); } return response; From 3199af570d574e8f123ab0648e5f603c513a043d Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 20:06:51 +0700 Subject: [PATCH 26/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20WebSocketServer=20=D0=B8=20=D0=BA=D0=BE=D0=BD?= =?UTF-8?q?=D1=82=D1=80=D0=BE=D0=BB=D0=BB=D0=B5=D1=80=20=D0=B4=D0=BB=D1=8F?= =?UTF-8?q?=20=D0=BD=D0=B5=D0=B3=D0=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/controllers/WebSocketController.js | 111 ++++++++++++++++++++++ server/controllers/WorkerController.js | 1 + server/controllers/index.js | 1 + server/index.js | 15 ++- server/routes.js | 3 +- 5 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 server/controllers/WebSocketController.js diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js new file mode 100644 index 00000000..e31aacb7 --- /dev/null +++ b/server/controllers/WebSocketController.js @@ -0,0 +1,111 @@ +const WebSocket = require ('ws'); +const WorkerState = require('../core/WorkerState');//singleton +const utils = require('../core/utils'); + +const cleanPeriod = 1*60*1000;//1 минута +const closeSocketOnIdle = 5*60*1000;//5 минут + +class WebSocketController { + constructor(wss, config) { + this.config = config; + this.workerState = new WorkerState(); + + this.wss = wss; + + wss.on('connection', (ws) => { + ws.on('message', (message) => { + this.onMessage(ws, message); + }); + }); + + setTimeout(() => { this.periodicClean(); }, cleanPeriod); + } + + periodicClean() { + try { + const now = Date.now(); + this.wss.clients.forEach((ws) => { + if (!ws.lastActivity || now - ws.lastActivity > closeSocketOnIdle - 50) { + ws.terminate(); + } + }); + } finally { + setTimeout(() => { this.periodicClean(); }, cleanPeriod); + } + } + + async onMessage(ws, message) { + let req = {}; + try { + ws.lastActivity = Date.now(); + req = JSON.parse(message); + switch (req.action) { + case 'test': + this.test(req, ws); break; + case 'worker-get-state': + this.workerGetState(req, ws); break; + case 'worker-get-state-finish': + this.workerGetStateFinish(req, ws); break; + + default: + throw new Error(`Action not found: ${req.action}`); + } + } catch (e) { + this.send({error: e.message}, req, ws); + } + } + + send(res, req, ws) { + if (ws.readyState == WebSocket.OPEN) { + ws.lastActivity = Date.now(); + let r = Object.assign({}, res); + if (req.requestId) + r.requestId = req.requestId; + ws.send(JSON.stringify(r)); + } + } + + //Actions + async test(req, ws) { + this.send({message: 'Liberama project is awesome'}, req, ws); + } + + async workerGetState(req, ws) { + if (!req.workerId) + throw new Error(`key 'workerId' is wrong`); + + const state = this.workerState.getState(req.workerId); + this.send((state ? state : {}), req, ws); + } + + async workerGetStateFinish(req, ws) { + if (!req.workerId) + throw new Error(`key 'workerId' is wrong`); + + const refreshPause = 200; + let i = 0; + let state = {}; + while (1) {// eslint-disable-line no-constant-condition + const prevProgress = state.progress || -1; + const prevState = state.state || ''; + state = this.workerState.getState(req.workerId); + + this.send((state ? state : {}), req, ws); + if (!state) break; + + if (state.state != 'finish' && state.state != 'error') + await utils.sleep(refreshPause); + else + break; + + i++; + if (i > 2*60*1000/refreshPause) {//2 мин ждем телодвижений воркера + this.send({state: 'error', error: 'Время ожидания процесса истекло'}, req, ws); + } + i = (prevProgress != state.progress || prevState != state.state ? 1 : i); + } + } + +} + +module.exports = WebSocketController; diff --git a/server/controllers/WorkerController.js b/server/controllers/WorkerController.js index 00742c0e..e0ef77a9 100644 --- a/server/controllers/WorkerController.js +++ b/server/controllers/WorkerController.js @@ -26,6 +26,7 @@ class WorkerController extends BaseController { return false; } + //TODO: удалить бесполезную getStateFinish async getStateFinish(req, res) { const request = req.body; let error = ''; diff --git a/server/controllers/index.js b/server/controllers/index.js index fbbad7b7..9c2f4f07 100644 --- a/server/controllers/index.js +++ b/server/controllers/index.js @@ -2,4 +2,5 @@ module.exports = { MiscController: require('./MiscController'), ReaderController: require('./ReaderController'), WorkerController: require('./WorkerController'), + WebSocketController: require('./WebSocketController'), } \ No newline at end of file diff --git a/server/index.js b/server/index.js index 94bf47d9..9a15a6b2 100644 --- a/server/index.js +++ b/server/index.js @@ -4,6 +4,8 @@ const path = require('path'); const argv = require('minimist')(process.argv.slice(2)); const express = require('express'); const compression = require('compression'); +const http = require('http'); +const WebSocket = require ('ws'); async function init() { //config @@ -46,10 +48,13 @@ async function main() { const config = new (require('./config'))().config;//singleton //servers - for (let server of config.servers) { - if (server.mode !== 'none') { + for (let serverCfg of config.servers) { + if (serverCfg.mode !== 'none') { const app = express(); - const serverConfig = Object.assign({}, config, server); + const server = http.createServer(app); + const wss = new WebSocket.Server({ server, maxPayload: 10*1024*1024 }); + + const serverConfig = Object.assign({}, config, serverCfg); let devModule = undefined; if (serverConfig.branch == 'development') { @@ -73,7 +78,7 @@ async function main() { } })); - require('./routes').initRoutes(app, serverConfig); + require('./routes').initRoutes(app, wss, serverConfig); if (devModule) { devModule.logErrors(app); @@ -84,7 +89,7 @@ async function main() { }); } - app.listen(serverConfig.port, serverConfig.ip, function() { + server.listen(serverConfig.port, serverConfig.ip, function() { log(`Server-${serverConfig.serverName} is ready on ${serverConfig.ip}:${serverConfig.port}, mode: ${serverConfig.mode}`); }); } diff --git a/server/routes.js b/server/routes.js index 09b96239..93eb129a 100644 --- a/server/routes.js +++ b/server/routes.js @@ -2,10 +2,11 @@ const c = require('./controllers'); const utils = require('./core/utils'); const multer = require('multer'); -function initRoutes(app, config) { +function initRoutes(app, wss, config) { const misc = new c.MiscController(config); const reader = new c.ReaderController(config); const worker = new c.WorkerController(config); + new c.WebSocketController(wss, config); //access const [aAll, aNormal, aSite, aReader, aOmnireader] = // eslint-disable-line no-unused-vars From 4cccb56ee3a0663aeefa2672c259e919d5c9d101 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 20:15:33 +0700 Subject: [PATCH 27/58] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D0=BB=20=D0=BA=D0=BE=D0=BC=D0=BC=D0=B5=D0=BD=D1=82=D0=B0?= =?UTF-8?q?=D1=80=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/controllers/WebSocketController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js index e31aacb7..1c9728c9 100644 --- a/server/controllers/WebSocketController.js +++ b/server/controllers/WebSocketController.js @@ -65,7 +65,7 @@ class WebSocketController { } } - //Actions + //Actions ------------------------------------------------------------------ async test(req, ws) { this.send({message: 'Liberama project is awesome'}, req, ws); } From e36dc4a913905e83b5ada7e85601e1f379dda568 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 20:28:46 +0700 Subject: [PATCH 28/58] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D1=88=D0=B8=D0=B5=20=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA?= =?UTF-8?q?=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/WebSocketConnection.js | 5 +++-- client/api/reader.js | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/api/WebSocketConnection.js b/client/api/WebSocketConnection.js index 69e54561..e212311b 100644 --- a/client/api/WebSocketConnection.js +++ b/client/api/WebSocketConnection.js @@ -52,12 +52,13 @@ class WebSocketConnection { return this.listeners.length != len; } - open() { + open(url) { return new Promise((resolve, reject) => { if (this.ws && this.ws.readyState == WebSocket.OPEN) { resolve(this.ws); } else { - this.ws = new WebSocket(`ws://${window.location.host}`); + url = url || `ws://${window.location.host}/ws`; + this.ws = new WebSocket(url); if (this.timer) { clearTimeout(this.timer); diff --git a/client/api/reader.js b/client/api/reader.js index 7531b798..47b61c6e 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -36,6 +36,7 @@ class Reader { return response; } catch (e) { // + console.error(e); } //с WebSocket проблема, проверяем по http From 9c13261929caef092d2c9fb77c2cc3242362af38 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 20:58:57 +0700 Subject: [PATCH 29/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=BD=D0=B0=D1=81=D1=82=D1=80=D0=BE=D0=B9?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=B4=D0=BB=D1=8F=20=D0=B2=D0=B5=D0=B1=D1=81?= =?UTF-8?q?=D0=BE=D0=BA=D0=B5=D1=82=D0=BE=D0=B2,=20=D0=B4=D0=BE=D0=B1?= =?UTF-8?q?=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=20=D0=BA=D0=BE=D0=BD=D1=84=D0=B8?= =?UTF-8?q?=D0=B3=20nginx=20omnireader=5Fhttp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/omnireader/omnireader | 7 ++++ docs/omnireader/omnireader_http | 58 +++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 docs/omnireader/omnireader_http diff --git a/docs/omnireader/omnireader b/docs/omnireader/omnireader index a638b392..dc4bba13 100644 --- a/docs/omnireader/omnireader +++ b/docs/omnireader/omnireader @@ -18,6 +18,13 @@ server { proxy_pass http://127.0.0.1:44081; } + location /ws { + proxy_pass http://127.0.0.1:44081; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location / { root /home/liberama/public; diff --git a/docs/omnireader/omnireader_http b/docs/omnireader/omnireader_http new file mode 100644 index 00000000..a5d84d83 --- /dev/null +++ b/docs/omnireader/omnireader_http @@ -0,0 +1,58 @@ +server { + listen 80; + server_name omnireader.ru; + + client_max_body_size 50m; + + gzip on; + gzip_min_length 1024; + gzip_proxied expired no-cache no-store private auth; + gzip_types *; + + location /api { + proxy_pass http://127.0.0.1:44081; + } + + location /ws { + proxy_pass http://127.0.0.1:44081; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + location / { + root /home/liberama/public; + + location /tmp { + add_header Content-Type text/xml; + add_header Content-Encoding gzip; + } + + location ~* \.(?:manifest|appcache|html)$ { + expires -1; + } + } +} + +server { + listen 80; + server_name old.omnireader.ru; + + client_max_body_size 50m; + + gzip on; + gzip_min_length 1024; + gzip_proxied expired no-cache no-store private auth; + gzip_types *; + + root /home/oldreader; + + index index.html; + + # Обработка php файлов с помощью fpm + location ~ \.php$ { + try_files $uri =404; + include /etc/nginx/fastcgi.conf; + fastcgi_pass unix:/run/php/php7.2-fpm.sock; + } +} From 4972f085a30b7635dc01b273a3b8428b4acd47df Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 20:59:52 +0700 Subject: [PATCH 30/58] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B0=D1=8F=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/WebSocketConnection.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client/api/WebSocketConnection.js b/client/api/WebSocketConnection.js index e212311b..49babb65 100644 --- a/client/api/WebSocketConnection.js +++ b/client/api/WebSocketConnection.js @@ -58,6 +58,7 @@ class WebSocketConnection { resolve(this.ws); } else { url = url || `ws://${window.location.host}/ws`; + this.ws = new WebSocket(url); if (this.timer) { From fdbf508bbf329bb5b4876cf13c579b2d22715307 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 21:17:10 +0700 Subject: [PATCH 31/58] =?UTF-8?q?=D0=98=D1=81=D0=BF=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D0=B7=D1=83=D0=B5=D0=BC=20=D0=BF=D1=80=D0=BE=D1=82=D0=BE=D0=BA?= =?UTF-8?q?=D0=BE=D0=BB=20WSS=20=D0=BF=D1=80=D0=B8=20=D0=BD=D0=B5=D0=BE?= =?UTF-8?q?=D0=B1=D1=85=D0=BE=D0=B4=D0=B8=D0=BC=D0=BE=D1=81=D1=82=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/WebSocketConnection.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/client/api/WebSocketConnection.js b/client/api/WebSocketConnection.js index 49babb65..a3a67f2b 100644 --- a/client/api/WebSocketConnection.js +++ b/client/api/WebSocketConnection.js @@ -57,7 +57,12 @@ class WebSocketConnection { if (this.ws && this.ws.readyState == WebSocket.OPEN) { resolve(this.ws); } else { - url = url || `ws://${window.location.host}/ws`; + let protocol = 'ws:'; + if (window.location.protocol == 'https:') { + protocol = 'wss:' + } + + url = url || `${protocol}//${window.location.host}/ws`; this.ws = new WebSocket(url); From be86a1535186e12a7ad8876d7dc5f88edd63e522 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 21:37:28 +0700 Subject: [PATCH 32/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D0=BB=20=D0=BD=D0=B0=D1=81=D1=82=D1=80=D0=BE=D0=B9=D0=BA=D1=83?= =?UTF-8?q?=20proxy=5Fread=5Ftimeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/omnireader/omnireader | 1 + docs/omnireader/omnireader_http | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/omnireader/omnireader b/docs/omnireader/omnireader index dc4bba13..3e535374 100644 --- a/docs/omnireader/omnireader +++ b/docs/omnireader/omnireader @@ -8,6 +8,7 @@ server { server_name omnireader.ru; client_max_body_size 50m; + proxy_read_timeout 1h; gzip on; gzip_min_length 1024; diff --git a/docs/omnireader/omnireader_http b/docs/omnireader/omnireader_http index a5d84d83..768fce76 100644 --- a/docs/omnireader/omnireader_http +++ b/docs/omnireader/omnireader_http @@ -3,6 +3,7 @@ server { server_name omnireader.ru; client_max_body_size 50m; + proxy_read_timeout 1h; gzip on; gzip_min_length 1024; From 12e7a783b096210b932f50ee9f41f499249edf33 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 22 Jan 2020 22:06:12 +0700 Subject: [PATCH 33/58] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D1=88=D0=B8=D0=B5=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=B1=D0=BB=D0=BE=D0=BA=D0=B8=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=BA=D0=BD=D0=BE=D0=BF=D0=BE=D0=BA?= =?UTF-8?q?=20=D0=BF=D0=B0=D0=BD=D0=B5=D0=BB=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/Reader.vue | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/client/components/Reader/Reader.vue b/client/components/Reader/Reader.vue index 0a6dab35..ee89f5af 100644 --- a/client/components/Reader/Reader.vue +++ b/client/components/Reader/Reader.vue @@ -719,15 +719,16 @@ class Reader extends Vue { case 'scrolling': case 'search': case 'copyText': - case 'recentBooks': + case 'refresh': case 'offlineMode': + case 'recentBooks': case 'settings': - if (this[`${button}Active`]) + if (this.progressActive) { + classResult = classDisabled; + } else if (this[`${button}Active`]) { classResult = classActive; + } break; - } - - switch (button) { case 'undoAction': if (this.actionCur <= 0) classResult = classDisabled; From 8df80ce7384ba7ac615e3b4bc179e59f0d437b40 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 23 Jan 2020 15:16:49 +0700 Subject: [PATCH 34/58] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B0=D1=8F=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/api/reader.js b/client/api/reader.js index 47b61c6e..6149a1fc 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -171,7 +171,7 @@ class Reader { //получение файла estSize = (estSize > 0 ? estSize : 1000000); const options = { - onDownloadProgress: progress => { + onDownloadProgress: (progress) => { while (progress.loaded > estSize) estSize *= 1.5; if (callback) From 00cc63b7cd8ed504f1ee0aac5d2cd81da7d3cebc Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 23 Jan 2020 15:54:46 +0700 Subject: [PATCH 35/58] =?UTF-8?q?WebSocket:=20=D0=B4=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=B2=D0=BB=D0=B5=D0=BD=20=D0=BC=D0=B5=D1=82=D0=BE=D0=B4=20get?= =?UTF-8?q?-config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/misc.js | 16 ++++++++++++++-- client/api/reader.js | 7 ++----- ...ocketConnection.js => webSocketConnection.js} | 2 +- server/controllers/WebSocketController.js | 12 ++++++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) rename client/api/{WebSocketConnection.js => webSocketConnection.js} (99%) diff --git a/client/api/misc.js b/client/api/misc.js index 79d56d0e..1e63138a 100644 --- a/client/api/misc.js +++ b/client/api/misc.js @@ -1,4 +1,5 @@ import axios from 'axios'; +import wsc from './webSocketConnection'; const api = axios.create({ baseURL: '/api' @@ -6,9 +7,20 @@ const api = axios.create({ class Misc { async loadConfig() { - const response = await api.post('/config', {params: [ + + const query = {params: [ 'name', 'version', 'mode', 'maxUploadFileSize', 'useExternalBookConverter', 'branch', - ]}); + ]}; + + try { + await wsc.open(); + return await wsc.message(wsc.send(Object.assign({action: 'get-config'}, query))); + } catch (e) { + console.error(e); + } + + //если WebSocket проблема, работаем по http + const response = await api.post('/config', query); return response.data; } } diff --git a/client/api/reader.js b/client/api/reader.js index 6149a1fc..1ff1e187 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -1,6 +1,6 @@ import axios from 'axios'; import * as utils from '../share/utils'; -import WebSocketConnection from './WebSocketConnection'; +import wsc from './webSocketConnection'; const api = axios.create({ baseURL: '/api/reader' @@ -12,7 +12,6 @@ const workerApi = axios.create({ class Reader { constructor() { - this.wsc = new WebSocketConnection(); } async getStateFinish(workerId, callback) { @@ -21,7 +20,6 @@ class Reader { let response = {}; try { - const wsc = this.wsc; await wsc.open(); const requestId = wsc.send({action: 'worker-get-state-finish', workerId}); @@ -35,11 +33,10 @@ class Reader { } return response; } catch (e) { - // console.error(e); } - //с WebSocket проблема, проверяем по http + //если WebSocket проблема, работаем по http const refreshPause = 500; let i = 0; response = {}; diff --git a/client/api/WebSocketConnection.js b/client/api/webSocketConnection.js similarity index 99% rename from client/api/WebSocketConnection.js rename to client/api/webSocketConnection.js index a3a67f2b..e5b79789 100644 --- a/client/api/WebSocketConnection.js +++ b/client/api/webSocketConnection.js @@ -169,4 +169,4 @@ class WebSocketConnection { } } -export default WebSocketConnection; \ No newline at end of file +export default new WebSocketConnection(); \ No newline at end of file diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js index 1c9728c9..a3f18098 100644 --- a/server/controllers/WebSocketController.js +++ b/server/controllers/WebSocketController.js @@ -1,4 +1,6 @@ const WebSocket = require ('ws'); +const _ = require('lodash'); + const WorkerState = require('../core/WorkerState');//singleton const utils = require('../core/utils'); @@ -42,6 +44,8 @@ class WebSocketController { switch (req.action) { case 'test': this.test(req, ws); break; + case 'get-config': + this.getConfig(req, ws); break; case 'worker-get-state': this.workerGetState(req, ws); break; case 'worker-get-state-finish': @@ -70,6 +74,14 @@ class WebSocketController { this.send({message: 'Liberama project is awesome'}, req, ws); } + async getConfig(req, ws) { + if (Array.isArray(req.params)) { + this.send(_.pick(this.config, req.params), req, ws); + } else { + throw new Error('params is not an array'); + } + } + async workerGetState(req, ws) { if (!req.workerId) throw new Error(`key 'workerId' is wrong`); From f3da5a902662e35fcbe46d9beea0045411f4c8c5 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 23 Jan 2020 15:56:26 +0700 Subject: [PATCH 36/58] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D0=BB=20=D0=BA=D0=BE=D0=BC=D0=BC=D0=B5=D0=BD=D1=82=D0=B0?= =?UTF-8?q?=D1=80=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/misc.js | 2 +- client/api/reader.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/api/misc.js b/client/api/misc.js index 1e63138a..970b4799 100644 --- a/client/api/misc.js +++ b/client/api/misc.js @@ -19,7 +19,7 @@ class Misc { console.error(e); } - //если WebSocket проблема, работаем по http + //если с WebSocket проблема, работаем по http const response = await api.post('/config', query); return response.data; } diff --git a/client/api/reader.js b/client/api/reader.js index 1ff1e187..4ba39515 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -36,7 +36,7 @@ class Reader { console.error(e); } - //если WebSocket проблема, работаем по http + //если с WebSocket проблема, работаем по http const refreshPause = 500; let i = 0; response = {}; From 3456b3d90e82e41480d25ea2e04066e70c8bff21 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 23 Jan 2020 16:25:06 +0700 Subject: [PATCH 37/58] =?UTF-8?q?WebSocket:=20=D0=B4=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=B2=D0=BB=D0=B5=D0=BD=20=D0=BC=D0=B5=D1=82=D0=BE=D0=B4=20wor?= =?UTF-8?q?ker-get-state-finish,=20=D0=BD=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D1=88=D0=BE=D0=B9=20=D1=80=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE?= =?UTF-8?q?=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 79 ++++++++----------- .../RecentBooksPage/RecentBooksPage.vue | 2 +- server/controllers/WebSocketController.js | 14 ++++ 3 files changed, 46 insertions(+), 49 deletions(-) diff --git a/client/api/reader.js b/client/api/reader.js index 4ba39515..9156c36a 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -14,7 +14,7 @@ class Reader { constructor() { } - async getStateFinish(workerId, callback) { + async getWorkerStateFinish(workerId, callback) { if (!callback) callback = () => {}; let response = {}; @@ -77,12 +77,12 @@ class Reader { callback({totalSteps: 4}); callback(response.data); - response = await this.getStateFinish(workerId, callback); + response = await this.getWorkerStateFinish(workerId, callback); if (response) { if (response.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл callback({step: 4}); - const book = await this.loadCachedBook(response.path, callback, false, (response.size ? response.size : -1)); + const book = await this.loadCachedBook(response.path, callback, response.size); return Object.assign({}, response, {data: book.data}); } @@ -100,69 +100,52 @@ class Reader { } } - async checkUrl(url) { - let fileExists = false; + async checkCachedBook(url) { + let estSize = -1; try { - await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); - fileExists = true; + const response = await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); + + if (response.headers['content-length']) { + estSize = response.headers['content-length']; + } } catch (e) { - // - } + //восстановим при необходимости файл на сервере из удаленного облака + let response = null + + try { + await wsc.open(); + response = await wsc.message(wsc.send({action: 'reader-restore-cached-file', path: url})); + } catch (e) { + console.error(e); + //если с WebSocket проблема, работаем по http + response = await api.post('/restore-cached-file', {path: url}); + response = response.data; + } - //восстановим при необходимости файл на сервере из удаленного облака - if (!fileExists) { - let response = await api.post('/restore-cached-file', {path: url}); - - const workerId = response.data.workerId; + const workerId = response.workerId; if (!workerId) throw new Error('Неверный ответ api'); - response = await this.getStateFinish(workerId); + response = await this.getWorkerStateFinish(workerId); if (response.state == 'error') { throw new Error(response.error); } + if (response.size && estSize < 0) { + estSize = response.size; + } } - return true; + return estSize; } - async loadCachedBook(url, callback, restore = true, estSize = -1) { + async loadCachedBook(url, callback, estSize = -1) { if (!callback) callback = () => {}; - let response = null; callback({state: 'loading', progress: 0}); //получение размера файла - let fileExists = false; - if (estSize < 0) { - try { - response = await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); - - if (response.headers['content-length']) { - estSize = response.headers['content-length']; - } - fileExists = true; - } catch (e) { - // - } - } - - //восстановим при необходимости файл на сервере из удаленного облака - if (restore && !fileExists) { - response = await api.post('/restore-cached-file', {path: url}); - - const workerId = response.data.workerId; - if (!workerId) - throw new Error('Неверный ответ api'); - - response = await this.getStateFinish(workerId); - if (response.state == 'error') { - throw new Error(response.error); - } - - if (response.size && estSize < 0) { - estSize = response.size; - } + if (estSize && estSize < 0) { + estSize = await this.checkCachedBook(url); } //получение файла diff --git a/client/components/Reader/RecentBooksPage/RecentBooksPage.vue b/client/components/Reader/RecentBooksPage/RecentBooksPage.vue index e3511dbe..312f5020 100644 --- a/client/components/Reader/RecentBooksPage/RecentBooksPage.vue +++ b/client/components/Reader/RecentBooksPage/RecentBooksPage.vue @@ -272,7 +272,7 @@ class RecentBooksPage extends Vue { async downloadBook(fb2path) { try { - await readerApi.checkUrl(fb2path); + await readerApi.checkCachedBook(fb2path); const d = this.$refs.download; d.href = fb2path; diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js index a3f18098..2bd599e8 100644 --- a/server/controllers/WebSocketController.js +++ b/server/controllers/WebSocketController.js @@ -1,6 +1,8 @@ const WebSocket = require ('ws'); const _ = require('lodash'); +const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton +const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton const WorkerState = require('../core/WorkerState');//singleton const utils = require('../core/utils'); @@ -10,6 +12,8 @@ const closeSocketOnIdle = 5*60*1000;//5 минут class WebSocketController { constructor(wss, config) { this.config = config; + this.readerStorage = new ReaderStorage(); + this.readerWorker = new ReaderWorker(config); this.workerState = new WorkerState(); this.wss = wss; @@ -50,6 +54,8 @@ class WebSocketController { this.workerGetState(req, ws); break; case 'worker-get-state-finish': this.workerGetStateFinish(req, ws); break; + case 'reader-restore-cached-file': + this.readerRestoreCachedFile(req, ws); break; default: throw new Error(`Action not found: ${req.action}`); @@ -118,6 +124,14 @@ class WebSocketController { } } + async readerRestoreCachedFile(req, ws) { + if (!req.path) + throw new Error(`key 'path' is empty`); + + const workerId = this.readerWorker.restoreCachedFile(req.path); + const state = this.workerState.getState(workerId); + this.send((state ? state : {}), req, ws); + } } module.exports = WebSocketController; From 17670aabf9a8e96aaa9434585e514bb3c2526344 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 23 Jan 2020 16:59:08 +0700 Subject: [PATCH 38/58] =?UTF-8?q?WebSocket:=20=D0=B4=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=B2=D0=BB=D0=B5=D0=BD=20=D0=BC=D0=B5=D1=82=D0=BE=D0=B4=20rea?= =?UTF-8?q?der-storage,=20=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B8?= =?UTF-8?q?=20=D0=B1=D0=B0=D0=B3=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 15 ++++++-- client/api/webSocketConnection.js | 6 +++- server/controllers/WebSocketController.js | 43 ++++++++++++++++++----- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/client/api/reader.js b/client/api/reader.js index 9156c36a..2409aefa 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -195,13 +195,22 @@ class Reader { } async storage(request) { - let response = await api.post('/storage', request); + let response = null; + try { + await wsc.open(); + response = await wsc.message(wsc.send({action: 'reader-storage', body: request})); + } catch (e) { + console.error(e); + //если с WebSocket проблема, работаем по http + response = await api.post('/storage', request); + response = response.data; + } - const state = response.data.state; + const state = response.state; if (!state) throw new Error('Неверный ответ api'); - return response.data; + return response; } } diff --git a/client/api/webSocketConnection.js b/client/api/webSocketConnection.js index e5b79789..bbcb21a3 100644 --- a/client/api/webSocketConnection.js +++ b/client/api/webSocketConnection.js @@ -111,7 +111,11 @@ class WebSocketConnection { requestId, timeout, onMessage: (mes) => { - resolve(mes); + if (mes.error) { + reject(mes.error); + } else { + resolve(mes); + } }, onError: (e) => { reject(e); diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js index 2bd599e8..87659128 100644 --- a/server/controllers/WebSocketController.js +++ b/server/controllers/WebSocketController.js @@ -4,6 +4,7 @@ const _ = require('lodash'); const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton const WorkerState = require('../core/WorkerState');//singleton +const log = new (require('../core/AppLogger'))().log;//singleton const utils = require('../core/utils'); const cleanPeriod = 1*60*1000;//1 минута @@ -12,6 +13,8 @@ const closeSocketOnIdle = 5*60*1000;//5 минут class WebSocketController { constructor(wss, config) { this.config = config; + this.isDevelopment = (config.branch == 'development'); + this.readerStorage = new ReaderStorage(); this.readerWorker = new ReaderWorker(config); this.workerState = new WorkerState(); @@ -43,19 +46,25 @@ class WebSocketController { async onMessage(ws, message) { let req = {}; try { + if (this.isDevelopment) { + log(`WebSocket-IN: ${message.substr(0, 4000)}`); + } + ws.lastActivity = Date.now(); req = JSON.parse(message); switch (req.action) { case 'test': - this.test(req, ws); break; + await this.test(req, ws); break; case 'get-config': - this.getConfig(req, ws); break; + await this.getConfig(req, ws); break; case 'worker-get-state': - this.workerGetState(req, ws); break; + await this.workerGetState(req, ws); break; case 'worker-get-state-finish': - this.workerGetStateFinish(req, ws); break; + await this.workerGetStateFinish(req, ws); break; case 'reader-restore-cached-file': - this.readerRestoreCachedFile(req, ws); break; + await this.readerRestoreCachedFile(req, ws); break; + case 'reader-storage': + await this.readerStorageDo(req, ws); break; default: throw new Error(`Action not found: ${req.action}`); @@ -68,10 +77,17 @@ class WebSocketController { send(res, req, ws) { if (ws.readyState == WebSocket.OPEN) { ws.lastActivity = Date.now(); - let r = Object.assign({}, res); + let r = res; if (req.requestId) - r.requestId = req.requestId; - ws.send(JSON.stringify(r)); + r = Object.assign({requestId: req.requestId}, r); + + const message = JSON.stringify(r); + ws.send(message); + + if (this.isDevelopment) { + log(`WebSocket-OUT: ${message.substr(0, 4000)}`); + } + } } @@ -132,6 +148,17 @@ class WebSocketController { const state = this.workerState.getState(workerId); this.send((state ? state : {}), req, ws); } + + async readerStorageDo(req, ws) { + if (!req.body) + throw new Error(`key 'body' is empty`); + if (!req.body.action) + throw new Error(`key 'action' is empty`); + if (!req.body.items || Array.isArray(req.body.data)) + throw new Error(`key 'items' is empty`); + + this.send(await this.readerStorage.doAction(req.body), req, ws); + } } module.exports = WebSocketController; From 1d352a76ceafc4eab8d3834bb76f77e1458e2c9d Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 23 Jan 2020 17:00:17 +0700 Subject: [PATCH 39/58] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BE=D0=BF=D0=B5=D1=87=D0=B0=D1=82=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/controllers/ReaderController.js | 4 ++-- server/controllers/WebSocketController.js | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/controllers/ReaderController.js b/server/controllers/ReaderController.js index 09596225..5e9a4879 100644 --- a/server/controllers/ReaderController.js +++ b/server/controllers/ReaderController.js @@ -35,9 +35,9 @@ class ReaderController extends BaseController { const request = req.body; let error = ''; try { - if (!request.action) + if (!request.action) throw new Error(`key 'action' is empty`); - if (!request.items || Array.isArray(request.data)) + if (!request.items || Array.isArray(request.data)) throw new Error(`key 'items' is empty`); return await this.readerStorage.doAction(request); diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js index 87659128..33b61a0b 100644 --- a/server/controllers/WebSocketController.js +++ b/server/controllers/WebSocketController.js @@ -150,11 +150,11 @@ class WebSocketController { } async readerStorageDo(req, ws) { - if (!req.body) + if (!req.body) throw new Error(`key 'body' is empty`); - if (!req.body.action) + if (!req.body.action) throw new Error(`key 'action' is empty`); - if (!req.body.items || Array.isArray(req.body.data)) + if (!req.body.items || Array.isArray(req.body.data)) throw new Error(`key 'items' is empty`); this.send(await this.readerStorage.doAction(req.body), req, ws); From 2569d00bd0f8bee2ce3d082bf365b87c0daaef6f Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 26 Jan 2020 13:47:25 +0700 Subject: [PATCH 40/58] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B8=D0=B5=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/api/reader.js b/client/api/reader.js index 2409aefa..c17cccef 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -27,6 +27,9 @@ class Reader { response = await wsc.message(requestId); callback(response); + if (!response.state) + throw new Error('Неверный ответ api'); + if (response.state == 'finish' || response.state == 'error') { break; } @@ -47,6 +50,9 @@ class Reader { response = response.data; callback(response); + if (!response.state) + throw new Error('Неверный ответ api'); + if (response.state == 'finish' || response.state == 'error') { break; } From 7997c486cf406dea4dd31757c99703f84d33bcf1 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 26 Jan 2020 15:07:14 +0700 Subject: [PATCH 41/58] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B8=D0=B9=20?= =?UTF-8?q?=D1=80=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE=D1=80=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/FileDownloader.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/server/core/FileDownloader.js b/server/core/FileDownloader.js index 78c0df3a..d6c53897 100644 --- a/server/core/FileDownloader.js +++ b/server/core/FileDownloader.js @@ -1,9 +1,8 @@ const got = require('got'); -const maxDownloadSize = 50*1024*1024; - class FileDownloader { - constructor() { + constructor(limitDownloadSize = 0) { + this.limitDownloadSize = limitDownloadSize; } async load(url, callback) { @@ -24,9 +23,11 @@ class FileDownloader { let prevProg = 0; const request = got(url, options).on('downloadProgress', progress => { - if (progress.transferred > maxDownloadSize) { - errMes = 'file too big'; - request.cancel(); + if (this.limitDownloadSize) { + if (progress.transferred > this.limitDownloadSize) { + errMes = 'Файл слишком большой'; + request.cancel(); + } } let prog = 0; From 639f726c837675bd21eedef07e3175251260ac8f Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 26 Jan 2020 15:17:45 +0700 Subject: [PATCH 42/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20=D0=BB=D0=B8=D0=BC=D0=B8=D1=82=20=D0=BD=D0=B0=20?= =?UTF-8?q?=D1=80=D0=B0=D0=B7=D0=BC=D0=B5=D1=80=20=D1=84=D0=B0=D0=B9=D0=BB?= =?UTF-8?q?=D0=B0=20=D0=BF=D1=80=D0=B8=20=D1=80=D0=B0=D1=81=D0=BF=D0=B0?= =?UTF-8?q?=D0=BA=D0=BE=D0=B2=D0=BA=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/FileDecompressor.js | 26 ++++++++++++++++++++++---- server/core/Reader/ReaderWorker.js | 4 ++-- server/core/ZipStreamer.js | 11 ++++++++++- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/server/core/FileDecompressor.js b/server/core/FileDecompressor.js index 6460cf46..cc7a4c0f 100644 --- a/server/core/FileDecompressor.js +++ b/server/core/FileDecompressor.js @@ -10,8 +10,9 @@ const utils = require('./utils'); const FileDetector = require('./FileDetector'); class FileDecompressor { - constructor() { + constructor(limitFileSize = 0) { this.detector = new FileDetector(); + this.limitFileSize = limitFileSize; } async decompressNested(filename, outputDir) { @@ -113,7 +114,7 @@ class FileDecompressor { async unZip(filename, outputDir) { const zip = new ZipStreamer(); - return await zip.unpack(filename, outputDir); + return await zip.unpack(filename, outputDir, null, this.limitFileSize); } unBz2(filename, outputDir) { @@ -125,9 +126,16 @@ class FileDecompressor { } unTar(filename, outputDir) { - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { (async() => { const files = []; + if (this.limitFileSize) { + if ((await fs.stat(filename)).size > this.limitFileSize) { + reject('Файл слишком большой'); + return; + } + } + const tarExtract = tar.extract(outputDir, { map: (header) => { files.push({path: header.name, size: header.size}); @@ -149,7 +157,7 @@ class FileDecompressor { }); inputStream.pipe(tarExtract); - }); + })().catch(reject); }); } decompressByStream(stream, filename, outputDir) { @@ -174,6 +182,16 @@ class FileDecompressor { }); stream.on('error', reject); + + if (this.limitFileSize) { + let readSize = 0; + stream.on('data', (buffer) => { + readSize += buffer.length; + if (readSize > this.limitFileSize) + stream.destroy(new Error('Файл слишком большой')); + }); + } + inputStream.on('error', reject); outputStream.on('error', reject); diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index ab407406..74c7eb2c 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -27,8 +27,8 @@ class ReaderWorker { fs.ensureDirSync(this.config.tempPublicDir); this.workerState = new WorkerState(); - this.down = new FileDownloader(); - this.decomp = new FileDecompressor(); + this.down = new FileDownloader(config.maxUploadFileSize); + this.decomp = new FileDecompressor(2*config.maxUploadFileSize); this.bookConverter = new BookConverter(this.config); this.remoteWebDavStorage = false; diff --git a/server/core/ZipStreamer.js b/server/core/ZipStreamer.js index 03f59ff5..c761eaa9 100644 --- a/server/core/ZipStreamer.js +++ b/server/core/ZipStreamer.js @@ -52,7 +52,7 @@ class ZipStreamer { })().catch(reject); }); } - unpack(zipFile, outputDir, entryCallback) { + unpack(zipFile, outputDir, entryCallback, limitFileSize = 0) { return new Promise((resolve, reject) => { entryCallback = (entryCallback ? entryCallback : () => {}); const unzip = new unzipStream({file: zipFile}); @@ -67,6 +67,15 @@ class ZipStreamer { }); unzip.on('ready', () => { + if (limitFileSize) { + for (const entry of Object.values(unzip.entries())) { + if (!entry.isDirectory && entry.size > limitFileSize) { + reject('Файл слишком большой'); + return; + } + } + } + unzip.extract(null, outputDir, (err) => { if (err) reject(err); unzip.close(); From 4b9475310fc1e9a12fc834877de89fec1845e572 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 26 Jan 2020 16:23:20 +0700 Subject: [PATCH 43/58] =?UTF-8?q?=D0=A3=D0=B1=D1=80=D0=B0=D0=BB=20=D0=BD?= =?UTF-8?q?=D0=B5=D0=BD=D1=83=D0=B6=D0=BD=D1=8B=D0=B9=20this.taken?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/db/SqliteConnectionPool.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/db/SqliteConnectionPool.js b/server/db/SqliteConnectionPool.js index a1acb919..a5189a9b 100644 --- a/server/db/SqliteConnectionPool.js +++ b/server/db/SqliteConnectionPool.js @@ -14,7 +14,6 @@ class SqliteConnectionPool { if (!Number.isInteger(connCount) || connCount <= 0) return; this.connections = []; - this.taken = new Set(); this.freed = new Set(); for (let i = 0; i < connCount; i++) { @@ -22,7 +21,6 @@ class SqliteConnectionPool { client.configure('busyTimeout', 10000); //ms client.ret = () => { - this.taken.delete(i); this.freed.add(i); }; @@ -52,7 +50,6 @@ class SqliteConnectionPool { } this.freed.delete(freeConnIndex); - this.taken.add(freeConnIndex); return this.connections[freeConnIndex]; } From b7568975e78672120ca22e75515cce23cc8508a0 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 26 Jan 2020 18:31:31 +0700 Subject: [PATCH 44/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=BE=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=BA=D0=B0=20state=20=3D=20'queue'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/ProgressPage/ProgressPage.vue | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/client/components/Reader/ProgressPage/ProgressPage.vue b/client/components/Reader/ProgressPage/ProgressPage.vue index 0ac1a0bc..49b31c29 100644 --- a/client/components/Reader/ProgressPage/ProgressPage.vue +++ b/client/components/Reader/ProgressPage/ProgressPage.vue @@ -16,6 +16,7 @@ const ruMessage = { 'start': ' ', 'finish': ' ', 'error': ' ', + 'queue': 'очередь', 'download': 'скачивание', 'decompress': 'распаковка', 'convert': 'конвертирование', @@ -49,8 +50,13 @@ class ProgressPage extends Vue { } setState(state) { - if (state.state) - this.text = (ruMessage[state.state] ? ruMessage[state.state] : state.state); + if (state.state) { + if (state.state == 'queue') { + this.text = 'Номер в очереди: ' + (state.place ? state.place : ''); + } else { + this.text = (ruMessage[state.state] ? ruMessage[state.state] : state.state); + } + } this.step = (state.step ? state.step : this.step); this.totalSteps = (state.totalSteps > this.totalSteps ? state.totalSteps : this.totalSteps); this.progress = state.progress || 0; From a50d61c3ce320e97bcdd7ba49f686f2ed6a23810 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 26 Jan 2020 18:37:14 +0700 Subject: [PATCH 45/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=20=D0=BE=D1=87=D0=B5=D1=80=D0=B5=D0=B4=D1=8C?= =?UTF-8?q?=20=D1=81=D0=BA=D0=B0=D1=87=D0=B8=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F?= =?UTF-8?q?=20=D0=B8=20=D0=BA=D0=BE=D0=BD=D0=B2=D0=B5=D1=80=D1=82=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/Reader/BookConverter/ConvertBase.js | 15 +++++++++------ server/core/Reader/ReaderWorker.js | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/server/core/Reader/BookConverter/ConvertBase.js b/server/core/Reader/BookConverter/ConvertBase.js index d3ad74ce..bf24af84 100644 --- a/server/core/Reader/BookConverter/ConvertBase.js +++ b/server/core/Reader/BookConverter/ConvertBase.js @@ -3,11 +3,10 @@ const iconv = require('iconv-lite'); const chardet = require('chardet'); const he = require('he'); +const LimitedQueue = require('../../LimitedQueue'); const textUtils = require('./textUtils'); const utils = require('../../utils'); -let execConverterCounter = 0; - class ConvertBase { constructor(config) { this.config = config; @@ -15,6 +14,7 @@ class ConvertBase { this.calibrePath = `${config.dataDir}/calibre/ebook-convert`; this.sofficePath = '/usr/bin/soffice'; this.pdfToHtmlPath = '/usr/bin/pdftohtml'; + this.queue = new LimitedQueue(2, 20, 3); } async run(data, opts) {// eslint-disable-line no-unused-vars @@ -33,11 +33,14 @@ class ConvertBase { } async execConverter(path, args, onData) { - execConverterCounter++; + let q = null; try { - if (execConverterCounter > 10) - throw new Error('Слишком большая очередь конвертирования. Пожалуйста, попробуйте позже.'); + q = await this.queue.get(() => {onData();}); + } catch (e) { + throw new Error('Слишком большая очередь конвертирования. Пожалуйста, попробуйте позже.'); + } + try { const result = await utils.spawnProcess(path, {args, onData}); if (result.code != 0) { let error = result.code; @@ -54,7 +57,7 @@ class ConvertBase { throw new Error(e); } } finally { - execConverterCounter--; + q.ret(); } } diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 74c7eb2c..7507887c 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -1,6 +1,7 @@ const fs = require('fs-extra'); const path = require('path'); +const LimitedQueue = require('../LimitedQueue'); const WorkerState = require('../WorkerState');//singleton const FileDownloader = require('../FileDownloader'); const FileDecompressor = require('../FileDecompressor'); @@ -26,6 +27,7 @@ class ReaderWorker { this.config.tempPublicDir = `${config.publicDir}/tmp`; fs.ensureDirSync(this.config.tempPublicDir); + this.queue = new LimitedQueue(5, 100, 3); this.workerState = new WorkerState(); this.down = new FileDownloader(config.maxUploadFileSize); this.decomp = new FileDecompressor(2*config.maxUploadFileSize); @@ -53,7 +55,21 @@ class ReaderWorker { let downloadedFilename = ''; let isUploaded = false; let convertFilename = ''; + + let q = null; try { + wState.set({state: 'queue', step: 1, totalSteps: 1}); + try { + let qSize = 0; + q = await this.queue.get((place) => { + wState.set({place, progress: (qSize ? Math.round((qSize - place)/qSize*100) : 0)}); + if (!qSize) + qSize = place; + }); + } catch (e) { + throw new Error('Слишком большая очередь загрузки. Пожалуйста, попробуйте позже.'); + } + wState.set({state: 'download', step: 1, totalSteps: 3, url}); const tempFilename = utils.randomHexString(30); @@ -123,6 +139,8 @@ class ReaderWorker { wState.set({state: 'error', error: e.message}); } finally { //clean + if (q) + q.ret(); if (decompDir) await fs.remove(decompDir); if (downloadedFilename && !isUploaded) From 3da6befe1058816f122e269f7e37bde9a83c970b Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 26 Jan 2020 18:38:09 +0700 Subject: [PATCH 46/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20=D0=BA=D0=BB=D0=B0=D1=81=D1=81=20LimitedQueue=20?= =?UTF-8?q?=D0=B4=D0=BB=D1=8F=20=D0=BE=D1=80=D0=B3=D0=B0=D0=BD=D0=B8=D0=B7?= =?UTF-8?q?=D0=B0=D1=86=D0=B8=D0=B8=20=D0=BE=D1=87=D0=B5=D1=80=D0=B5=D0=B4?= =?UTF-8?q?=D0=B5=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/LimitedQueue.js | 120 ++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 server/core/LimitedQueue.js diff --git a/server/core/LimitedQueue.js b/server/core/LimitedQueue.js new file mode 100644 index 00000000..fd9d68e0 --- /dev/null +++ b/server/core/LimitedQueue.js @@ -0,0 +1,120 @@ +const cleanPeriod = 60*1000;//1 минута +const cleanTimeout = 60;//timeout в минутах (cleanPeriod) + +class LimitedQueue { + constructor(enqueueAfter = 10, size = 100, timeout = cleanTimeout) {//timeout в минутах (cleanPeriod) + this.size = size; + this.timeout = timeout; + + this.freed = enqueueAfter; + this.listeners = []; + + this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod); + } + + _addListener(listener) { + this.listeners.push(Object.assign({regTime: Date.now()}, listener)); + } + + //отсылаем сообщение первому ожидающему и удаляем его из списка + _emitFree() { + if (this.listeners.length > 0) { + let listener = this.listeners.shift(); + listener.onFree(); + + const now = Date.now(); + for (let i = 0; i < this.listeners.length; i++) { + listener = this.listeners[i]; + listener.regTime = now; + listener.onPlaceChange(i + 1); + } + + } + } + + get(onPlaceChange) { + return new Promise((resolve, reject) => { + if (this.destroyed) + reject('destroyed'); + + const take = () => { + if (this.freed <= 0) + throw new Error('Ошибка получения ресурсов в очереди ожидания'); + + this.freed--; + + let returned = false; + return { + ret: () => { + if (!returned) { + this.freed++; + this._emitFree(); + returned = true; + } + } + }; + }; + + if (this.freed > 0) { + resolve(take()); + } else { + if (this.listeners.length < this.size) { + this._addListener({ + onFree: () => { + resolve(take()); + }, + onError: (err) => { + reject(err); + }, + onPlaceChange: (i) => { + if (onPlaceChange) + onPlaceChange(i); + } + }); + if (onPlaceChange) + onPlaceChange(this.listeners.length); + } else { + reject('Превышен размер очереди ожидания'); + } + } + }); + } + + destroy() { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + + for (const listener of this.listeners) { + listener.onError('destroy'); + } + this.listeners = []; + + this.destroyed = true; + } + + periodicClean() { + try { + this.timer = null; + + const now = Date.now(); + //чистка listeners, убираем зависшие в очереди на одном месте + let newListeners = []; + for (const listener of this.listeners) { + if (now - listener.regTime < this.timeout*cleanPeriod - 50) { + newListeners.push(listener); + } else { + listener.onError('Время ожидания в очереди истекло'); + } + } + this.listeners = newListeners; + } finally { + if (!this.destroyed) { + this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod); + } + } + } +} + +module.exports = LimitedQueue; \ No newline at end of file From f8b7b8b6985eacf16a02a83d0275f6e5bbb9c8aa Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 27 Jan 2020 18:57:42 +0700 Subject: [PATCH 47/58] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20LimitedQueue,=20=D0=B8=D1=81?= =?UTF-8?q?=D0=BF=D1=80=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B1?= =?UTF-8?q?=D0=B0=D0=B3=D0=BE=D0=B2,=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B0=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=BA=D0=B0=20=D1=84=D0=BB=D0=B0=D0=B3=D0=B0=20abort?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/FileDownloader.js | 12 ++- server/core/LimitedQueue.js | 73 +++++++++---------- .../core/Reader/BookConverter/ConvertBase.js | 2 +- server/core/Reader/ReaderWorker.js | 22 +++++- 4 files changed, 64 insertions(+), 45 deletions(-) diff --git a/server/core/FileDownloader.js b/server/core/FileDownloader.js index d6c53897..018acaf4 100644 --- a/server/core/FileDownloader.js +++ b/server/core/FileDownloader.js @@ -5,7 +5,7 @@ class FileDownloader { this.limitDownloadSize = limitDownloadSize; } - async load(url, callback) { + async load(url, callback, abort) { let errMes = ''; const options = { encoding: null, @@ -22,7 +22,9 @@ class FileDownloader { } let prevProg = 0; - const request = got(url, options).on('downloadProgress', progress => { + const request = got(url, options); + + request.on('downloadProgress', progress => { if (this.limitDownloadSize) { if (progress.transferred > this.limitDownloadSize) { errMes = 'Файл слишком большой'; @@ -39,8 +41,12 @@ class FileDownloader { if (prog != prevProg && callback) callback(prog); prevProg = prog; - }); + if (abort && abort()) { + errMes = 'abort'; + request.cancel(); + } + }); try { return (await request).body; diff --git a/server/core/LimitedQueue.js b/server/core/LimitedQueue.js index fd9d68e0..122ee270 100644 --- a/server/core/LimitedQueue.js +++ b/server/core/LimitedQueue.js @@ -1,19 +1,16 @@ -const cleanPeriod = 60*1000;//1 минута -const cleanTimeout = 60;//timeout в минутах (cleanPeriod) - class LimitedQueue { - constructor(enqueueAfter = 10, size = 100, timeout = cleanTimeout) {//timeout в минутах (cleanPeriod) + constructor(enqueueAfter = 10, size = 100, timeout = 60*60*1000) {//timeout в ms this.size = size; this.timeout = timeout; + this.abortCount = 0; + this.enqueueAfter = enqueueAfter; this.freed = enqueueAfter; this.listeners = []; - - this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod); } _addListener(listener) { - this.listeners.push(Object.assign({regTime: Date.now()}, listener)); + this.listeners.push(listener); } //отсылаем сообщение первому ожидающему и удаляем его из списка @@ -22,13 +19,11 @@ class LimitedQueue { let listener = this.listeners.shift(); listener.onFree(); - const now = Date.now(); for (let i = 0; i < this.listeners.length; i++) { - listener = this.listeners[i]; - listener.regTime = now; - listener.onPlaceChange(i + 1); + this.listeners[i].onPlaceChange(i + 1); } + this.resetTimeout(); } } @@ -42,16 +37,21 @@ class LimitedQueue { throw new Error('Ошибка получения ресурсов в очереди ожидания'); this.freed--; + this.resetTimeout(); - let returned = false; + let aCount = this.abortCount; return { ret: () => { - if (!returned) { + if (aCount == this.abortCount) { this.freed++; this._emitFree(); - returned = true; + aCount = -1; } - } + }, + abort: () => { + return (aCount != this.abortCount); + }, + resetTimeout: this.resetTimeout.bind(this) }; }; @@ -80,6 +80,27 @@ class LimitedQueue { }); } + resetTimeout() { + if (this.timer) + clearTimeout(this.timer); + this.timer = setTimeout(() => { this.clean(); }, this.timeout); + } + + clean() { + this.timer = null; + + if (this.freed < this.enqueueAfter) { + this.abortCount++; + //чистка listeners + for (const listener of this.listeners) { + listener.onError('Время ожидания в очереди истекло'); + } + this.listeners = []; + + this.freed = this.enqueueAfter; + } + } + destroy() { if (this.timer) { clearTimeout(this.timer); @@ -93,28 +114,6 @@ class LimitedQueue { this.destroyed = true; } - - periodicClean() { - try { - this.timer = null; - - const now = Date.now(); - //чистка listeners, убираем зависшие в очереди на одном месте - let newListeners = []; - for (const listener of this.listeners) { - if (now - listener.regTime < this.timeout*cleanPeriod - 50) { - newListeners.push(listener); - } else { - listener.onError('Время ожидания в очереди истекло'); - } - } - this.listeners = newListeners; - } finally { - if (!this.destroyed) { - this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod); - } - } - } } module.exports = LimitedQueue; \ No newline at end of file diff --git a/server/core/Reader/BookConverter/ConvertBase.js b/server/core/Reader/BookConverter/ConvertBase.js index bf24af84..70a1aa7d 100644 --- a/server/core/Reader/BookConverter/ConvertBase.js +++ b/server/core/Reader/BookConverter/ConvertBase.js @@ -14,7 +14,7 @@ class ConvertBase { this.calibrePath = `${config.dataDir}/calibre/ebook-convert`; this.sofficePath = '/usr/bin/soffice'; this.pdfToHtmlPath = '/usr/bin/pdftohtml'; - this.queue = new LimitedQueue(2, 20, 3); + this.queue = new LimitedQueue(2, 20, 3*60*1000); } async run(data, opts) {// eslint-disable-line no-unused-vars diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 7507887c..ac04138f 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -27,7 +27,7 @@ class ReaderWorker { this.config.tempPublicDir = `${config.publicDir}/tmp`; fs.ensureDirSync(this.config.tempPublicDir); - this.queue = new LimitedQueue(5, 100, 3); + this.queue = new LimitedQueue(5, 100, 3*60*1000); this.workerState = new WorkerState(); this.down = new FileDownloader(config.maxUploadFileSize); this.decomp = new FileDecompressor(2*config.maxUploadFileSize); @@ -56,6 +56,9 @@ class ReaderWorker { let isUploaded = false; let convertFilename = ''; + const overLoadMes = 'Слишком большая очередь загрузки. Пожалуйста, попробуйте позже.'; + const overLoadErr = new Error(overLoadMes); + let q = null; try { wState.set({state: 'queue', step: 1, totalSteps: 1}); @@ -67,7 +70,7 @@ class ReaderWorker { qSize = place; }); } catch (e) { - throw new Error('Слишком большая очередь загрузки. Пожалуйста, попробуйте позже.'); + throw overLoadErr; } wState.set({state: 'download', step: 1, totalSteps: 3, url}); @@ -76,10 +79,11 @@ class ReaderWorker { const tempFilename2 = utils.randomHexString(30); const decompDirname = utils.randomHexString(30); + //download or use uploaded if (url.indexOf('file://') != 0) {//download const downdata = await this.down.load(url, (progress) => { wState.set({progress}); - }); + }, q.abort); downloadedFilename = `${this.config.tempDownloadDir}/${tempFilename}`; await fs.writeFile(downloadedFilename, downdata); @@ -92,6 +96,10 @@ class ReaderWorker { } wState.set({progress: 100}); + if (q.abort()) + throw overLoadErr; + q.resetTimeout(); + //decompress wState.set({state: 'decompress', step: 2, progress: 0}); decompDir = `${this.config.tempDownloadDir}/${decompDirname}`; @@ -104,12 +112,16 @@ class ReaderWorker { } wState.set({progress: 100}); + if (q.abort()) + throw overLoadErr; + q.resetTimeout(); + //конвертирование в fb2 wState.set({state: 'convert', step: 3, progress: 0}); convertFilename = `${this.config.tempDownloadDir}/${tempFilename2}`; await this.bookConverter.convertToFb2(decompFiles, convertFilename, opts, progress => { wState.set({progress}); - }); + }, q.abort); //сжимаем файл в tmp, если там уже нет с тем же именем-sha256 const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, this.config.tempPublicDir); @@ -136,6 +148,8 @@ class ReaderWorker { } catch (e) { log(LM_ERR, e.stack); + if (e.message == 'abort') + e.message = overLoadMes; wState.set({state: 'error', error: e.message}); } finally { //clean From 57fc64af79f38c911d07bd23b240c38a27b5ec59 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 27 Jan 2020 19:34:10 +0700 Subject: [PATCH 48/58] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20abort=20=D0=BA=D0=BE=D0=BD=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D1=82=D0=B5=D1=80=D0=BE=D0=B2=20=D0=BF=D1=80=D0=B8=20=D0=B8?= =?UTF-8?q?=D1=81=D1=82=D0=B5=D1=87=D0=B5=D0=BD=D0=B8=D0=B8=20=D0=B2=D1=80?= =?UTF-8?q?=D0=B5=D0=BC=D0=B5=D0=BD=D0=B8=20=D0=BE=D0=B6=D0=B8=D0=B4=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D1=8F=20=D0=BF=D0=BE=D0=B4=D0=B2=D0=B8=D0=B6=D0=B5?= =?UTF-8?q?=D0=BA=20=D0=BE=D1=87=D0=B5=D1=80=D0=B5=D0=B4=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/Reader/BookConverter/ConvertBase.js | 8 +++++--- .../core/Reader/BookConverter/ConvertDoc.js | 6 +++--- .../core/Reader/BookConverter/ConvertDocX.js | 8 ++++---- .../core/Reader/BookConverter/ConvertEpub.js | 4 ++-- .../core/Reader/BookConverter/ConvertMobi.js | 4 ++-- .../core/Reader/BookConverter/ConvertPdf.js | 4 ++-- .../core/Reader/BookConverter/ConvertRtf.js | 6 +++--- server/core/Reader/BookConverter/index.js | 9 ++++++--- server/core/Reader/ReaderWorker.js | 2 +- server/core/utils.js | 20 +++++++++++++------ 10 files changed, 42 insertions(+), 29 deletions(-) diff --git a/server/core/Reader/BookConverter/ConvertBase.js b/server/core/Reader/BookConverter/ConvertBase.js index 70a1aa7d..98074a06 100644 --- a/server/core/Reader/BookConverter/ConvertBase.js +++ b/server/core/Reader/BookConverter/ConvertBase.js @@ -14,7 +14,7 @@ class ConvertBase { this.calibrePath = `${config.dataDir}/calibre/ebook-convert`; this.sofficePath = '/usr/bin/soffice'; this.pdfToHtmlPath = '/usr/bin/pdftohtml'; - this.queue = new LimitedQueue(2, 20, 3*60*1000); + this.queue = new LimitedQueue(2, 20, 3*60*1000);//3 минуты ожидание подвижек } async run(data, opts) {// eslint-disable-line no-unused-vars @@ -32,7 +32,7 @@ class ConvertBase { throw new Error('Внешний конвертер pdftohtml не найден'); } - async execConverter(path, args, onData) { + async execConverter(path, args, onData, abort) { let q = null; try { q = await this.queue.get(() => {onData();}); @@ -41,7 +41,7 @@ class ConvertBase { } try { - const result = await utils.spawnProcess(path, {args, onData}); + const result = await utils.spawnProcess(path, {args, onData, abort}); if (result.code != 0) { let error = result.code; if (this.config.branch == 'development') @@ -51,6 +51,8 @@ class ConvertBase { } catch(e) { if (e.status == 'killed') { throw new Error('Слишком долгое ожидание конвертера'); + } else if (e.status == 'abort') { + throw new Error('abort'); } else if (e.status == 'error') { throw new Error(e.error); } else { diff --git a/server/core/Reader/BookConverter/ConvertDoc.js b/server/core/Reader/BookConverter/ConvertDoc.js index ce627eb8..a0c24191 100644 --- a/server/core/Reader/BookConverter/ConvertDoc.js +++ b/server/core/Reader/BookConverter/ConvertDoc.js @@ -16,7 +16,7 @@ class ConvertDoc extends ConvertDocX { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const docFile = `${outFile}.doc`; @@ -24,9 +24,9 @@ class ConvertDoc extends ConvertDocX { const fb2File = `${outFile}.fb2`; await fs.copy(inputFiles.sourceFile, docFile); - await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, docFile]); + await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, docFile], null, abort); - return await super.convert(docxFile, fb2File, callback); + return await super.convert(docxFile, fb2File, callback, abort); } } diff --git a/server/core/Reader/BookConverter/ConvertDocX.js b/server/core/Reader/BookConverter/ConvertDocX.js index d8c85b78..02b112c5 100644 --- a/server/core/Reader/BookConverter/ConvertDocX.js +++ b/server/core/Reader/BookConverter/ConvertDocX.js @@ -20,12 +20,12 @@ class ConvertDocX extends ConvertBase { return false; } - async convert(docxFile, fb2File, callback) { + async convert(docxFile, fb2File, callback, abort) { let perc = 0; await this.execConverter(this.calibrePath, [docxFile, fb2File], () => { perc = (perc < 100 ? perc + 5 : 50); callback(perc); - }); + }, abort); return await fs.readFile(fb2File); } @@ -35,7 +35,7 @@ class ConvertDocX extends ConvertBase { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const docxFile = `${outFile}.docx`; @@ -43,7 +43,7 @@ class ConvertDocX extends ConvertBase { await fs.copy(inputFiles.sourceFile, docxFile); - return await this.convert(docxFile, fb2File, callback); + return await this.convert(docxFile, fb2File, callback, abort); } } diff --git a/server/core/Reader/BookConverter/ConvertEpub.js b/server/core/Reader/BookConverter/ConvertEpub.js index 79b25b8e..f0f9ba71 100644 --- a/server/core/Reader/BookConverter/ConvertEpub.js +++ b/server/core/Reader/BookConverter/ConvertEpub.js @@ -28,7 +28,7 @@ class ConvertEpub extends ConvertBase { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const epubFile = `${outFile}.epub`; @@ -40,7 +40,7 @@ class ConvertEpub extends ConvertBase { await this.execConverter(this.calibrePath, [epubFile, fb2File], () => { perc = (perc < 100 ? perc + 5 : 50); callback(perc); - }); + }, abort); return await fs.readFile(fb2File); } diff --git a/server/core/Reader/BookConverter/ConvertMobi.js b/server/core/Reader/BookConverter/ConvertMobi.js index e8d6172b..ee8535ef 100644 --- a/server/core/Reader/BookConverter/ConvertMobi.js +++ b/server/core/Reader/BookConverter/ConvertMobi.js @@ -16,7 +16,7 @@ class ConvertMobi extends ConvertBase { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const mobiFile = `${outFile}.mobi`; @@ -28,7 +28,7 @@ class ConvertMobi extends ConvertBase { await this.execConverter(this.calibrePath, [mobiFile, fb2File], () => { perc = (perc < 100 ? perc + 5 : 50); callback(perc); - }); + }, abort); return await fs.readFile(fb2File); } diff --git a/server/core/Reader/BookConverter/ConvertPdf.js b/server/core/Reader/BookConverter/ConvertPdf.js index be885369..397a1868 100644 --- a/server/core/Reader/BookConverter/ConvertPdf.js +++ b/server/core/Reader/BookConverter/ConvertPdf.js @@ -18,7 +18,7 @@ class ConvertPdf extends ConvertHtml { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${utils.randomHexString(10)}.xml`; @@ -27,7 +27,7 @@ class ConvertPdf extends ConvertHtml { await this.execConverter(this.pdfToHtmlPath, ['-c', '-s', '-xml', inputFiles.sourceFile, outFile], () => { perc = (perc < 80 ? perc + 10 : 40); callback(perc); - }); + }, abort); callback(80); const data = await fs.readFile(outFile); diff --git a/server/core/Reader/BookConverter/ConvertRtf.js b/server/core/Reader/BookConverter/ConvertRtf.js index 61f489f7..5f167cd6 100644 --- a/server/core/Reader/BookConverter/ConvertRtf.js +++ b/server/core/Reader/BookConverter/ConvertRtf.js @@ -16,7 +16,7 @@ class ConvertRtf extends ConvertDocX { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const rtfFile = `${outFile}.rtf`; @@ -24,9 +24,9 @@ class ConvertRtf extends ConvertDocX { const fb2File = `${outFile}.fb2`; await fs.copy(inputFiles.sourceFile, rtfFile); - await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, rtfFile]); + await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, rtfFile], null, abort); - return await super.convert(docxFile, fb2File, callback); + return await super.convert(docxFile, fb2File, callback, abort); } } diff --git a/server/core/Reader/BookConverter/index.js b/server/core/Reader/BookConverter/index.js index 89b73f5a..d91de21e 100644 --- a/server/core/Reader/BookConverter/index.js +++ b/server/core/Reader/BookConverter/index.js @@ -26,11 +26,14 @@ class BookConverter { } } - async convertToFb2(inputFiles, outputFile, opts, callback) { + async convertToFb2(inputFiles, outputFile, opts, callback, abort) { + if (abort && abort()) + throw new Error('abort'); + const selectedFileType = await this.detector.detectFile(inputFiles.selectedFile); const data = await fs.readFile(inputFiles.selectedFile); - const convertOpts = Object.assign({}, opts, {inputFiles, callback, dataType: selectedFileType}); + const convertOpts = Object.assign({}, opts, {inputFiles, callback, abort, dataType: selectedFileType}); let result = false; for (const convert of this.convertFactory) { result = await convert.run(data, convertOpts); @@ -41,7 +44,7 @@ class BookConverter { } if (!result && inputFiles.nesting) { - result = await this.convertToFb2(inputFiles.nesting, outputFile, opts, callback); + result = await this.convertToFb2(inputFiles.nesting, outputFile, opts, callback, abort); } if (!result) { diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index ac04138f..078bafca 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -27,7 +27,7 @@ class ReaderWorker { this.config.tempPublicDir = `${config.publicDir}/tmp`; fs.ensureDirSync(this.config.tempPublicDir); - this.queue = new LimitedQueue(5, 100, 3*60*1000); + this.queue = new LimitedQueue(5, 100, 5*60*1000);//5 минут ожидание подвижек this.workerState = new WorkerState(); this.down = new FileDownloader(config.maxUploadFileSize); this.decomp = new FileDecompressor(2*config.maxUploadFileSize); diff --git a/server/core/utils.js b/server/core/utils.js index 9591c7d7..53f7a2b3 100644 --- a/server/core/utils.js +++ b/server/core/utils.js @@ -37,8 +37,8 @@ async function touchFile(filename) { } function spawnProcess(cmd, opts) { - let {args, killAfter, onData} = opts; - killAfter = (killAfter ? killAfter : 120*1000); + let {args, killAfter, onData, abort} = opts; + killAfter = (killAfter ? killAfter : 120);//seconds onData = (onData ? onData : () => {}); args = (args ? args : []); @@ -67,10 +67,18 @@ function spawnProcess(cmd, opts) { reject({status: 'error', error, stdout, stderr}); }); - await sleep(killAfter); - if (!resolved) { - process.kill(proc.pid); - reject({status: 'killed', stdout, stderr}); + while (!resolved) { + await sleep(1000); + killAfter -= 1; + if (killAfter <= 0 || (abort && abort())) { + process.kill(proc.pid); + if (killAfter <= 0) { + reject({status: 'killed', stdout, stderr}); + } else { + reject({status: 'abort', stdout, stderr}); + } + break; + } } })().catch(reject); }); } From a5cb2641fd2339675a72e84be4dd5582da8938ba Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 27 Jan 2020 19:42:30 +0700 Subject: [PATCH 49/58] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B0=D1=8F=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/LimitedQueue.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/core/LimitedQueue.js b/server/core/LimitedQueue.js index 122ee270..d0784652 100644 --- a/server/core/LimitedQueue.js +++ b/server/core/LimitedQueue.js @@ -22,8 +22,6 @@ class LimitedQueue { for (let i = 0; i < this.listeners.length; i++) { this.listeners[i].onPlaceChange(i + 1); } - - this.resetTimeout(); } } @@ -46,6 +44,7 @@ class LimitedQueue { this.freed++; this._emitFree(); aCount = -1; + this.resetTimeout(); } }, abort: () => { From 76f7d7bc905bbfe0bdf369c78b0ce8ea86b61e86 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Mon, 27 Jan 2020 19:52:56 +0700 Subject: [PATCH 50/58] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B0=D1=8F=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/LimitedQueue.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/core/LimitedQueue.js b/server/core/LimitedQueue.js index d0784652..393a53a8 100644 --- a/server/core/LimitedQueue.js +++ b/server/core/LimitedQueue.js @@ -110,6 +110,7 @@ class LimitedQueue { listener.onError('destroy'); } this.listeners = []; + this.abortCount++; this.destroyed = true; } From 1226acefd62a75ed7d2bb87c598d1b0b2d97566b Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 28 Jan 2020 14:51:09 +0700 Subject: [PATCH 51/58] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D1=88=D0=B8=D0=B5=20=D0=B8=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F,=20queue=20=D1=82=D0=B5=D0=BF=D0=B5?= =?UTF-8?q?=D1=80=D1=8C=20=D0=B2=20=D0=BE=D0=B4=D0=BD=D0=BE=D0=BC=20=D1=8D?= =?UTF-8?q?=D0=BA=D0=B7=D0=B5=D0=BC=D0=BF=D0=BB=D1=8F=D1=80=D0=B5=20=D0=BD?= =?UTF-8?q?=D0=B0=20=D0=BA=D0=BB=D0=B0=D1=81=D1=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/BookConverter/ConvertBase.js | 5 +++-- server/core/Reader/ReaderWorker.js | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/server/core/Reader/BookConverter/ConvertBase.js b/server/core/Reader/BookConverter/ConvertBase.js index 98074a06..11d07900 100644 --- a/server/core/Reader/BookConverter/ConvertBase.js +++ b/server/core/Reader/BookConverter/ConvertBase.js @@ -7,6 +7,8 @@ const LimitedQueue = require('../../LimitedQueue'); const textUtils = require('./textUtils'); const utils = require('../../utils'); +const queue = new LimitedQueue(2, 20, 3*60*1000);//3 минуты ожидание подвижек + class ConvertBase { constructor(config) { this.config = config; @@ -14,7 +16,6 @@ class ConvertBase { this.calibrePath = `${config.dataDir}/calibre/ebook-convert`; this.sofficePath = '/usr/bin/soffice'; this.pdfToHtmlPath = '/usr/bin/pdftohtml'; - this.queue = new LimitedQueue(2, 20, 3*60*1000);//3 минуты ожидание подвижек } async run(data, opts) {// eslint-disable-line no-unused-vars @@ -35,7 +36,7 @@ class ConvertBase { async execConverter(path, args, onData, abort) { let q = null; try { - q = await this.queue.get(() => {onData();}); + q = await queue.get(() => {onData();}); } catch (e) { throw new Error('Слишком большая очередь конвертирования. Пожалуйста, попробуйте позже.'); } diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 078bafca..dafc628a 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -12,6 +12,7 @@ const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton const cleanDirPeriod = 60*60*1000;//1 раз в час +const queue = new LimitedQueue(5, 100, 5*60*1000);//5 минут ожидание подвижек let instance = null; @@ -27,7 +28,6 @@ class ReaderWorker { this.config.tempPublicDir = `${config.publicDir}/tmp`; fs.ensureDirSync(this.config.tempPublicDir); - this.queue = new LimitedQueue(5, 100, 5*60*1000);//5 минут ожидание подвижек this.workerState = new WorkerState(); this.down = new FileDownloader(config.maxUploadFileSize); this.decomp = new FileDecompressor(2*config.maxUploadFileSize); @@ -64,7 +64,7 @@ class ReaderWorker { wState.set({state: 'queue', step: 1, totalSteps: 1}); try { let qSize = 0; - q = await this.queue.get((place) => { + q = await queue.get((place) => { wState.set({place, progress: (qSize ? Math.round((qSize - place)/qSize*100) : 0)}); if (!qSize) qSize = place; From efb24137208a58dbe5801db5e2f3512d2d8d8fc6 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 28 Jan 2020 19:44:52 +0700 Subject: [PATCH 52/58] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D1=88=D0=BE=D0=B5=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D1=81=D0=BE=D0=B4=D0=B5=D1=80=D0=B6=D0=B8=D0=BC?= =?UTF-8?q?=D0=BE=D0=B3=D0=BE=20=D1=81=D1=82=D1=80=D0=B0=D0=BD=D0=B8=D1=86?= =?UTF-8?q?=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DonateHelpPage/DonateHelpPage.vue | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue b/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue index 637ce6e7..4ae7dbfb 100644 --- a/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue +++ b/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue @@ -1,30 +1,54 @@ @@ -200,6 +245,7 @@ class Reader extends Vue { whatsNewVisible = false; whatsNewContent = ''; + donationVisible = false; created() { this.loading = true; @@ -258,9 +304,10 @@ class Reader extends Vue { this.checkActivateDonateHelpPage(); this.loading = false; - await this.showWhatsNew(); - this.updateRoute(); + + await this.showWhatsNew(); + await this.showDonation(); })(); } @@ -272,6 +319,7 @@ class Reader extends Vue { this.clickControl = settings.clickControl; this.blinkCachedLoad = settings.blinkCachedLoad; this.showWhatsNewDialog = settings.showWhatsNewDialog; + this.showDonationDialog2020 = settings.showDonationDialog2020; this.showToolButton = settings.showToolButton; this.enableSitesFilter = settings.enableSitesFilter; @@ -337,6 +385,41 @@ class Reader extends Vue { } } + async showDonation() { + await utils.sleep(3000); + const today = utils.formatDate(new Date(), 'coDate'); + + if (this.mode == 'omnireader' && today < '2020-03-01' && this.showDonationDialog2020 && this.donationRemindDate != today) { + this.donationVisible = true; + } + } + + donationDialogDisable() { + this.donationVisible = false; + if (this.showDonationDialog2020) { + const newSettings = Object.assign({}, this.settings, { showDonationDialog2020: false }); + this.commit('reader/setSettings', newSettings); + } + } + + donationDialogRemind() { + this.donationVisible = false; + this.commit('reader/setDonationRemindDate', utils.formatDate(new Date(), 'coDate')); + } + + openDonate() { + this.donationVisible = false; + this.donateToggle(); + } + + async copyLink(link) { + const result = await utils.copyTextToClipboard(link); + if (result) + this.$notify.success({message: `Ссылка ${link} успешно скопирована в буфер обмена`}); + else + this.$notify.error({message: 'Копирование не удалось'}); + } + openVersionHistory() { this.whatsNewVisible = false; this.versionHistoryToggle(); @@ -455,6 +538,10 @@ class Reader extends Vue { return this.$store.state.reader.whatsNewContentHash; } + get donationRemindDate() { + return this.$store.state.reader.donationRemindDate; + } + addAction(pos) { let a = this.actionList; if (!a.length || a[a.length - 1] != pos) { diff --git a/client/components/Reader/SettingsPage/SettingsPage.vue b/client/components/Reader/SettingsPage/SettingsPage.vue index d7e736c2..3f5a4b0c 100644 --- a/client/components/Reader/SettingsPage/SettingsPage.vue +++ b/client/components/Reader/SettingsPage/SettingsPage.vue @@ -471,6 +471,14 @@ Показывать уведомление "Что нового" + + + + Показывать "Оплатим хостинг вместе" + + diff --git a/client/element.js b/client/element.js index 370ab1b4..39148440 100644 --- a/client/element.js +++ b/client/element.js @@ -19,6 +19,7 @@ import ElCheckbox from 'element-ui/lib/checkbox'; import ElTabs from 'element-ui/lib/tabs'; import ElTabPane from 'element-ui/lib/tab-pane'; import ElTooltip from 'element-ui/lib/tooltip'; +import ElRow from 'element-ui/lib/row'; import ElCol from 'element-ui/lib/col'; import ElContainer from 'element-ui/lib/container'; import ElAside from 'element-ui/lib/aside'; @@ -43,7 +44,7 @@ import MessageBox from 'element-ui/lib/message-box'; const components = { ElMenu, ElMenuItem, ElButton, ElButtonGroup, ElCheckbox, ElTabs, ElTabPane, ElTooltip, - ElCol, ElContainer, ElAside, ElMain, ElHeader, + ElRow, ElCol, ElContainer, ElAside, ElMain, ElHeader, ElInput, ElInputNumber, ElSelect, ElOption, ElTable, ElTableColumn, ElProgress, ElSlider, ElForm, ElFormItem, ElColorPicker, ElDialog, diff --git a/client/store/modules/reader.js b/client/store/modules/reader.js index 7e25c9c4..814b0984 100644 --- a/client/store/modules/reader.js +++ b/client/store/modules/reader.js @@ -182,6 +182,7 @@ const settingDefaults = { imageFitWidth: true, showServerStorageMessages: true, showWhatsNewDialog: true, + showDonationDialog2020: true, enableSitesFilter: true, fontShifts: {}, @@ -204,6 +205,7 @@ const state = { profilesRev: 0, allowProfilesSave: false,//подстраховка для разработки whatsNewContentHash: '', + donationRemindDate: '', currentProfile: '', settings: Object.assign({}, settingDefaults), settingsRev: {}, @@ -238,6 +240,9 @@ const mutations = { setWhatsNewContentHash(state, value) { state.whatsNewContentHash = value; }, + setDonationRemindDate(state, value) { + state.donationRemindDate = value; + }, setCurrentProfile(state, value) { state.currentProfile = value; }, From 56ad41d10cc17f8c2c2fe649827f129a96a8c2b4 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 28 Jan 2020 20:18:02 +0700 Subject: [PATCH 54/58] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20=D1=82=D0=B5=D0=BA=D1=81=D1=82=D0=B0=20=D0=BE?= =?UTF-8?q?=D0=B1=D1=8A=D1=8F=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/Reader.vue | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/client/components/Reader/Reader.vue b/client/components/Reader/Reader.vue index 54dce485..ff14f991 100644 --- a/client/components/Reader/Reader.vue +++ b/client/components/Reader/Reader.vue @@ -97,8 +97,8 @@
Стартовала ежегодная акция "Оплатим хостинг вместе".

- Для оплаты годового хостинга читалки, необходимо собрать сумму около 2000 рублей. - У автора эти деньги есть. Однако будет справедливо, если каждый + Для оплаты годового хостинга читалки, необходимо собрать около 2000 рублей. + В настоящий момент у автора эта сумма есть в наличии. Однако будет справедливо, если каждый сможет проголосовать рублем за то, чтобы читалка так и оставалась:
    @@ -117,15 +117,17 @@ на читалку через тематические форумы, соцсети, мессенджеры и пр. - Чем нас больше, тем легче оставаться на плаву и тем больше мотивации у автора, чтобы продолжать работать над проектом. + Чем нас больше, тем легче оставаться на плаву и тем больше мотивации у разработчика, чтобы продолжать работать над проектом.

    - Если соберется бóльшая сумма, то разработка децентрализованной библиотеки для свободного обмена книгами будет по воможности ускорена. + Если соберется бóльшая сумма, то разработка децентрализованной библиотеки для свободного обмена книгами будет по возможности ускорена.

    P.S. При необходимости можно воспользоваться подходящим обменником на bestchange.ru

    - Помочь проекту + + Помочь проекту +
From 0b53ad4b4d861e9dd6541c28d6cd03a06c740691 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 28 Jan 2020 20:20:10 +0700 Subject: [PATCH 55/58] =?UTF-8?q?=D0=92=D0=B5=D1=80=D1=81=D0=B8=D1=8F=200.?= =?UTF-8?q?8.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/versionHistory.js | 12 ++++++++++++ package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/client/components/Reader/versionHistory.js b/client/components/Reader/versionHistory.js index b05fa59d..98d1ced4 100644 --- a/client/components/Reader/versionHistory.js +++ b/client/components/Reader/versionHistory.js @@ -1,4 +1,16 @@ export const versionHistory = [ +{ + showUntil: '2020-01-27', + header: '0.8.3 (2020-01-28)', + content: +` +
    +
  • добавлено всплывающее окно с акцией "Оплатим хостинг вместе"
    • +
  • внутренние оптимизации
    • +
+` +}, + { showUntil: '2020-01-19', header: '0.8.2 (2020-01-20)', diff --git a/package.json b/package.json index 359d8191..bddedc8e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.8.2", + "version": "0.8.3", "author": "Book Pauk ", "license": "CC0-1.0", "repository": "bookpauk/liberama", From 07c85280cdb3e5958171e7b2c59d21beba7dbb4a Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 28 Jan 2020 21:27:54 +0700 Subject: [PATCH 56/58] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D1=8B=20=D1=82=D0=B0=D0=B9=D0=BC=D0=B0=D1=83?= =?UTF-8?q?=D1=82=D1=8B=20=D0=B4=D0=BB=D1=8F=20=D0=BA=D0=BE=D0=BD=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D1=82=D0=B5=D1=80=D0=B0=20calibre,=20=D0=B4=D0=BE?= =?UTF-8?q?=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=20=D1=84=D0=BB=D0=B0=D0=B3?= =?UTF-8?q?=20=D0=B7=D0=B0=D0=BF=D1=83=D1=81=D0=BA=D0=B0=20-vv,=20=D1=81?= =?UTF-8?q?=D0=BE=D0=BE=D1=82=D0=B2=D0=B5=D1=82=D1=81=D1=82=D0=B2=D0=B5?= =?UTF-8?q?=D0=BD=D0=BD=D0=BE=20=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=BE=20=D0=B2=D1=8B=D1=87=D0=B8=D1=81=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=BF=D1=80=D0=BE=D0=B3=D1=80=D0=B5=D1=81?= =?UTF-8?q?=D1=81=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/BookConverter/ConvertBase.js | 10 +++++++++- server/core/Reader/BookConverter/ConvertDocX.js | 4 ++-- server/core/Reader/BookConverter/ConvertEpub.js | 4 ++-- server/core/Reader/BookConverter/ConvertMobi.js | 4 ++-- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/server/core/Reader/BookConverter/ConvertBase.js b/server/core/Reader/BookConverter/ConvertBase.js index 11d07900..b2a5f712 100644 --- a/server/core/Reader/BookConverter/ConvertBase.js +++ b/server/core/Reader/BookConverter/ConvertBase.js @@ -42,7 +42,15 @@ class ConvertBase { } try { - const result = await utils.spawnProcess(path, {args, onData, abort}); + const result = await utils.spawnProcess(path, { + killAfter: 600, + args, + onData: (data) => { + q.resetTimeout(); + onData(data); + }, + abort + }); if (result.code != 0) { let error = result.code; if (this.config.branch == 'development') diff --git a/server/core/Reader/BookConverter/ConvertDocX.js b/server/core/Reader/BookConverter/ConvertDocX.js index 02b112c5..8be94ebd 100644 --- a/server/core/Reader/BookConverter/ConvertDocX.js +++ b/server/core/Reader/BookConverter/ConvertDocX.js @@ -22,8 +22,8 @@ class ConvertDocX extends ConvertBase { async convert(docxFile, fb2File, callback, abort) { let perc = 0; - await this.execConverter(this.calibrePath, [docxFile, fb2File], () => { - perc = (perc < 100 ? perc + 5 : 50); + await this.execConverter(this.calibrePath, [docxFile, fb2File, '-vv'], () => { + perc = (perc < 100 ? perc + 1 : 50); callback(perc); }, abort); diff --git a/server/core/Reader/BookConverter/ConvertEpub.js b/server/core/Reader/BookConverter/ConvertEpub.js index f0f9ba71..45c36d18 100644 --- a/server/core/Reader/BookConverter/ConvertEpub.js +++ b/server/core/Reader/BookConverter/ConvertEpub.js @@ -37,8 +37,8 @@ class ConvertEpub extends ConvertBase { await fs.copy(inputFiles.sourceFile, epubFile); let perc = 0; - await this.execConverter(this.calibrePath, [epubFile, fb2File], () => { - perc = (perc < 100 ? perc + 5 : 50); + await this.execConverter(this.calibrePath, [epubFile, fb2File, '-vv'], () => { + perc = (perc < 100 ? perc + 1 : 50); callback(perc); }, abort); diff --git a/server/core/Reader/BookConverter/ConvertMobi.js b/server/core/Reader/BookConverter/ConvertMobi.js index ee8535ef..4bae0ca7 100644 --- a/server/core/Reader/BookConverter/ConvertMobi.js +++ b/server/core/Reader/BookConverter/ConvertMobi.js @@ -25,8 +25,8 @@ class ConvertMobi extends ConvertBase { await fs.copy(inputFiles.sourceFile, mobiFile); let perc = 0; - await this.execConverter(this.calibrePath, [mobiFile, fb2File], () => { - perc = (perc < 100 ? perc + 5 : 50); + await this.execConverter(this.calibrePath, [mobiFile, fb2File, '-vv'], () => { + perc = (perc < 100 ? perc + 1 : 50); callback(perc); }, abort); From 0be05325e42762df5ca15da438edb1e3a62fea40 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 29 Jan 2020 01:02:05 +0700 Subject: [PATCH 57/58] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=20=D0=B1=D0=B0=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/Reader/BookConverter/ConvertBase.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/core/Reader/BookConverter/ConvertBase.js b/server/core/Reader/BookConverter/ConvertBase.js index b2a5f712..0c21bd12 100644 --- a/server/core/Reader/BookConverter/ConvertBase.js +++ b/server/core/Reader/BookConverter/ConvertBase.js @@ -34,6 +34,8 @@ class ConvertBase { } async execConverter(path, args, onData, abort) { + onData = (onData ? onData : () => {}); + let q = null; try { q = await queue.get(() => {onData();}); From 404b87d78de94897ec4b945805c414ad297201f5 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 30 Jan 2020 16:34:05 +0700 Subject: [PATCH 58/58] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D1=88=D0=B8=D0=B5=20=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA?= =?UTF-8?q?=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/ProgressPage/ProgressPage.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/components/Reader/ProgressPage/ProgressPage.vue b/client/components/Reader/ProgressPage/ProgressPage.vue index 49b31c29..361a075a 100644 --- a/client/components/Reader/ProgressPage/ProgressPage.vue +++ b/client/components/Reader/ProgressPage/ProgressPage.vue @@ -47,12 +47,13 @@ class ProgressPage extends Vue { hide() { this.visible = false; + this.text = ''; } setState(state) { if (state.state) { if (state.state == 'queue') { - this.text = 'Номер в очереди: ' + (state.place ? state.place : ''); + this.text = (state.place ? 'Номер в очереди: ' + state.place : ''); } else { this.text = (ruMessage[state.state] ? ruMessage[state.state] : state.state); }