diff --git a/client/api/misc.js b/client/api/misc.js index 79d56d0e..970b4799 100644 --- a/client/api/misc.js +++ b/client/api/misc.js @@ -1,4 +1,5 @@ import axios from 'axios'; +import wsc from './webSocketConnection'; const api = axios.create({ baseURL: '/api' @@ -6,9 +7,20 @@ const api = axios.create({ class Misc { async loadConfig() { - const response = await api.post('/config', {params: [ + + const query = {params: [ 'name', 'version', 'mode', 'maxUploadFileSize', 'useExternalBookConverter', 'branch', - ]}); + ]}; + + try { + await wsc.open(); + return await wsc.message(wsc.send(Object.assign({action: 'get-config'}, query))); + } catch (e) { + console.error(e); + } + + //если с WebSocket проблема, работаем по http + const response = await api.post('/config', query); return response.data; } } diff --git a/client/api/reader.js b/client/api/reader.js index b3e2da87..c17cccef 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -1,6 +1,6 @@ import axios from 'axios'; - import * as utils from '../share/utils'; +import wsc from './webSocketConnection'; const api = axios.create({ baseURL: '/api/reader' @@ -11,8 +11,67 @@ const workerApi = axios.create({ }); class Reader { + constructor() { + } + + async getWorkerStateFinish(workerId, callback) { + if (!callback) callback = () => {}; + + let response = {}; + + try { + await wsc.open(); + const requestId = wsc.send({action: 'worker-get-state-finish', workerId}); + + while (1) {// eslint-disable-line no-constant-condition + response = await wsc.message(requestId); + callback(response); + + if (!response.state) + throw new Error('Неверный ответ api'); + + if (response.state == 'finish' || response.state == 'error') { + break; + } + } + return response; + } catch (e) { + console.error(e); + } + + //если с WebSocket проблема, работаем по http + const refreshPause = 500; + let i = 0; + response = {}; + while (1) {// eslint-disable-line no-constant-condition + const prevProgress = response.progress || 0; + const prevState = response.state || 0; + response = await workerApi.post('/get-state', {workerId}); + response = response.data; + callback(response); + + if (!response.state) + throw new Error('Неверный ответ api'); + + if (response.state == 'finish' || response.state == 'error') { + break; + } + + if (i > 0) + await utils.sleep(refreshPause); + + i++; + if (i > 120*1000/refreshPause) {//2 мин ждем телодвижений воркера + throw new Error('Слишком долгое время ожидания'); + } + //проверка воркера + i = (prevProgress != response.progress || prevState != response.state ? 1 : i); + } + + return response; + } + async loadBook(opts, callback) { - const refreshPause = 300; if (!callback) callback = () => {}; let response = await api.post('/load-book', opts); @@ -22,62 +81,90 @@ class Reader { throw new Error('Неверный ответ api'); callback({totalSteps: 4}); + callback(response.data); - let i = 0; - while (1) {// eslint-disable-line no-constant-condition - callback(response.data); + response = await this.getWorkerStateFinish(workerId, callback); - if (response.data.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл + if (response) { + if (response.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл callback({step: 4}); - const book = await this.loadCachedBook(response.data.path, callback); - return Object.assign({}, response.data, {data: book.data}); + const book = await this.loadCachedBook(response.path, callback, response.size); + return Object.assign({}, response, {data: book.data}); } - if (response.data.state == 'error') { - let errMes = response.data.error; + + if (response.state == 'error') { + let errMes = response.error; if (errMes.indexOf('getaddrinfo') >= 0 || errMes.indexOf('ECONNRESET') >= 0 || errMes.indexOf('EINVAL') >= 0 || errMes.indexOf('404') >= 0) - errMes = `Ресурс не найден по адресу: ${response.data.url}`; + errMes = `Ресурс не найден по адресу: ${response.url}`; throw new Error(errMes); } - if (i > 0) - await utils.sleep(refreshPause); + } else { + throw new Error('Пустой ответ сервера'); + } + } - i++; - if (i > 120*1000/refreshPause) {//2 мин ждем телодвижений воркера - throw new Error('Слишком долгое время ожидания'); + async checkCachedBook(url) { + let estSize = -1; + try { + const response = await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); + + if (response.headers['content-length']) { + estSize = response.headers['content-length']; + } + } catch (e) { + //восстановим при необходимости файл на сервере из удаленного облака + let response = null + + try { + await wsc.open(); + response = await wsc.message(wsc.send({action: 'reader-restore-cached-file', path: url})); + } catch (e) { + console.error(e); + //если с WebSocket проблема, работаем по http + response = await api.post('/restore-cached-file', {path: url}); + response = response.data; + } + + const workerId = response.workerId; + if (!workerId) + throw new Error('Неверный ответ api'); + + response = await this.getWorkerStateFinish(workerId); + if (response.state == 'error') { + throw new Error(response.error); + } + if (response.size && estSize < 0) { + estSize = response.size; } - //проверка воркера - const prevProgress = response.data.progress; - const prevState = response.data.state; - response = await workerApi.post('/get-state', {workerId}); - i = (prevProgress != response.data.progress || prevState != response.data.state ? 1 : i); } + + return estSize; } - async checkUrl(url) { - return await axios.head(url, {headers: {'Cache-Control': 'no-cache'}}); - } - - async loadCachedBook(url, callback) { - const response = await axios.head(url); - - let estSize = 1000000; - if (response.headers['content-length']) { - estSize = response.headers['content-length']; - } + async loadCachedBook(url, callback, estSize = -1) { + if (!callback) callback = () => {}; callback({state: 'loading', progress: 0}); + + //получение размера файла + if (estSize && estSize < 0) { + estSize = await this.checkCachedBook(url); + } + + //получение файла + estSize = (estSize > 0 ? estSize : 1000000); const options = { - onDownloadProgress: progress => { + onDownloadProgress: (progress) => { while (progress.loaded > estSize) estSize *= 1.5; if (callback) callback({progress: Math.round((progress.loaded*100)/estSize)}); } } - //загрузка + return await axios.get(url, options); } @@ -114,13 +201,22 @@ class Reader { } async storage(request) { - let response = await api.post('/storage', request); + let response = null; + try { + await wsc.open(); + response = await wsc.message(wsc.send({action: 'reader-storage', body: request})); + } catch (e) { + console.error(e); + //если с WebSocket проблема, работаем по http + response = await api.post('/storage', request); + response = response.data; + } - const state = response.data.state; + const state = response.state; if (!state) throw new Error('Неверный ответ api'); - return response.data; + return response; } } diff --git a/client/api/webSocketConnection.js b/client/api/webSocketConnection.js new file mode 100644 index 00000000..bbcb21a3 --- /dev/null +++ b/client/api/webSocketConnection.js @@ -0,0 +1,176 @@ +const cleanPeriod = 60*1000;//1 минута + +class WebSocketConnection { + //messageLifeTime в минутах (cleanPeriod) + constructor(messageLifeTime = 5) { + this.ws = null; + this.timer = null; + this.listeners = []; + this.messageQueue = []; + this.messageLifeTime = messageLifeTime; + this.requestId = 0; + } + + addListener(listener) { + if (this.listeners.indexOf(listener) < 0) + this.listeners.push(Object.assign({regTime: Date.now()}, listener)); + } + + //рассылаем сообщение и удаляем те обработчики, которые его получили + emit(mes, isError) { + const len = this.listeners.length; + if (len > 0) { + let newListeners = []; + for (const listener of this.listeners) { + let emitted = false; + if (isError) { + if (listener.onError) + listener.onError(mes); + emitted = true; + } else { + if (listener.onMessage) { + if (listener.requestId) { + if (listener.requestId === mes.requestId) { + listener.onMessage(mes); + emitted = true; + } + } else { + listener.onMessage(mes); + emitted = true; + } + } else { + emitted = true; + } + } + + if (!emitted) + newListeners.push(listener); + } + this.listeners = newListeners; + } + + return this.listeners.length != len; + } + + open(url) { + return new Promise((resolve, reject) => { + if (this.ws && this.ws.readyState == WebSocket.OPEN) { + resolve(this.ws); + } else { + let protocol = 'ws:'; + if (window.location.protocol == 'https:') { + protocol = 'wss:' + } + + url = url || `${protocol}//${window.location.host}/ws`; + + this.ws = new WebSocket(url); + + if (this.timer) { + clearTimeout(this.timer); + } + this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod); + + let resolved = false; + this.ws.onopen = (e) => { + resolved = true; + resolve(e); + }; + + this.ws.onmessage = (e) => { + try { + const mes = JSON.parse(e.data); + this.messageQueue.push({regTime: Date.now(), mes}); + + let newMessageQueue = []; + for (const message of this.messageQueue) { + if (!this.emit(message.mes)) { + newMessageQueue.push(message); + } + } + + this.messageQueue = newMessageQueue; + } catch (e) { + this.emit(e.message, true); + } + }; + + this.ws.onerror = (e) => { + this.emit(e.message, true); + if (!resolved) + reject(e); + }; + } + }); + } + + //timeout в минутах (cleanPeriod) + message(requestId, timeout = 2) { + return new Promise((resolve, reject) => { + this.addListener({ + requestId, + timeout, + onMessage: (mes) => { + if (mes.error) { + reject(mes.error); + } else { + resolve(mes); + } + }, + onError: (e) => { + reject(e); + } + }); + }); + } + + send(req) { + if (this.ws && this.ws.readyState == WebSocket.OPEN) { + const requestId = ++this.requestId; + this.ws.send(JSON.stringify(Object.assign({requestId}, req))); + return requestId; + } else { + throw new Error('WebSocket connection is not ready'); + } + } + + close() { + if (this.ws && this.ws.readyState == WebSocket.OPEN) { + this.ws.close(); + } + } + + periodicClean() { + try { + this.timer = null; + + const now = Date.now(); + //чистка listeners + let newListeners = []; + for (const listener of this.listeners) { + if (now - listener.regTime < listener.timeout*cleanPeriod - 50) { + newListeners.push(listener); + } else { + if (listener.onError) + listener.onError('Время ожидания ответа истекло'); + } + } + this.listeners = newListeners; + + //чистка messageQueue + let newMessageQueue = []; + for (const message of this.messageQueue) { + if (now - message.regTime < this.messageLifeTime*cleanPeriod - 50) { + newMessageQueue.push(message); + } + } + this.messageQueue = newMessageQueue; + } finally { + if (this.ws.readyState == WebSocket.OPEN) { + this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod); + } + } + } +} + +export default new WebSocketConnection(); \ No newline at end of file diff --git a/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue b/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue index 637ce6e7..4ae7dbfb 100644 --- a/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue +++ b/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue @@ -1,30 +1,54 @@ @@ -200,6 +247,7 @@ class Reader extends Vue { whatsNewVisible = false; whatsNewContent = ''; + donationVisible = false; created() { this.loading = true; @@ -258,9 +306,10 @@ class Reader extends Vue { this.checkActivateDonateHelpPage(); this.loading = false; - await this.showWhatsNew(); - this.updateRoute(); + + await this.showWhatsNew(); + await this.showDonation(); })(); } @@ -272,6 +321,7 @@ class Reader extends Vue { this.clickControl = settings.clickControl; this.blinkCachedLoad = settings.blinkCachedLoad; this.showWhatsNewDialog = settings.showWhatsNewDialog; + this.showDonationDialog2020 = settings.showDonationDialog2020; this.showToolButton = settings.showToolButton; this.enableSitesFilter = settings.enableSitesFilter; @@ -337,6 +387,41 @@ class Reader extends Vue { } } + async showDonation() { + await utils.sleep(3000); + const today = utils.formatDate(new Date(), 'coDate'); + + if (this.mode == 'omnireader' && today < '2020-03-01' && this.showDonationDialog2020 && this.donationRemindDate != today) { + this.donationVisible = true; + } + } + + donationDialogDisable() { + this.donationVisible = false; + if (this.showDonationDialog2020) { + const newSettings = Object.assign({}, this.settings, { showDonationDialog2020: false }); + this.commit('reader/setSettings', newSettings); + } + } + + donationDialogRemind() { + this.donationVisible = false; + this.commit('reader/setDonationRemindDate', utils.formatDate(new Date(), 'coDate')); + } + + openDonate() { + this.donationVisible = false; + this.donateToggle(); + } + + async copyLink(link) { + const result = await utils.copyTextToClipboard(link); + if (result) + this.$notify.success({message: `Ссылка ${link} успешно скопирована в буфер обмена`}); + else + this.$notify.error({message: 'Копирование не удалось'}); + } + openVersionHistory() { this.whatsNewVisible = false; this.versionHistoryToggle(); @@ -455,6 +540,10 @@ class Reader extends Vue { return this.$store.state.reader.whatsNewContentHash; } + get donationRemindDate() { + return this.$store.state.reader.donationRemindDate; + } + addAction(pos) { let a = this.actionList; if (!a.length || a[a.length - 1] != pos) { @@ -719,15 +808,16 @@ class Reader extends Vue { case 'scrolling': case 'search': case 'copyText': - case 'recentBooks': + case 'refresh': case 'offlineMode': + case 'recentBooks': case 'settings': - if (this[`${button}Active`]) + if (this.progressActive) { + classResult = classDisabled; + } else if (this[`${button}Active`]) { classResult = classActive; + } break; - } - - switch (button) { case 'undoAction': if (this.actionCur <= 0) classResult = classDisabled; diff --git a/client/components/Reader/RecentBooksPage/RecentBooksPage.vue b/client/components/Reader/RecentBooksPage/RecentBooksPage.vue index e3511dbe..312f5020 100644 --- a/client/components/Reader/RecentBooksPage/RecentBooksPage.vue +++ b/client/components/Reader/RecentBooksPage/RecentBooksPage.vue @@ -272,7 +272,7 @@ class RecentBooksPage extends Vue { async downloadBook(fb2path) { try { - await readerApi.checkUrl(fb2path); + await readerApi.checkCachedBook(fb2path); const d = this.$refs.download; d.href = fb2path; diff --git a/client/components/Reader/SettingsPage/SettingsPage.vue b/client/components/Reader/SettingsPage/SettingsPage.vue index d7e736c2..3f5a4b0c 100644 --- a/client/components/Reader/SettingsPage/SettingsPage.vue +++ b/client/components/Reader/SettingsPage/SettingsPage.vue @@ -471,6 +471,14 @@ Показывать уведомление "Что нового" + + + + Показывать "Оплатим хостинг вместе" + + diff --git a/client/components/Reader/share/bookManager.js b/client/components/Reader/share/bookManager.js index 3ef65532..71c77381 100644 --- a/client/components/Reader/share/bookManager.js +++ b/client/components/Reader/share/bookManager.js @@ -464,7 +464,7 @@ class BookManager { addEventListener(listener) { if (this.eventListeners.indexOf(listener) < 0) - this.eventListeners.push(listener); + this.eventListeners.push(listener); } removeEventListener(listener) { diff --git a/client/components/Reader/versionHistory.js b/client/components/Reader/versionHistory.js index 7d7f0afb..98d1ced4 100644 --- a/client/components/Reader/versionHistory.js +++ b/client/components/Reader/versionHistory.js @@ -1,4 +1,27 @@ export const versionHistory = [ +{ + showUntil: '2020-01-27', + header: '0.8.3 (2020-01-28)', + content: +` + +` +}, + +{ + showUntil: '2020-01-19', + header: '0.8.2 (2020-01-20)', + content: +` + +` +}, + { showUntil: '2020-01-06', header: '0.8.1 (2020-01-07)', diff --git a/client/element.js b/client/element.js index 370ab1b4..39148440 100644 --- a/client/element.js +++ b/client/element.js @@ -19,6 +19,7 @@ import ElCheckbox from 'element-ui/lib/checkbox'; import ElTabs from 'element-ui/lib/tabs'; import ElTabPane from 'element-ui/lib/tab-pane'; import ElTooltip from 'element-ui/lib/tooltip'; +import ElRow from 'element-ui/lib/row'; import ElCol from 'element-ui/lib/col'; import ElContainer from 'element-ui/lib/container'; import ElAside from 'element-ui/lib/aside'; @@ -43,7 +44,7 @@ import MessageBox from 'element-ui/lib/message-box'; const components = { ElMenu, ElMenuItem, ElButton, ElButtonGroup, ElCheckbox, ElTabs, ElTabPane, ElTooltip, - ElCol, ElContainer, ElAside, ElMain, ElHeader, + ElRow, ElCol, ElContainer, ElAside, ElMain, ElHeader, ElInput, ElInputNumber, ElSelect, ElOption, ElTable, ElTableColumn, ElProgress, ElSlider, ElForm, ElFormItem, ElColorPicker, ElDialog, diff --git a/client/store/modules/reader.js b/client/store/modules/reader.js index 7e25c9c4..814b0984 100644 --- a/client/store/modules/reader.js +++ b/client/store/modules/reader.js @@ -182,6 +182,7 @@ const settingDefaults = { imageFitWidth: true, showServerStorageMessages: true, showWhatsNewDialog: true, + showDonationDialog2020: true, enableSitesFilter: true, fontShifts: {}, @@ -204,6 +205,7 @@ const state = { profilesRev: 0, allowProfilesSave: false,//подстраховка для разработки whatsNewContentHash: '', + donationRemindDate: '', currentProfile: '', settings: Object.assign({}, settingDefaults), settingsRev: {}, @@ -238,6 +240,9 @@ const mutations = { setWhatsNewContentHash(state, value) { state.whatsNewContentHash = value; }, + setDonationRemindDate(state, value) { + state.donationRemindDate = value; + }, setCurrentProfile(state, value) { state.currentProfile = value; }, diff --git a/docs/omnireader/omnireader b/docs/omnireader/omnireader index a638b392..3e535374 100644 --- a/docs/omnireader/omnireader +++ b/docs/omnireader/omnireader @@ -8,6 +8,7 @@ server { server_name omnireader.ru; client_max_body_size 50m; + proxy_read_timeout 1h; gzip on; gzip_min_length 1024; @@ -18,6 +19,13 @@ server { proxy_pass http://127.0.0.1:44081; } + location /ws { + proxy_pass http://127.0.0.1:44081; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location / { root /home/liberama/public; diff --git a/docs/omnireader/omnireader_http b/docs/omnireader/omnireader_http new file mode 100644 index 00000000..768fce76 --- /dev/null +++ b/docs/omnireader/omnireader_http @@ -0,0 +1,59 @@ +server { + listen 80; + server_name omnireader.ru; + + client_max_body_size 50m; + proxy_read_timeout 1h; + + gzip on; + gzip_min_length 1024; + gzip_proxied expired no-cache no-store private auth; + gzip_types *; + + location /api { + proxy_pass http://127.0.0.1:44081; + } + + location /ws { + proxy_pass http://127.0.0.1:44081; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + location / { + root /home/liberama/public; + + location /tmp { + add_header Content-Type text/xml; + add_header Content-Encoding gzip; + } + + location ~* \.(?:manifest|appcache|html)$ { + expires -1; + } + } +} + +server { + listen 80; + server_name old.omnireader.ru; + + client_max_body_size 50m; + + gzip on; + gzip_min_length 1024; + gzip_proxied expired no-cache no-store private auth; + gzip_types *; + + root /home/oldreader; + + index index.html; + + # Обработка php файлов с помощью fpm + location ~ \.php$ { + try_files $uri =404; + include /etc/nginx/fastcgi.conf; + fastcgi_pass unix:/run/php/php7.2-fpm.sock; + } +} diff --git a/package-lock.json b/package-lock.json index 641d76f6..117a6764 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.8.1", + "version": "0.8.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -1725,6 +1725,11 @@ } } }, + "base-64": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", + "integrity": "sha1-eAqZyE59YAJgNhURxId2E78k9rs=" + }, "base-x": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/base-x/-/base-x-3.0.7.tgz", @@ -5719,6 +5724,11 @@ "parse-passwd": "^1.0.0" } }, + "hot-patcher": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/hot-patcher/-/hot-patcher-0.5.0.tgz", + "integrity": "sha512-2Uu2W0s8+dnqXzdlg0MRsRzPoDCs1wVjOGSyMRRaMzLDX4bgHw6xDYKccsWafXPPxQpkQfEjgW6+17pwcg60bw==" + }, "hsl-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/hsl-regex/-/hsl-regex-1.0.0.tgz", @@ -6902,6 +6912,11 @@ } } }, + "merge": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/merge/-/merge-1.2.1.tgz", + "integrity": "sha512-VjFo4P5Whtj4vsLzsYBu5ayHhoHJ0UqNm7ibvShmbmoz7tGi0vXaoJbGdB+GmDMLUdg8DpQXEIeVDAe8MaABvQ==" + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -7882,6 +7897,11 @@ "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", "dev": true }, + "path-posix": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/path-posix/-/path-posix-1.0.0.tgz", + "integrity": "sha1-BrJhE/Vr6rBCVFojv6iAA8ysJg8=" + }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", @@ -10428,6 +10448,11 @@ "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=", "dev": true }, + "querystringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.1.tgz", + "integrity": "sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA==" + }, "randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -10714,6 +10739,11 @@ "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", "dev": true }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=" + }, "resize-observer-polyfill": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz", @@ -12494,6 +12524,11 @@ } } }, + "url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==" + }, "url-loader": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/url-loader/-/url-loader-1.1.2.tgz", @@ -12513,6 +12548,15 @@ } } }, + "url-parse": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", + "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "url-parse-lax": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", @@ -12742,6 +12786,32 @@ "neo-async": "^2.5.0" } }, + "webdav": { + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/webdav/-/webdav-2.10.1.tgz", + "integrity": "sha512-3UfnjGTAqSM9MW3Rpt1KrY1KneYK0wPCFryHTncqw1OP1pyiniT3uYhVpgmH6za/TkWOfnTnKCDKhwrLJFdzow==", + "requires": { + "axios": "^0.19.0", + "base-64": "^0.1.0", + "hot-patcher": "^0.5.0", + "merge": "^1.2.1", + "minimatch": "^3.0.4", + "path-posix": "^1.0.0", + "url-join": "^4.0.1", + "url-parse": "^1.4.7", + "xml2js": "^0.4.19" + }, + "dependencies": { + "axios": { + "version": "0.19.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.19.1.tgz", + "integrity": "sha512-Yl+7nfreYKaLRvAvjNPkvfjnQHJM1yLBY3zhqAwcJSwR/6ETkanUgylgtIvkvz0xJ+p/vZuNw8X7Hnb7Whsbpw==", + "requires": { + "follow-redirects": "1.5.10" + } + } + } + }, "webpack": { "version": "4.40.2", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.40.2.tgz", @@ -13010,6 +13080,25 @@ "mkdirp": "^0.5.1" } }, + "ws": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz", + "integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A==" + }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 22622e9f..3499ebcf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.8.1", + "version": "0.8.3", "author": "Book Pauk ", "license": "CC0-1.0", "repository": "bookpauk/liberama", @@ -85,6 +85,8 @@ "vue-router": "^3.1.3", "vuex": "^3.1.1", "vuex-persistedstate": "^2.5.4", + "webdav": "^2.10.1", + "ws": "^7.2.1", "zip-stream": "^2.1.2" } } diff --git a/server/config/base.js b/server/config/base.js index 33007a57..7370cec7 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -21,7 +21,7 @@ module.exports = { maxTempPublicDirSize: 512*1024*1024,//512Мб maxUploadPublicDirSize: 200*1024*1024,//100Мб - useExternalBookConverter: false, + useExternalBookConverter: false, db: [ { @@ -45,5 +45,14 @@ module.exports = { }, ], + remoteWebDavStorage: false, + /* + remoteWebDavStorage: { + url: '127.0.0.1:1900', + username: '', + password: '', + }, + */ + }; diff --git a/server/config/index.js b/server/config/index.js index bd17f7b3..7ffa99f1 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -10,6 +10,7 @@ const propsToSave = [ 'useExternalBookConverter', 'servers', + 'remoteWebDavStorage', ]; let instance = null; diff --git a/server/controllers/ReaderController.js b/server/controllers/ReaderController.js index c2288ec7..5e9a4879 100644 --- a/server/controllers/ReaderController.js +++ b/server/controllers/ReaderController.js @@ -35,9 +35,9 @@ class ReaderController extends BaseController { const request = req.body; let error = ''; try { - if (!request.action) + if (!request.action) throw new Error(`key 'action' is empty`); - if (!request.items || Array.isArray(request.data)) + if (!request.items || Array.isArray(request.data)) throw new Error(`key 'items' is empty`); return await this.readerStorage.doAction(request); @@ -62,6 +62,24 @@ class ReaderController extends BaseController { res.status(400).send({error}); return false; } + + async restoreCachedFile(req, res) { + const request = req.body; + let error = ''; + try { + if (!request.path) + throw new Error(`key 'path' is empty`); + + const workerId = this.readerWorker.restoreCachedFile(request.path); + const state = this.workerState.getState(workerId); + return (state ? state : {}); + } catch (e) { + error = e.message; + } + //bad request + res.status(400).send({error}); + return false; + } } module.exports = ReaderController; diff --git a/server/controllers/WebSocketController.js b/server/controllers/WebSocketController.js new file mode 100644 index 00000000..33b61a0b --- /dev/null +++ b/server/controllers/WebSocketController.js @@ -0,0 +1,164 @@ +const WebSocket = require ('ws'); +const _ = require('lodash'); + +const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton +const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton +const WorkerState = require('../core/WorkerState');//singleton +const log = new (require('../core/AppLogger'))().log;//singleton +const utils = require('../core/utils'); + +const cleanPeriod = 1*60*1000;//1 минута +const closeSocketOnIdle = 5*60*1000;//5 минут + +class WebSocketController { + constructor(wss, config) { + this.config = config; + this.isDevelopment = (config.branch == 'development'); + + this.readerStorage = new ReaderStorage(); + this.readerWorker = new ReaderWorker(config); + this.workerState = new WorkerState(); + + this.wss = wss; + + wss.on('connection', (ws) => { + ws.on('message', (message) => { + this.onMessage(ws, message); + }); + }); + + setTimeout(() => { this.periodicClean(); }, cleanPeriod); + } + + periodicClean() { + try { + const now = Date.now(); + this.wss.clients.forEach((ws) => { + if (!ws.lastActivity || now - ws.lastActivity > closeSocketOnIdle - 50) { + ws.terminate(); + } + }); + } finally { + setTimeout(() => { this.periodicClean(); }, cleanPeriod); + } + } + + async onMessage(ws, message) { + let req = {}; + try { + if (this.isDevelopment) { + log(`WebSocket-IN: ${message.substr(0, 4000)}`); + } + + ws.lastActivity = Date.now(); + req = JSON.parse(message); + switch (req.action) { + case 'test': + await this.test(req, ws); break; + case 'get-config': + await this.getConfig(req, ws); break; + case 'worker-get-state': + await this.workerGetState(req, ws); break; + case 'worker-get-state-finish': + await this.workerGetStateFinish(req, ws); break; + case 'reader-restore-cached-file': + await this.readerRestoreCachedFile(req, ws); break; + case 'reader-storage': + await this.readerStorageDo(req, ws); break; + + default: + throw new Error(`Action not found: ${req.action}`); + } + } catch (e) { + this.send({error: e.message}, req, ws); + } + } + + send(res, req, ws) { + if (ws.readyState == WebSocket.OPEN) { + ws.lastActivity = Date.now(); + let r = res; + if (req.requestId) + r = Object.assign({requestId: req.requestId}, r); + + const message = JSON.stringify(r); + ws.send(message); + + if (this.isDevelopment) { + log(`WebSocket-OUT: ${message.substr(0, 4000)}`); + } + + } + } + + //Actions ------------------------------------------------------------------ + async test(req, ws) { + this.send({message: 'Liberama project is awesome'}, req, ws); + } + + async getConfig(req, ws) { + if (Array.isArray(req.params)) { + this.send(_.pick(this.config, req.params), req, ws); + } else { + throw new Error('params is not an array'); + } + } + + async workerGetState(req, ws) { + if (!req.workerId) + throw new Error(`key 'workerId' is wrong`); + + const state = this.workerState.getState(req.workerId); + this.send((state ? state : {}), req, ws); + } + + async workerGetStateFinish(req, ws) { + if (!req.workerId) + throw new Error(`key 'workerId' is wrong`); + + const refreshPause = 200; + let i = 0; + let state = {}; + while (1) {// eslint-disable-line no-constant-condition + const prevProgress = state.progress || -1; + const prevState = state.state || ''; + state = this.workerState.getState(req.workerId); + + this.send((state ? state : {}), req, ws); + if (!state) break; + + if (state.state != 'finish' && state.state != 'error') + await utils.sleep(refreshPause); + else + break; + + i++; + if (i > 2*60*1000/refreshPause) {//2 мин ждем телодвижений воркера + this.send({state: 'error', error: 'Время ожидания процесса истекло'}, req, ws); + } + i = (prevProgress != state.progress || prevState != state.state ? 1 : i); + } + } + + async readerRestoreCachedFile(req, ws) { + if (!req.path) + throw new Error(`key 'path' is empty`); + + const workerId = this.readerWorker.restoreCachedFile(req.path); + const state = this.workerState.getState(workerId); + this.send((state ? state : {}), req, ws); + } + + async readerStorageDo(req, ws) { + if (!req.body) + throw new Error(`key 'body' is empty`); + if (!req.body.action) + throw new Error(`key 'action' is empty`); + if (!req.body.items || Array.isArray(req.body.data)) + throw new Error(`key 'items' is empty`); + + this.send(await this.readerStorage.doAction(req.body), req, ws); + } +} + +module.exports = WebSocketController; diff --git a/server/controllers/WorkerController.js b/server/controllers/WorkerController.js index 69bd4982..e0ef77a9 100644 --- a/server/controllers/WorkerController.js +++ b/server/controllers/WorkerController.js @@ -1,5 +1,6 @@ const BaseController = require('./BaseController'); const WorkerState = require('../core/WorkerState');//singleton +const utils = require('../core/utils'); class WorkerController extends BaseController { constructor(config) { @@ -15,6 +16,7 @@ class WorkerController extends BaseController { throw new Error(`key 'workerId' is wrong`); const state = this.workerState.getState(request.workerId); + return (state ? state : {}); } catch (e) { error = e.message; @@ -23,6 +25,60 @@ class WorkerController extends BaseController { res.status(400).send({error}); return false; } + + //TODO: удалить бесполезную getStateFinish + async getStateFinish(req, res) { + const request = req.body; + let error = ''; + try { + if (!request.workerId) + throw new Error(`key 'workerId' is wrong`); + + res.writeHead(200, { + 'Content-Type': 'text/json; charset=utf-8', + }); + + const splitter = '-- aod2t5hDXU32bUFyqlFE next status --'; + const refreshPause = 200; + let i = 0; + let prevProgress = -1; + let prevState = ''; + let state; + while (1) {// eslint-disable-line no-constant-condition + state = this.workerState.getState(request.workerId); + if (!state) break; + + res.write(splitter + JSON.stringify(state)); + res.flush(); + + if (state.state != 'finish' && state.state != 'error') + await utils.sleep(refreshPause); + else + break; + + i++; + if (i > 2*60*1000/refreshPause) {//2 мин ждем телодвижений воркера + res.write(splitter + JSON.stringify({state: 'error', error: 'Слишком долгое время ожидания'})); + break; + } + i = (prevProgress != state.progress || prevState != state.state ? 1 : i); + prevProgress = state.progress; + prevState = state.state; + } + + if (!state) { + res.write(splitter + JSON.stringify({})); + } + + res.end(); + return false; + } catch (e) { + error = e.message; + } + //bad request + res.status(400).send({error}); + return false; + } } module.exports = WorkerController; diff --git a/server/controllers/index.js b/server/controllers/index.js index fbbad7b7..9c2f4f07 100644 --- a/server/controllers/index.js +++ b/server/controllers/index.js @@ -2,4 +2,5 @@ module.exports = { MiscController: require('./MiscController'), ReaderController: require('./ReaderController'), WorkerController: require('./WorkerController'), + WebSocketController: require('./WebSocketController'), } \ No newline at end of file diff --git a/server/core/FileDecompressor.js b/server/core/FileDecompressor.js index 7b7c68fc..cc7a4c0f 100644 --- a/server/core/FileDecompressor.js +++ b/server/core/FileDecompressor.js @@ -5,12 +5,14 @@ const unbzip2Stream = require('unbzip2-stream'); const tar = require('tar-fs'); const ZipStreamer = require('./ZipStreamer'); +const appLogger = new (require('./AppLogger'))();//singleton const utils = require('./utils'); const FileDetector = require('./FileDetector'); class FileDecompressor { - constructor() { + constructor(limitFileSize = 0) { this.detector = new FileDetector(); + this.limitFileSize = limitFileSize; } async decompressNested(filename, outputDir) { @@ -112,7 +114,7 @@ class FileDecompressor { async unZip(filename, outputDir) { const zip = new ZipStreamer(); - return await zip.unpack(filename, outputDir); + return await zip.unpack(filename, outputDir, null, this.limitFileSize); } unBz2(filename, outputDir) { @@ -124,9 +126,16 @@ class FileDecompressor { } unTar(filename, outputDir) { - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { (async() => { const files = []; + if (this.limitFileSize) { + if ((await fs.stat(filename)).size > this.limitFileSize) { + reject('Файл слишком большой'); + return; + } + } + const tarExtract = tar.extract(outputDir, { map: (header) => { files.push({path: header.name, size: header.size}); @@ -148,7 +157,7 @@ class FileDecompressor { }); inputStream.pipe(tarExtract); - }); + })().catch(reject); }); } decompressByStream(stream, filename, outputDir) { @@ -173,6 +182,16 @@ class FileDecompressor { }); stream.on('error', reject); + + if (this.limitFileSize) { + let readSize = 0; + stream.on('data', (buffer) => { + readSize += buffer.length; + if (readSize > this.limitFileSize) + stream.destroy(new Error('Файл слишком большой')); + }); + } + inputStream.on('error', reject); outputStream.on('error', reject); @@ -189,9 +208,9 @@ class FileDecompressor { }); } - async gzipFile(inputFile, outputFile) { + async gzipFile(inputFile, outputFile, level = 1) { return new Promise((resolve, reject) => { - const gzip = zlib.createGzip({level: 1}); + const gzip = zlib.createGzip({level}); const input = fs.createReadStream(inputFile); const output = fs.createWriteStream(outputFile); @@ -208,7 +227,21 @@ class FileDecompressor { const outFilename = `${outDir}/${hash}`; if (!await fs.pathExists(outFilename)) { - await this.gzipFile(filename, outFilename); + await this.gzipFile(filename, outFilename, 1); + + // переупакуем через некоторое время на максималках + const filenameCopy = `${filename}.copy`; + await fs.copy(filename, filenameCopy); + + (async() => { + await utils.sleep(5000); + const filenameGZ = `${filename}.gz`; + await this.gzipFile(filenameCopy, filenameGZ, 9); + + await fs.move(filenameGZ, outFilename, {overwrite: true}); + + await fs.remove(filenameCopy); + })().catch((e) => { if (appLogger.inited) appLogger.log(LM_ERR, `FileDecompressor.gzipFileIfNotExists: ${e.message}`) }); } else { await utils.touchFile(outFilename); } diff --git a/server/core/FileDownloader.js b/server/core/FileDownloader.js index 78c0df3a..018acaf4 100644 --- a/server/core/FileDownloader.js +++ b/server/core/FileDownloader.js @@ -1,12 +1,11 @@ const got = require('got'); -const maxDownloadSize = 50*1024*1024; - class FileDownloader { - constructor() { + constructor(limitDownloadSize = 0) { + this.limitDownloadSize = limitDownloadSize; } - async load(url, callback) { + async load(url, callback, abort) { let errMes = ''; const options = { encoding: null, @@ -23,10 +22,14 @@ class FileDownloader { } let prevProg = 0; - const request = got(url, options).on('downloadProgress', progress => { - if (progress.transferred > maxDownloadSize) { - errMes = 'file too big'; - request.cancel(); + const request = got(url, options); + + request.on('downloadProgress', progress => { + if (this.limitDownloadSize) { + if (progress.transferred > this.limitDownloadSize) { + errMes = 'Файл слишком большой'; + request.cancel(); + } } let prog = 0; @@ -38,8 +41,12 @@ class FileDownloader { if (prog != prevProg && callback) callback(prog); prevProg = prog; - }); + if (abort && abort()) { + errMes = 'abort'; + request.cancel(); + } + }); try { return (await request).body; diff --git a/server/core/LimitedQueue.js b/server/core/LimitedQueue.js new file mode 100644 index 00000000..393a53a8 --- /dev/null +++ b/server/core/LimitedQueue.js @@ -0,0 +1,119 @@ +class LimitedQueue { + constructor(enqueueAfter = 10, size = 100, timeout = 60*60*1000) {//timeout в ms + this.size = size; + this.timeout = timeout; + + this.abortCount = 0; + this.enqueueAfter = enqueueAfter; + this.freed = enqueueAfter; + this.listeners = []; + } + + _addListener(listener) { + this.listeners.push(listener); + } + + //отсылаем сообщение первому ожидающему и удаляем его из списка + _emitFree() { + if (this.listeners.length > 0) { + let listener = this.listeners.shift(); + listener.onFree(); + + for (let i = 0; i < this.listeners.length; i++) { + this.listeners[i].onPlaceChange(i + 1); + } + } + } + + get(onPlaceChange) { + return new Promise((resolve, reject) => { + if (this.destroyed) + reject('destroyed'); + + const take = () => { + if (this.freed <= 0) + throw new Error('Ошибка получения ресурсов в очереди ожидания'); + + this.freed--; + this.resetTimeout(); + + let aCount = this.abortCount; + return { + ret: () => { + if (aCount == this.abortCount) { + this.freed++; + this._emitFree(); + aCount = -1; + this.resetTimeout(); + } + }, + abort: () => { + return (aCount != this.abortCount); + }, + resetTimeout: this.resetTimeout.bind(this) + }; + }; + + if (this.freed > 0) { + resolve(take()); + } else { + if (this.listeners.length < this.size) { + this._addListener({ + onFree: () => { + resolve(take()); + }, + onError: (err) => { + reject(err); + }, + onPlaceChange: (i) => { + if (onPlaceChange) + onPlaceChange(i); + } + }); + if (onPlaceChange) + onPlaceChange(this.listeners.length); + } else { + reject('Превышен размер очереди ожидания'); + } + } + }); + } + + resetTimeout() { + if (this.timer) + clearTimeout(this.timer); + this.timer = setTimeout(() => { this.clean(); }, this.timeout); + } + + clean() { + this.timer = null; + + if (this.freed < this.enqueueAfter) { + this.abortCount++; + //чистка listeners + for (const listener of this.listeners) { + listener.onError('Время ожидания в очереди истекло'); + } + this.listeners = []; + + this.freed = this.enqueueAfter; + } + } + + destroy() { + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + + for (const listener of this.listeners) { + listener.onError('destroy'); + } + this.listeners = []; + this.abortCount++; + + this.destroyed = true; + } +} + +module.exports = LimitedQueue; \ No newline at end of file diff --git a/server/core/Reader/BookConverter/ConvertBase.js b/server/core/Reader/BookConverter/ConvertBase.js index d3ad74ce..0c21bd12 100644 --- a/server/core/Reader/BookConverter/ConvertBase.js +++ b/server/core/Reader/BookConverter/ConvertBase.js @@ -3,10 +3,11 @@ const iconv = require('iconv-lite'); const chardet = require('chardet'); const he = require('he'); +const LimitedQueue = require('../../LimitedQueue'); const textUtils = require('./textUtils'); const utils = require('../../utils'); -let execConverterCounter = 0; +const queue = new LimitedQueue(2, 20, 3*60*1000);//3 минуты ожидание подвижек class ConvertBase { constructor(config) { @@ -32,13 +33,26 @@ class ConvertBase { throw new Error('Внешний конвертер pdftohtml не найден'); } - async execConverter(path, args, onData) { - execConverterCounter++; + async execConverter(path, args, onData, abort) { + onData = (onData ? onData : () => {}); + + let q = null; try { - if (execConverterCounter > 10) - throw new Error('Слишком большая очередь конвертирования. Пожалуйста, попробуйте позже.'); + q = await queue.get(() => {onData();}); + } catch (e) { + throw new Error('Слишком большая очередь конвертирования. Пожалуйста, попробуйте позже.'); + } - const result = await utils.spawnProcess(path, {args, onData}); + try { + const result = await utils.spawnProcess(path, { + killAfter: 600, + args, + onData: (data) => { + q.resetTimeout(); + onData(data); + }, + abort + }); if (result.code != 0) { let error = result.code; if (this.config.branch == 'development') @@ -48,13 +62,15 @@ class ConvertBase { } catch(e) { if (e.status == 'killed') { throw new Error('Слишком долгое ожидание конвертера'); + } else if (e.status == 'abort') { + throw new Error('abort'); } else if (e.status == 'error') { throw new Error(e.error); } else { throw new Error(e); } } finally { - execConverterCounter--; + q.ret(); } } diff --git a/server/core/Reader/BookConverter/ConvertDoc.js b/server/core/Reader/BookConverter/ConvertDoc.js index ce627eb8..a0c24191 100644 --- a/server/core/Reader/BookConverter/ConvertDoc.js +++ b/server/core/Reader/BookConverter/ConvertDoc.js @@ -16,7 +16,7 @@ class ConvertDoc extends ConvertDocX { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const docFile = `${outFile}.doc`; @@ -24,9 +24,9 @@ class ConvertDoc extends ConvertDocX { const fb2File = `${outFile}.fb2`; await fs.copy(inputFiles.sourceFile, docFile); - await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, docFile]); + await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, docFile], null, abort); - return await super.convert(docxFile, fb2File, callback); + return await super.convert(docxFile, fb2File, callback, abort); } } diff --git a/server/core/Reader/BookConverter/ConvertDocX.js b/server/core/Reader/BookConverter/ConvertDocX.js index d8c85b78..8be94ebd 100644 --- a/server/core/Reader/BookConverter/ConvertDocX.js +++ b/server/core/Reader/BookConverter/ConvertDocX.js @@ -20,12 +20,12 @@ class ConvertDocX extends ConvertBase { return false; } - async convert(docxFile, fb2File, callback) { + async convert(docxFile, fb2File, callback, abort) { let perc = 0; - await this.execConverter(this.calibrePath, [docxFile, fb2File], () => { - perc = (perc < 100 ? perc + 5 : 50); + await this.execConverter(this.calibrePath, [docxFile, fb2File, '-vv'], () => { + perc = (perc < 100 ? perc + 1 : 50); callback(perc); - }); + }, abort); return await fs.readFile(fb2File); } @@ -35,7 +35,7 @@ class ConvertDocX extends ConvertBase { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const docxFile = `${outFile}.docx`; @@ -43,7 +43,7 @@ class ConvertDocX extends ConvertBase { await fs.copy(inputFiles.sourceFile, docxFile); - return await this.convert(docxFile, fb2File, callback); + return await this.convert(docxFile, fb2File, callback, abort); } } diff --git a/server/core/Reader/BookConverter/ConvertEpub.js b/server/core/Reader/BookConverter/ConvertEpub.js index 79b25b8e..45c36d18 100644 --- a/server/core/Reader/BookConverter/ConvertEpub.js +++ b/server/core/Reader/BookConverter/ConvertEpub.js @@ -28,7 +28,7 @@ class ConvertEpub extends ConvertBase { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const epubFile = `${outFile}.epub`; @@ -37,10 +37,10 @@ class ConvertEpub extends ConvertBase { await fs.copy(inputFiles.sourceFile, epubFile); let perc = 0; - await this.execConverter(this.calibrePath, [epubFile, fb2File], () => { - perc = (perc < 100 ? perc + 5 : 50); + await this.execConverter(this.calibrePath, [epubFile, fb2File, '-vv'], () => { + perc = (perc < 100 ? perc + 1 : 50); callback(perc); - }); + }, abort); return await fs.readFile(fb2File); } diff --git a/server/core/Reader/BookConverter/ConvertMobi.js b/server/core/Reader/BookConverter/ConvertMobi.js index e8d6172b..4bae0ca7 100644 --- a/server/core/Reader/BookConverter/ConvertMobi.js +++ b/server/core/Reader/BookConverter/ConvertMobi.js @@ -16,7 +16,7 @@ class ConvertMobi extends ConvertBase { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const mobiFile = `${outFile}.mobi`; @@ -25,10 +25,10 @@ class ConvertMobi extends ConvertBase { await fs.copy(inputFiles.sourceFile, mobiFile); let perc = 0; - await this.execConverter(this.calibrePath, [mobiFile, fb2File], () => { - perc = (perc < 100 ? perc + 5 : 50); + await this.execConverter(this.calibrePath, [mobiFile, fb2File, '-vv'], () => { + perc = (perc < 100 ? perc + 1 : 50); callback(perc); - }); + }, abort); return await fs.readFile(fb2File); } diff --git a/server/core/Reader/BookConverter/ConvertPdf.js b/server/core/Reader/BookConverter/ConvertPdf.js index be885369..397a1868 100644 --- a/server/core/Reader/BookConverter/ConvertPdf.js +++ b/server/core/Reader/BookConverter/ConvertPdf.js @@ -18,7 +18,7 @@ class ConvertPdf extends ConvertHtml { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${utils.randomHexString(10)}.xml`; @@ -27,7 +27,7 @@ class ConvertPdf extends ConvertHtml { await this.execConverter(this.pdfToHtmlPath, ['-c', '-s', '-xml', inputFiles.sourceFile, outFile], () => { perc = (perc < 80 ? perc + 10 : 40); callback(perc); - }); + }, abort); callback(80); const data = await fs.readFile(outFile); diff --git a/server/core/Reader/BookConverter/ConvertRtf.js b/server/core/Reader/BookConverter/ConvertRtf.js index 61f489f7..5f167cd6 100644 --- a/server/core/Reader/BookConverter/ConvertRtf.js +++ b/server/core/Reader/BookConverter/ConvertRtf.js @@ -16,7 +16,7 @@ class ConvertRtf extends ConvertDocX { return false; await this.checkExternalConverterPresent(); - const {inputFiles, callback} = opts; + const {inputFiles, callback, abort} = opts; const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const rtfFile = `${outFile}.rtf`; @@ -24,9 +24,9 @@ class ConvertRtf extends ConvertDocX { const fb2File = `${outFile}.fb2`; await fs.copy(inputFiles.sourceFile, rtfFile); - await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, rtfFile]); + await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, rtfFile], null, abort); - return await super.convert(docxFile, fb2File, callback); + return await super.convert(docxFile, fb2File, callback, abort); } } diff --git a/server/core/Reader/BookConverter/index.js b/server/core/Reader/BookConverter/index.js index 89b73f5a..d91de21e 100644 --- a/server/core/Reader/BookConverter/index.js +++ b/server/core/Reader/BookConverter/index.js @@ -26,11 +26,14 @@ class BookConverter { } } - async convertToFb2(inputFiles, outputFile, opts, callback) { + async convertToFb2(inputFiles, outputFile, opts, callback, abort) { + if (abort && abort()) + throw new Error('abort'); + const selectedFileType = await this.detector.detectFile(inputFiles.selectedFile); const data = await fs.readFile(inputFiles.selectedFile); - const convertOpts = Object.assign({}, opts, {inputFiles, callback, dataType: selectedFileType}); + const convertOpts = Object.assign({}, opts, {inputFiles, callback, abort, dataType: selectedFileType}); let result = false; for (const convert of this.convertFactory) { result = await convert.run(data, convertOpts); @@ -41,7 +44,7 @@ class BookConverter { } if (!result && inputFiles.nesting) { - result = await this.convertToFb2(inputFiles.nesting, outputFile, opts, callback); + result = await this.convertToFb2(inputFiles.nesting, outputFile, opts, callback, abort); } if (!result) { diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 2b8d1180..dafc628a 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -1,14 +1,19 @@ const fs = require('fs-extra'); const path = require('path'); +const LimitedQueue = require('../LimitedQueue'); const WorkerState = require('../WorkerState');//singleton const FileDownloader = require('../FileDownloader'); const FileDecompressor = require('../FileDecompressor'); const BookConverter = require('./BookConverter'); +const RemoteWebDavStorage = require('../RemoteWebDavStorage'); const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton +const cleanDirPeriod = 60*60*1000;//1 раз в час +const queue = new LimitedQueue(5, 100, 5*60*1000);//5 минут ожидание подвижек + let instance = null; //singleton @@ -24,12 +29,19 @@ class ReaderWorker { fs.ensureDirSync(this.config.tempPublicDir); this.workerState = new WorkerState(); - this.down = new FileDownloader(); - this.decomp = new FileDecompressor(); + this.down = new FileDownloader(config.maxUploadFileSize); + this.decomp = new FileDecompressor(2*config.maxUploadFileSize); this.bookConverter = new BookConverter(this.config); - this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час - this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, 60*60*1000);//1 раз в час + this.remoteWebDavStorage = false; + if (config.remoteWebDavStorage) { + this.remoteWebDavStorage = new RemoteWebDavStorage( + Object.assign({maxContentLength: config.maxUploadFileSize}, config.remoteWebDavStorage) + ); + } + + this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, cleanDirPeriod); + this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, cleanDirPeriod); instance = this; } @@ -39,22 +51,39 @@ class ReaderWorker { async loadBook(opts, wState) { const url = opts.url; - let errMes = ''; let decompDir = ''; let downloadedFilename = ''; let isUploaded = false; let convertFilename = ''; + + const overLoadMes = 'Слишком большая очередь загрузки. Пожалуйста, попробуйте позже.'; + const overLoadErr = new Error(overLoadMes); + + let q = null; try { + wState.set({state: 'queue', step: 1, totalSteps: 1}); + try { + let qSize = 0; + q = await queue.get((place) => { + wState.set({place, progress: (qSize ? Math.round((qSize - place)/qSize*100) : 0)}); + if (!qSize) + qSize = place; + }); + } catch (e) { + throw overLoadErr; + } + wState.set({state: 'download', step: 1, totalSteps: 3, url}); const tempFilename = utils.randomHexString(30); const tempFilename2 = utils.randomHexString(30); const decompDirname = utils.randomHexString(30); + //download or use uploaded if (url.indexOf('file://') != 0) {//download const downdata = await this.down.load(url, (progress) => { wState.set({progress}); - }); + }, q.abort); downloadedFilename = `${this.config.tempDownloadDir}/${tempFilename}`; await fs.writeFile(downloadedFilename, downdata); @@ -67,6 +96,10 @@ class ReaderWorker { } wState.set({progress: 100}); + if (q.abort()) + throw overLoadErr; + q.resetTimeout(); + //decompress wState.set({state: 'decompress', step: 2, progress: 0}); decompDir = `${this.config.tempDownloadDir}/${decompDirname}`; @@ -79,27 +112,49 @@ class ReaderWorker { } wState.set({progress: 100}); + if (q.abort()) + throw overLoadErr; + q.resetTimeout(); + //конвертирование в fb2 wState.set({state: 'convert', step: 3, progress: 0}); convertFilename = `${this.config.tempDownloadDir}/${tempFilename2}`; await this.bookConverter.convertToFb2(decompFiles, convertFilename, opts, progress => { wState.set({progress}); - }); + }, q.abort); //сжимаем файл в tmp, если там уже нет с тем же именем-sha256 - const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, `${this.config.tempPublicDir}`); + const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, this.config.tempPublicDir); + const stat = await fs.stat(compFilename); wState.set({progress: 100}); //finish const finishFilename = path.basename(compFilename); - wState.finish({path: `/tmp/${finishFilename}`}); + wState.finish({path: `/tmp/${finishFilename}`, size: stat.size}); + + //лениво сохраним compFilename в удаленном хранилище + if (this.remoteWebDavStorage) { + (async() => { + await utils.sleep(20*1000); + try { + //log(`remoteWebDavStorage.putFile ${path.basename(compFilename)}`); + await this.remoteWebDavStorage.putFile(compFilename); + } catch (e) { + log(LM_ERR, e.stack); + } + })(); + } } catch (e) { log(LM_ERR, e.stack); - wState.set({state: 'error', error: (errMes ? errMes : e.message)}); + if (e.message == 'abort') + e.message = overLoadMes; + wState.set({state: 'error', error: e.message}); } finally { //clean + if (q) + q.ret(); if (decompDir) await fs.remove(decompDir); if (downloadedFilename && !isUploaded) @@ -133,6 +188,41 @@ class ReaderWorker { return `file://${hash}`; } + restoreCachedFile(filename) { + const workerId = this.workerState.generateWorkerId(); + const wState = this.workerState.getControl(workerId); + wState.set({state: 'start'}); + + (async() => { + try { + wState.set({state: 'download', step: 1, totalSteps: 1, path: filename, progress: 0}); + + const basename = path.basename(filename); + const targetName = `${this.config.tempPublicDir}/${basename}`; + + if (!await fs.pathExists(targetName)) { + let found = false; + if (this.remoteWebDavStorage) { + found = await this.remoteWebDavStorage.getFileSuccess(targetName); + } + + if (!found) { + throw new Error('404 Файл не найден'); + } + } + + const stat = await fs.stat(targetName); + wState.finish({path: `/tmp/${basename}`, size: stat.size, progress: 100}); + } catch (e) { + if (e.message.indexOf('404') < 0) + log(LM_ERR, e.stack); + wState.set({state: 'error', error: e.message}); + } + })(); + + return workerId; + } + async periodicCleanDir(dir, maxSize, timeout) { try { const list = await fs.readdir(dir); @@ -153,7 +243,19 @@ class ReaderWorker { let i = 0; while (i < files.length && size > maxSize) { const file = files[i]; - await fs.remove(`${dir}/${file.name}`); + const oldFile = `${dir}/${file.name}`; + + //отправляем только this.config.tempPublicDir + //TODO: убрать в будущем, т.к. уже делается ленивое сохранение compFilename в удаленном хранилище + if (this.remoteWebDavStorage && dir === this.config.tempPublicDir) { + try { + //log(`remoteWebDavStorage.putFile ${path.basename(oldFile)}`); + await this.remoteWebDavStorage.putFile(oldFile); + } catch (e) { + log(LM_ERR, e.stack); + } + } + await fs.remove(oldFile); size -= file.stat.size; i++; } diff --git a/server/core/RemoteWebDavStorage.js b/server/core/RemoteWebDavStorage.js new file mode 100644 index 00000000..21d0c107 --- /dev/null +++ b/server/core/RemoteWebDavStorage.js @@ -0,0 +1,107 @@ +const fs = require('fs-extra'); +const path = require('path'); + +const { createClient } = require('webdav'); + +class RemoteWebDavStorage { + constructor(config) { + this.config = Object.assign({}, config); + this.config.maxContentLength = this.config.maxContentLength || 10*1024*1024; + this.wdc = createClient(config.url, this.config); + } + + _convertStat(data) { + return { + isDirectory: function() { + return data.type === "directory"; + }, + isFile: function() { + return data.type === "file"; + }, + mtime: (new Date(data.lastmod)).getTime(), + name: data.basename, + size: data.size || 0 + }; + } + + async stat(filename) { + const stat = await this.wdc.stat(filename); + return this._convertStat(stat); + } + + async writeFile(filename, data) { + return await this.wdc.putFileContents(filename, data, { maxContentLength: this.config.maxContentLength }) + } + + async unlink(filename) { + return await this.wdc.deleteFile(filename); + } + + async readFile(filename) { + return await this.wdc.getFileContents(filename, { maxContentLength: this.config.maxContentLength }) + } + + async mkdir(dirname) { + return await this.wdc.createDirectory(dirname); + } + + async putFile(filename) { + if (!await fs.pathExists(filename)) { + throw new Error(`File not found: ${filename}`); + } + + const base = path.basename(filename); + let remoteFilename = `/${base}`; + + if (base.length > 3) { + const remoteDir = `/${base.substr(0, 3)}`; + try { + await this.mkdir(remoteDir); + } catch (e) { + // + } + remoteFilename = `${remoteDir}/${base}`; + } + + try { + const localStat = await fs.stat(filename); + const remoteStat = await this.stat(remoteFilename); + if (remoteStat.isFile && localStat.size == remoteStat.size) { + return; + } + await this.unlink(remoteFilename); + } catch (e) { + // + } + + const data = await fs.readFile(filename); + await this.writeFile(remoteFilename, data); + } + + async getFile(filename) { + if (await fs.pathExists(filename)) { + return; + } + + const base = path.basename(filename); + let remoteFilename = `/${base}`; + if (base.length > 3) { + remoteFilename = `/${base.substr(0, 3)}/${base}`; + } + + const data = await this.readFile(remoteFilename); + await fs.writeFile(filename, data); + } + + async getFileSuccess(filename) { + try { + await this.getFile(filename); + return true; + } catch (e) { + // + } + return false; + } +} + +module.exports = RemoteWebDavStorage; \ No newline at end of file diff --git a/server/core/ZipStreamer.js b/server/core/ZipStreamer.js index 03f59ff5..c761eaa9 100644 --- a/server/core/ZipStreamer.js +++ b/server/core/ZipStreamer.js @@ -52,7 +52,7 @@ class ZipStreamer { })().catch(reject); }); } - unpack(zipFile, outputDir, entryCallback) { + unpack(zipFile, outputDir, entryCallback, limitFileSize = 0) { return new Promise((resolve, reject) => { entryCallback = (entryCallback ? entryCallback : () => {}); const unzip = new unzipStream({file: zipFile}); @@ -67,6 +67,15 @@ class ZipStreamer { }); unzip.on('ready', () => { + if (limitFileSize) { + for (const entry of Object.values(unzip.entries())) { + if (!entry.isDirectory && entry.size > limitFileSize) { + reject('Файл слишком большой'); + return; + } + } + } + unzip.extract(null, outputDir, (err) => { if (err) reject(err); unzip.close(); diff --git a/server/core/utils.js b/server/core/utils.js index 9591c7d7..53f7a2b3 100644 --- a/server/core/utils.js +++ b/server/core/utils.js @@ -37,8 +37,8 @@ async function touchFile(filename) { } function spawnProcess(cmd, opts) { - let {args, killAfter, onData} = opts; - killAfter = (killAfter ? killAfter : 120*1000); + let {args, killAfter, onData, abort} = opts; + killAfter = (killAfter ? killAfter : 120);//seconds onData = (onData ? onData : () => {}); args = (args ? args : []); @@ -67,10 +67,18 @@ function spawnProcess(cmd, opts) { reject({status: 'error', error, stdout, stderr}); }); - await sleep(killAfter); - if (!resolved) { - process.kill(proc.pid); - reject({status: 'killed', stdout, stderr}); + while (!resolved) { + await sleep(1000); + killAfter -= 1; + if (killAfter <= 0 || (abort && abort())) { + process.kill(proc.pid); + if (killAfter <= 0) { + reject({status: 'killed', stdout, stderr}); + } else { + reject({status: 'abort', stdout, stderr}); + } + break; + } } })().catch(reject); }); } diff --git a/server/db/SqliteConnectionPool.js b/server/db/SqliteConnectionPool.js index a1acb919..a5189a9b 100644 --- a/server/db/SqliteConnectionPool.js +++ b/server/db/SqliteConnectionPool.js @@ -14,7 +14,6 @@ class SqliteConnectionPool { if (!Number.isInteger(connCount) || connCount <= 0) return; this.connections = []; - this.taken = new Set(); this.freed = new Set(); for (let i = 0; i < connCount; i++) { @@ -22,7 +21,6 @@ class SqliteConnectionPool { client.configure('busyTimeout', 10000); //ms client.ret = () => { - this.taken.delete(i); this.freed.add(i); }; @@ -52,7 +50,6 @@ class SqliteConnectionPool { } this.freed.delete(freeConnIndex); - this.taken.add(freeConnIndex); return this.connections[freeConnIndex]; } diff --git a/server/index.js b/server/index.js index 94bf47d9..9a15a6b2 100644 --- a/server/index.js +++ b/server/index.js @@ -4,6 +4,8 @@ const path = require('path'); const argv = require('minimist')(process.argv.slice(2)); const express = require('express'); const compression = require('compression'); +const http = require('http'); +const WebSocket = require ('ws'); async function init() { //config @@ -46,10 +48,13 @@ async function main() { const config = new (require('./config'))().config;//singleton //servers - for (let server of config.servers) { - if (server.mode !== 'none') { + for (let serverCfg of config.servers) { + if (serverCfg.mode !== 'none') { const app = express(); - const serverConfig = Object.assign({}, config, server); + const server = http.createServer(app); + const wss = new WebSocket.Server({ server, maxPayload: 10*1024*1024 }); + + const serverConfig = Object.assign({}, config, serverCfg); let devModule = undefined; if (serverConfig.branch == 'development') { @@ -73,7 +78,7 @@ async function main() { } })); - require('./routes').initRoutes(app, serverConfig); + require('./routes').initRoutes(app, wss, serverConfig); if (devModule) { devModule.logErrors(app); @@ -84,7 +89,7 @@ async function main() { }); } - app.listen(serverConfig.port, serverConfig.ip, function() { + server.listen(serverConfig.port, serverConfig.ip, function() { log(`Server-${serverConfig.serverName} is ready on ${serverConfig.ip}:${serverConfig.port}, mode: ${serverConfig.mode}`); }); } diff --git a/server/routes.js b/server/routes.js index ffdd92c8..93eb129a 100644 --- a/server/routes.js +++ b/server/routes.js @@ -2,10 +2,11 @@ const c = require('./controllers'); const utils = require('./core/utils'); const multer = require('multer'); -function initRoutes(app, config) { +function initRoutes(app, wss, config) { const misc = new c.MiscController(config); const reader = new c.ReaderController(config); const worker = new c.WorkerController(config); + new c.WebSocketController(wss, config); //access const [aAll, aNormal, aSite, aReader, aOmnireader] = // eslint-disable-line no-unused-vars @@ -28,7 +29,9 @@ function initRoutes(app, config) { ['POST', '/api/reader/load-book', reader.loadBook.bind(reader), [aAll], {}], ['POST', '/api/reader/storage', reader.storage.bind(reader), [aAll], {}], ['POST', '/api/reader/upload-file', [upload.single('file'), reader.uploadFile.bind(reader)], [aAll], {}], + ['POST', '/api/reader/restore-cached-file', reader.restoreCachedFile.bind(reader), [aAll], {}], ['POST', '/api/worker/get-state', worker.getState.bind(worker), [aAll], {}], + ['POST', '/api/worker/get-state-finish', worker.getStateFinish.bind(worker), [aAll], {}], ]; //to app