Compare commits

...

30 Commits

Author SHA1 Message Date
Book Pauk
eedca4db9b Merge branch 'release/0.8.2-5' 2020-01-23 17:12:37 +07:00
Book Pauk
1d352a76ce Поправка опечаток 2020-01-23 17:00:17 +07:00
Book Pauk
17670aabf9 WebSocket: добавлен метод reader-storage, поправки багов 2020-01-23 16:59:08 +07:00
Book Pauk
3456b3d90e WebSocket: добавлен метод worker-get-state-finish, небольшой рефакторинг 2020-01-23 16:25:06 +07:00
Book Pauk
f3da5a9026 Поправил комментарий 2020-01-23 15:56:26 +07:00
Book Pauk
00cc63b7cd WebSocket: добавлен метод get-config 2020-01-23 15:54:46 +07:00
Book Pauk
8df80ce738 Мелкая поправка 2020-01-23 15:16:49 +07:00
Book Pauk
12e7a783b0 Небольшие изменения блокирования кнопок панели 2020-01-22 22:06:12 +07:00
Book Pauk
be86a15351 Добавил настройку proxy_read_timeout 2020-01-22 21:37:28 +07:00
Book Pauk
2c5022e7b4 Merge tag '0.8.2-4' into develop
0.8.2-4
2020-01-22 21:17:58 +07:00
Book Pauk
f4a996fcb9 Merge branch 'release/0.8.2-4' 2020-01-22 21:17:52 +07:00
Book Pauk
fdbf508bbf Используем протокол WSS при необходимости 2020-01-22 21:17:10 +07:00
Book Pauk
500fafa5b2 Merge tag '0.8.2-3' into develop
0.8.2-3
2020-01-22 21:05:36 +07:00
Book Pauk
bfa315c68b Merge branch 'release/0.8.2-3' 2020-01-22 21:05:27 +07:00
Book Pauk
4972f085a3 Мелкая поправка 2020-01-22 20:59:52 +07:00
Book Pauk
9c13261929 Добавлена настройка для вебсокетов, добавлен конфиг nginx omnireader_http 2020-01-22 20:58:57 +07:00
Book Pauk
e36dc4a913 Небольшие поправки 2020-01-22 20:28:46 +07:00
Book Pauk
4cccb56ee3 Поправил комментарий 2020-01-22 20:15:33 +07:00
Book Pauk
3199af570d Добавлен WebSocketServer и контроллер для него 2020-01-22 20:06:51 +07:00
Book Pauk
7dad47b3c8 Добавлено использование WebSocketConnection 2020-01-22 20:02:42 +07:00
Book Pauk
fbd50bad1d Исправления багов 2020-01-22 20:02:05 +07:00
Book Pauk
10469bae7b Мелкая поправка 2020-01-22 20:01:21 +07:00
Book Pauk
b6a000a001 Добавлен пакет ws 2020-01-22 20:00:52 +07:00
Book Pauk
59539e7e90 Добавлен класс WebSocketConnection 2020-01-22 19:32:11 +07:00
Book Pauk
a2c41bc5ec Merge tag '0.8.2-2' into develop
0.8.2-2
2020-01-21 16:56:20 +07:00
Book Pauk
c4a06858fb Merge branch 'release/0.8.2-2' 2020-01-21 16:56:12 +07:00
Book Pauk
15b0f05a05 Добавил комментарий 2020-01-21 16:55:41 +07:00
Book Pauk
67feee9aa1 Поправлен баг 2020-01-21 16:53:34 +07:00
Book Pauk
185fb57b8c Удален нерабочий код 2020-01-21 16:25:30 +07:00
Book Pauk
e9039f8208 Merge tag '0.8.2-1' into develop
0.8.2-1
2020-01-21 16:14:21 +07:00
17 changed files with 543 additions and 105 deletions

View File

@@ -1,4 +1,5 @@
import axios from 'axios';
import wsc from './webSocketConnection';
const api = axios.create({
baseURL: '/api'
@@ -6,9 +7,20 @@ const api = axios.create({
class Misc {
async loadConfig() {
const response = await api.post('/config', {params: [
const query = {params: [
'name', 'version', 'mode', 'maxUploadFileSize', 'useExternalBookConverter', 'branch',
]});
]};
try {
await wsc.open();
return await wsc.message(wsc.send(Object.assign({action: 'get-config'}, query)));
} catch (e) {
console.error(e);
}
//если с WebSocket проблема, работаем по http
const response = await api.post('/config', query);
return response.data;
}
}

View File

@@ -1,4 +1,6 @@
import axios from 'axios';
import * as utils from '../share/utils';
import wsc from './webSocketConnection';
const api = axios.create({
baseURL: '/api/reader'
@@ -9,45 +11,56 @@ const workerApi = axios.create({
});
class Reader {
constructor() {
}
async getStateFinish(workerId, callback) {
async getWorkerStateFinish(workerId, callback) {
if (!callback) callback = () => {};
//присылается текст, состоящий из json-объектов state каждые 300ms, с разделителем splitter между ними
const splitter = '-- aod2t5hDXU32bUFyqlFE next status --';
let lastIndex = 0;
let response = await workerApi.post('/get-state-finish', {workerId}, {
onDownloadProgress: progress => {
//небольая оптимизация, вместо простого responseText.split
const xhr = progress.target;
let currIndex = xhr.responseText.length;
if (lastIndex == currIndex)
return;
const last = xhr.responseText.substring(lastIndex, currIndex);
lastIndex = currIndex;
let response = {};
//быстрее будет last.split
const res = last.split(splitter).pop();
if (res) {
try {
callback(JSON.parse(res));
} catch (e) {
//
}
}
}
});
await wsc.open();
const requestId = wsc.send({action: 'worker-get-state-finish', workerId});
//берем последний state
response = response.data.split(splitter).pop();
while (1) {// eslint-disable-line no-constant-condition
response = await wsc.message(requestId);
callback(response);
if (response) {
try {
response = JSON.parse(response);
} catch (e) {
response = false;
if (response.state == 'finish' || response.state == 'error') {
break;
}
}
return response;
} catch (e) {
console.error(e);
}
//если с WebSocket проблема, работаем по http
const refreshPause = 500;
let i = 0;
response = {};
while (1) {// eslint-disable-line no-constant-condition
const prevProgress = response.progress || 0;
const prevState = response.state || 0;
response = await workerApi.post('/get-state', {workerId});
response = response.data;
callback(response);
if (response.state == 'finish' || response.state == 'error') {
break;
}
if (i > 0)
await utils.sleep(refreshPause);
i++;
if (i > 120*1000/refreshPause) {//2 мин ждем телодвижений воркера
throw new Error('Слишком долгое время ожидания');
}
//проверка воркера
i = (prevProgress != response.progress || prevState != response.state ? 1 : i);
}
return response;
}
@@ -64,12 +77,12 @@ class Reader {
callback({totalSteps: 4});
callback(response.data);
response = await this.getStateFinish(workerId, callback);
response = await this.getWorkerStateFinish(workerId, callback);
if (response) {
if (response.state == 'finish') {//воркер закончил работу, можно скачивать кешированный на сервере файл
callback({step: 4});
const book = await this.loadCachedBook(response.path, callback, false, (response.size ? response.size : -1));
const book = await this.loadCachedBook(response.path, callback, response.size);
return Object.assign({}, response, {data: book.data});
}
@@ -87,75 +100,58 @@ class Reader {
}
}
async checkUrl(url) {
let fileExists = false;
async checkCachedBook(url) {
let estSize = -1;
try {
await axios.head(url, {headers: {'Cache-Control': 'no-cache'}});
fileExists = true;
} catch (e) {
//
}
//восстановим при необходимости файл на сервере из удаленного облака
if (!fileExists) {
let response = await api.post('/restore-cached-file', {path: url});
const workerId = response.data.workerId;
if (!workerId)
throw new Error('Неверный ответ api');
response = await this.getStateFinish(workerId);
if (response.state == 'error') {
throw new Error(response.error);
}
}
return true;
}
async loadCachedBook(url, callback, restore = true, estSize = -1) {
if (!callback) callback = () => {};
let response = null;
callback({state: 'loading', progress: 0});
//получение размера файла
let fileExists = false;
if (estSize < 0) {
try {
response = await axios.head(url, {headers: {'Cache-Control': 'no-cache'}});
const response = await axios.head(url, {headers: {'Cache-Control': 'no-cache'}});
if (response.headers['content-length']) {
estSize = response.headers['content-length'];
}
fileExists = true;
} catch (e) {
//
}
}
//восстановим при необходимости файл на сервере из удаленного облака
if (restore && !fileExists) {
response = await api.post('/restore-cached-file', {path: url});
let response = null
const workerId = response.data.workerId;
try {
await wsc.open();
response = await wsc.message(wsc.send({action: 'reader-restore-cached-file', path: url}));
} catch (e) {
console.error(e);
//если с WebSocket проблема, работаем по http
response = await api.post('/restore-cached-file', {path: url});
response = response.data;
}
const workerId = response.workerId;
if (!workerId)
throw new Error('Неверный ответ api');
response = await this.getStateFinish(workerId);
response = await this.getWorkerStateFinish(workerId);
if (response.state == 'error') {
throw new Error(response.error);
}
if (response.size && estSize < 0) {
estSize = response.size;
}
}
return estSize;
}
async loadCachedBook(url, callback, estSize = -1) {
if (!callback) callback = () => {};
callback({state: 'loading', progress: 0});
//получение размера файла
if (estSize && estSize < 0) {
estSize = await this.checkCachedBook(url);
}
//получение файла
estSize = (estSize > 0 ? estSize : 1000000);
const options = {
onDownloadProgress: progress => {
onDownloadProgress: (progress) => {
while (progress.loaded > estSize) estSize *= 1.5;
if (callback)
@@ -199,13 +195,22 @@ class Reader {
}
async storage(request) {
let response = await api.post('/storage', request);
let response = null;
try {
await wsc.open();
response = await wsc.message(wsc.send({action: 'reader-storage', body: request}));
} catch (e) {
console.error(e);
//если с WebSocket проблема, работаем по http
response = await api.post('/storage', request);
response = response.data;
}
const state = response.data.state;
const state = response.state;
if (!state)
throw new Error('Неверный ответ api');
return response.data;
return response;
}
}

View File

@@ -0,0 +1,176 @@
const cleanPeriod = 60*1000;//1 минута
class WebSocketConnection {
//messageLifeTime в минутах (cleanPeriod)
constructor(messageLifeTime = 5) {
this.ws = null;
this.timer = null;
this.listeners = [];
this.messageQueue = [];
this.messageLifeTime = messageLifeTime;
this.requestId = 0;
}
addListener(listener) {
if (this.listeners.indexOf(listener) < 0)
this.listeners.push(Object.assign({regTime: Date.now()}, listener));
}
//рассылаем сообщение и удаляем те обработчики, которые его получили
emit(mes, isError) {
const len = this.listeners.length;
if (len > 0) {
let newListeners = [];
for (const listener of this.listeners) {
let emitted = false;
if (isError) {
if (listener.onError)
listener.onError(mes);
emitted = true;
} else {
if (listener.onMessage) {
if (listener.requestId) {
if (listener.requestId === mes.requestId) {
listener.onMessage(mes);
emitted = true;
}
} else {
listener.onMessage(mes);
emitted = true;
}
} else {
emitted = true;
}
}
if (!emitted)
newListeners.push(listener);
}
this.listeners = newListeners;
}
return this.listeners.length != len;
}
open(url) {
return new Promise((resolve, reject) => {
if (this.ws && this.ws.readyState == WebSocket.OPEN) {
resolve(this.ws);
} else {
let protocol = 'ws:';
if (window.location.protocol == 'https:') {
protocol = 'wss:'
}
url = url || `${protocol}//${window.location.host}/ws`;
this.ws = new WebSocket(url);
if (this.timer) {
clearTimeout(this.timer);
}
this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod);
let resolved = false;
this.ws.onopen = (e) => {
resolved = true;
resolve(e);
};
this.ws.onmessage = (e) => {
try {
const mes = JSON.parse(e.data);
this.messageQueue.push({regTime: Date.now(), mes});
let newMessageQueue = [];
for (const message of this.messageQueue) {
if (!this.emit(message.mes)) {
newMessageQueue.push(message);
}
}
this.messageQueue = newMessageQueue;
} catch (e) {
this.emit(e.message, true);
}
};
this.ws.onerror = (e) => {
this.emit(e.message, true);
if (!resolved)
reject(e);
};
}
});
}
//timeout в минутах (cleanPeriod)
message(requestId, timeout = 2) {
return new Promise((resolve, reject) => {
this.addListener({
requestId,
timeout,
onMessage: (mes) => {
if (mes.error) {
reject(mes.error);
} else {
resolve(mes);
}
},
onError: (e) => {
reject(e);
}
});
});
}
send(req) {
if (this.ws && this.ws.readyState == WebSocket.OPEN) {
const requestId = ++this.requestId;
this.ws.send(JSON.stringify(Object.assign({requestId}, req)));
return requestId;
} else {
throw new Error('WebSocket connection is not ready');
}
}
close() {
if (this.ws && this.ws.readyState == WebSocket.OPEN) {
this.ws.close();
}
}
periodicClean() {
try {
this.timer = null;
const now = Date.now();
//чистка listeners
let newListeners = [];
for (const listener of this.listeners) {
if (now - listener.regTime < listener.timeout*cleanPeriod - 50) {
newListeners.push(listener);
} else {
if (listener.onError)
listener.onError('Время ожидания ответа истекло');
}
}
this.listeners = newListeners;
//чистка messageQueue
let newMessageQueue = [];
for (const message of this.messageQueue) {
if (now - message.regTime < this.messageLifeTime*cleanPeriod - 50) {
newMessageQueue.push(message);
}
}
this.messageQueue = newMessageQueue;
} finally {
if (this.ws.readyState == WebSocket.OPEN) {
this.timer = setTimeout(() => { this.periodicClean(); }, cleanPeriod);
}
}
}
}
export default new WebSocketConnection();

View File

@@ -719,15 +719,16 @@ class Reader extends Vue {
case 'scrolling':
case 'search':
case 'copyText':
case 'recentBooks':
case 'refresh':
case 'offlineMode':
case 'recentBooks':
case 'settings':
if (this[`${button}Active`])
if (this.progressActive) {
classResult = classDisabled;
} else if (this[`${button}Active`]) {
classResult = classActive;
break;
}
switch (button) {
break;
case 'undoAction':
if (this.actionCur <= 0)
classResult = classDisabled;

View File

@@ -272,7 +272,7 @@ class RecentBooksPage extends Vue {
async downloadBook(fb2path) {
try {
await readerApi.checkUrl(fb2path);
await readerApi.checkCachedBook(fb2path);
const d = this.$refs.download;
d.href = fb2path;

View File

@@ -8,6 +8,7 @@ server {
server_name omnireader.ru;
client_max_body_size 50m;
proxy_read_timeout 1h;
gzip on;
gzip_min_length 1024;
@@ -18,6 +19,13 @@ server {
proxy_pass http://127.0.0.1:44081;
}
location /ws {
proxy_pass http://127.0.0.1:44081;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
location / {
root /home/liberama/public;

View File

@@ -0,0 +1,59 @@
server {
listen 80;
server_name omnireader.ru;
client_max_body_size 50m;
proxy_read_timeout 1h;
gzip on;
gzip_min_length 1024;
gzip_proxied expired no-cache no-store private auth;
gzip_types *;
location /api {
proxy_pass http://127.0.0.1:44081;
}
location /ws {
proxy_pass http://127.0.0.1:44081;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
location / {
root /home/liberama/public;
location /tmp {
add_header Content-Type text/xml;
add_header Content-Encoding gzip;
}
location ~* \.(?:manifest|appcache|html)$ {
expires -1;
}
}
}
server {
listen 80;
server_name old.omnireader.ru;
client_max_body_size 50m;
gzip on;
gzip_min_length 1024;
gzip_proxied expired no-cache no-store private auth;
gzip_types *;
root /home/oldreader;
index index.html;
# Обработка php файлов с помощью fpm
location ~ \.php$ {
try_files $uri =404;
include /etc/nginx/fastcgi.conf;
fastcgi_pass unix:/run/php/php7.2-fpm.sock;
}
}

5
package-lock.json generated
View File

@@ -13075,6 +13075,11 @@
"mkdirp": "^0.5.1"
}
},
"ws": {
"version": "7.2.1",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz",
"integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A=="
},
"xml2js": {
"version": "0.4.23",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",

View File

@@ -85,6 +85,7 @@
"vuex": "^3.1.1",
"vuex-persistedstate": "^2.5.4",
"webdav": "^2.10.1",
"ws": "^7.2.1",
"zip-stream": "^2.1.2"
}
}

View File

@@ -0,0 +1,164 @@
const WebSocket = require ('ws');
const _ = require('lodash');
const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton
const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton
const WorkerState = require('../core/WorkerState');//singleton
const log = new (require('../core/AppLogger'))().log;//singleton
const utils = require('../core/utils');
const cleanPeriod = 1*60*1000;//1 минута
const closeSocketOnIdle = 5*60*1000;//5 минут
class WebSocketController {
constructor(wss, config) {
this.config = config;
this.isDevelopment = (config.branch == 'development');
this.readerStorage = new ReaderStorage();
this.readerWorker = new ReaderWorker(config);
this.workerState = new WorkerState();
this.wss = wss;
wss.on('connection', (ws) => {
ws.on('message', (message) => {
this.onMessage(ws, message);
});
});
setTimeout(() => { this.periodicClean(); }, cleanPeriod);
}
periodicClean() {
try {
const now = Date.now();
this.wss.clients.forEach((ws) => {
if (!ws.lastActivity || now - ws.lastActivity > closeSocketOnIdle - 50) {
ws.terminate();
}
});
} finally {
setTimeout(() => { this.periodicClean(); }, cleanPeriod);
}
}
async onMessage(ws, message) {
let req = {};
try {
if (this.isDevelopment) {
log(`WebSocket-IN: ${message.substr(0, 4000)}`);
}
ws.lastActivity = Date.now();
req = JSON.parse(message);
switch (req.action) {
case 'test':
await this.test(req, ws); break;
case 'get-config':
await this.getConfig(req, ws); break;
case 'worker-get-state':
await this.workerGetState(req, ws); break;
case 'worker-get-state-finish':
await this.workerGetStateFinish(req, ws); break;
case 'reader-restore-cached-file':
await this.readerRestoreCachedFile(req, ws); break;
case 'reader-storage':
await this.readerStorageDo(req, ws); break;
default:
throw new Error(`Action not found: ${req.action}`);
}
} catch (e) {
this.send({error: e.message}, req, ws);
}
}
send(res, req, ws) {
if (ws.readyState == WebSocket.OPEN) {
ws.lastActivity = Date.now();
let r = res;
if (req.requestId)
r = Object.assign({requestId: req.requestId}, r);
const message = JSON.stringify(r);
ws.send(message);
if (this.isDevelopment) {
log(`WebSocket-OUT: ${message.substr(0, 4000)}`);
}
}
}
//Actions ------------------------------------------------------------------
async test(req, ws) {
this.send({message: 'Liberama project is awesome'}, req, ws);
}
async getConfig(req, ws) {
if (Array.isArray(req.params)) {
this.send(_.pick(this.config, req.params), req, ws);
} else {
throw new Error('params is not an array');
}
}
async workerGetState(req, ws) {
if (!req.workerId)
throw new Error(`key 'workerId' is wrong`);
const state = this.workerState.getState(req.workerId);
this.send((state ? state : {}), req, ws);
}
async workerGetStateFinish(req, ws) {
if (!req.workerId)
throw new Error(`key 'workerId' is wrong`);
const refreshPause = 200;
let i = 0;
let state = {};
while (1) {// eslint-disable-line no-constant-condition
const prevProgress = state.progress || -1;
const prevState = state.state || '';
state = this.workerState.getState(req.workerId);
this.send((state ? state : {}), req, ws);
if (!state) break;
if (state.state != 'finish' && state.state != 'error')
await utils.sleep(refreshPause);
else
break;
i++;
if (i > 2*60*1000/refreshPause) {//2 мин ждем телодвижений воркера
this.send({state: 'error', error: 'Время ожидания процесса истекло'}, req, ws);
}
i = (prevProgress != state.progress || prevState != state.state ? 1 : i);
}
}
async readerRestoreCachedFile(req, ws) {
if (!req.path)
throw new Error(`key 'path' is empty`);
const workerId = this.readerWorker.restoreCachedFile(req.path);
const state = this.workerState.getState(workerId);
this.send((state ? state : {}), req, ws);
}
async readerStorageDo(req, ws) {
if (!req.body)
throw new Error(`key 'body' is empty`);
if (!req.body.action)
throw new Error(`key 'action' is empty`);
if (!req.body.items || Array.isArray(req.body.data))
throw new Error(`key 'items' is empty`);
this.send(await this.readerStorage.doAction(req.body), req, ws);
}
}
module.exports = WebSocketController;

View File

@@ -26,6 +26,7 @@ class WorkerController extends BaseController {
return false;
}
//TODO: удалить бесполезную getStateFinish
async getStateFinish(req, res) {
const request = req.body;
let error = '';
@@ -47,10 +48,6 @@ class WorkerController extends BaseController {
state = this.workerState.getState(request.workerId);
if (!state) break;
if (i == 0) {
state = Object.assign({dummy: '0'.repeat(1024)}, state);
}
res.write(splitter + JSON.stringify(state));
res.flush();

View File

@@ -2,4 +2,5 @@ module.exports = {
MiscController: require('./MiscController'),
ReaderController: require('./ReaderController'),
WorkerController: require('./WorkerController'),
WebSocketController: require('./WebSocketController'),
}

View File

@@ -212,7 +212,10 @@ class ReaderWorker {
while (i < files.length && size > maxSize) {
const file = files[i];
const oldFile = `${dir}/${file.name}`;
if (this.remoteWebDavStorage) {
//отправляем только this.config.tempPublicDir
//TODO: убрать в будущем, т.к. уже делается ленивое сохранение compFilename в удаленном хранилище
if (this.remoteWebDavStorage && dir === this.config.tempPublicDir) {
try {
//log(`remoteWebDavStorage.putFile ${path.basename(oldFile)}`);
await this.remoteWebDavStorage.putFile(oldFile);

View File

@@ -4,6 +4,8 @@ const path = require('path');
const argv = require('minimist')(process.argv.slice(2));
const express = require('express');
const compression = require('compression');
const http = require('http');
const WebSocket = require ('ws');
async function init() {
//config
@@ -46,10 +48,13 @@ async function main() {
const config = new (require('./config'))().config;//singleton
//servers
for (let server of config.servers) {
if (server.mode !== 'none') {
for (let serverCfg of config.servers) {
if (serverCfg.mode !== 'none') {
const app = express();
const serverConfig = Object.assign({}, config, server);
const server = http.createServer(app);
const wss = new WebSocket.Server({ server, maxPayload: 10*1024*1024 });
const serverConfig = Object.assign({}, config, serverCfg);
let devModule = undefined;
if (serverConfig.branch == 'development') {
@@ -73,7 +78,7 @@ async function main() {
}
}));
require('./routes').initRoutes(app, serverConfig);
require('./routes').initRoutes(app, wss, serverConfig);
if (devModule) {
devModule.logErrors(app);
@@ -84,7 +89,7 @@ async function main() {
});
}
app.listen(serverConfig.port, serverConfig.ip, function() {
server.listen(serverConfig.port, serverConfig.ip, function() {
log(`Server-${serverConfig.serverName} is ready on ${serverConfig.ip}:${serverConfig.port}, mode: ${serverConfig.mode}`);
});
}

View File

@@ -2,10 +2,11 @@ const c = require('./controllers');
const utils = require('./core/utils');
const multer = require('multer');
function initRoutes(app, config) {
function initRoutes(app, wss, config) {
const misc = new c.MiscController(config);
const reader = new c.ReaderController(config);
const worker = new c.WorkerController(config);
new c.WebSocketController(wss, config);
//access
const [aAll, aNormal, aSite, aReader, aOmnireader] = // eslint-disable-line no-unused-vars