Compare commits

...

15 Commits

Author SHA1 Message Date
Book Pauk
4ae7338f94 Merge branch 'release/0.7.3b' 2019-10-19 19:28:36 +07:00
Book Pauk
0d1e51cb21 Поправка бага 2019-10-19 19:27:18 +07:00
Book Pauk
475fb833ea Merge tag '0.7.3' into develop
0.7.3
2019-10-18 19:30:51 +07:00
Book Pauk
580b030ee4 Merge branch 'release/0.7.3' 2019-10-18 19:30:42 +07:00
Book Pauk
6a7cbc70d6 0.7.3 2019-10-18 19:30:21 +07:00
Book Pauk
d76f60639c Добавлен параметр "Включить html-фильтр для сайтов" в раздел "Вид"->"Текст" в настройках 2019-10-18 19:13:17 +07:00
Book Pauk
e2bea407ee 0.7.3 2019-10-18 17:12:30 +07:00
Book Pauk
558fed31aa Добавлен фильтр для некоторых сайтов 2019-10-18 17:07:28 +07:00
Book Pauk
f6513d40c8 Небольшая поправка 2019-10-18 15:32:05 +07:00
Book Pauk
259f9baa59 Merge branch 'feature/ss_fix2' into develop 2019-10-18 15:25:57 +07:00
Book Pauk
db5650e276 Мелкая поправка 2019-10-18 15:20:32 +07:00
Book Pauk
51ebbbc569 Закончена очередная переработка loadRecent & saveRecent 2019-10-18 15:19:53 +07:00
Book Pauk
5184661652 Мелкая поправка 2019-10-18 14:00:52 +07:00
Book Pauk
7853a14ce6 Начало очередной переделки ServerStorage 2019-10-18 13:53:44 +07:00
Book Pauk
a01e78ace9 Merge tag '0.7.2' into develop
0.7.2
2019-10-18 09:17:58 +07:00
14 changed files with 267 additions and 172 deletions

View File

@@ -11,11 +11,11 @@ const workerApi = axios.create({
});
class Reader {
async loadBook(url, callback) {
async loadBook(opts, callback) {
const refreshPause = 300;
if (!callback) callback = () => {};
let response = await api.post('/load-book', {type: 'url', url});
let response = await api.post('/load-book', opts);
const workerId = response.data.workerId;
if (!workerId)

View File

@@ -356,6 +356,7 @@ class Reader extends Vue {
this.showWhatsNewDialog = settings.showWhatsNewDialog;
this.showMigrationDialog = settings.showMigrationDialog;
this.showToolButton = settings.showToolButton;
this.enableSitesFilter = settings.enableSitesFilter;
this.updateHeaderMinWidth();
}
@@ -1003,7 +1004,7 @@ class Reader extends Vue {
// не удалось, скачиваем книгу полностью с конвертацией
let loadCached = true;
if (!book) {
book = await readerApi.loadBook(url, (state) => {
book = await readerApi.loadBook({url, enableSitesFilter: this.enableSitesFilter}, (state) => {
progress.setState(state);
});
loadCached = false;

View File

@@ -13,6 +13,11 @@ import readerApi from '../../../api/reader';
import * as utils from '../../../share/utils';
import * as cryptoUtils from '../../../share/cryptoUtils';
import localForage from 'localforage';
const ssCacheStore = localForage.createInstance({
name: 'ssCacheStore'
});
export default @Component({
watch: {
serverSyncEnabled: function() {
@@ -58,35 +63,52 @@ class ServerStorage extends Vue {
async init() {
try {
this.cachedRecent = await ssCacheStore.getItem('recent');
if (!this.cachedRecent)
await this.setCachedRecent({rev: 0, data: {}});
this.cachedRecentPatch = await ssCacheStore.getItem('recent-patch');
if (!this.cachedRecentPatch)
await this.setCachedRecentPatch({rev: 0, data: {}});
this.cachedRecentMod = await ssCacheStore.getItem('recent-mod');
if (!this.cachedRecentMod)
await this.setCachedRecentMod({rev: 0, data: {}});
if (!this.serverStorageKey) {
//генерируем новый ключ
await this.generateNewServerStorageKey();
} else {
await this.serverStorageKeyChanged();
}
bookManager.addEventListener(this.bookManagerEvent);
} finally {
this.inited = true;
}
}
async setCachedRecent(value) {
await ssCacheStore.setItem('recent', value);
this.cachedRecent = value;
}
async setCachedRecentPatch(value) {
await ssCacheStore.setItem('recent-patch', value);
this.cachedRecentPatch = value;
}
async setCachedRecentMod(value) {
await ssCacheStore.setItem('recent-mod', value);
this.cachedRecentMod = value;
}
async bookManagerEvent(eventName, itemKey) {
if (!this.serverSyncEnabled)
return;
if (eventName == 'recent-changed') {
if (itemKey) {
if (!this.recentDeltaInited) {
await this.loadRecent();
this.warning('Функции сохранения на сервер пока недоступны');
return;
}
if (!this.recentDelta)
this.recentDelta = {};
this.recentDelta[itemKey] = _.cloneDeep(bookManager.recent[itemKey]);
this.debouncedSaveRecent(itemKey);
}
}
@@ -119,8 +141,8 @@ class ServerStorage extends Vue {
await this.loadProfiles(force);
this.checkCurrentProfile();
await this.currentProfileChanged(force);
await this.loadRecent();
if (force)
const loadSuccess = await this.loadRecent();
if (loadSuccess && force)
await this.saveRecent();
}
}
@@ -340,46 +362,26 @@ class ServerStorage extends Vue {
}
}
async initRecentDelta() {
let recentDelta = null;
try {
recentDelta = await this.storageGet({recentDelta: {}});
} catch(e) {
this.error(`Ошибка соединения с сервером: ${e.message}`);
return;
}
if (recentDelta.state == 'success') {
recentDelta = recentDelta.items.recentDelta;
if (recentDelta.rev == 0)
recentDelta.data = {};
this.recentDelta = recentDelta.data;
this.recentDeltaInited = true;
} else {
this.warning(`Неверный ответ сервера: ${recentDelta.state}`);
}
}
async loadRecent(skipRevCheck = false, doNotifySuccess = true) {
if (!this.keyInited || !this.serverSyncEnabled || this.loadingRecent)
return;
this.loadingRecent = true;
try {
const oldRecentRev = bookManager.recentRev;
const oldRecentDeltaRev = bookManager.recentDeltaRev;
//проверим ревизию на сервере
let query = {recent: {}, recentPatch: {}, recentMod: {}};
let revs = null;
if (!skipRevCheck) {
try {
revs = await this.storageCheck({recent: {}, recentDelta: {}});
if (revs.state == 'success' && revs.items.recent.rev == oldRecentRev &&
revs.items.recentDelta.rev == oldRecentDeltaRev) {
if (!this.recentDeltaInited)
await this.initRecentDelta();
return;
revs = await this.storageCheck(query);
if (revs.state == 'success') {
if (revs.items.recent.rev != this.cachedRecent.rev) {
//no changes
} else if (revs.items.recentPatch.rev != this.cachedRecentPatch.rev) {
query = {recentPatch: {}, recentMod: {}};
} else if (revs.items.recentMod.rev != this.cachedRecentMod.rev) {
query = {recentMod: {}};
} else
return true;
}
} catch(e) {
this.error(`Ошибка соединения с сервером: ${e.message}`);
@@ -389,46 +391,53 @@ class ServerStorage extends Vue {
let recent = null;
try {
recent = await this.storageGet({recent: {}, recentDelta: {}});
recent = await this.storageGet(query);
} catch(e) {
this.error(`Ошибка соединения с сервером: ${e.message}`);
return;
}
if (recent.state == 'success') {
let recentDelta = recent.items.recentDelta;
recent = recent.items.recent;
let newRecent = recent.items.recent;
let newRecentPatch = recent.items.recentPatch;
let newRecentMod = recent.items.recentMod;
if (recent.rev == 0)
recent.data = {};
let newRecent = {};
if (recentDelta && recentDelta.data) {
if (recentDelta.data.diff) {
newRecent = recent.data;
const key = recentDelta.data.diff.key;
if (newRecent[key])
newRecent[key] = utils.applyObjDiff(newRecent[key], recentDelta.data.diff);
} else {
newRecent = Object.assign(recent.data, recentDelta.data);
}
this.recentDelta = recentDelta.data;
} else {
newRecent = recent.data;
this.recentDelta = {};
if (!newRecent) {
newRecent = _.cloneDeep(this.cachedRecent);
}
if (!newRecentPatch) {
newRecentPatch = _.cloneDeep(this.cachedRecentPatch);
}
if (!newRecentMod) {
newRecentMod = _.cloneDeep(this.cachedRecentMod);
}
this.recentDeltaInited = true;
if (newRecent.rev == 0) newRecent.data = {};
if (newRecentPatch.rev == 0) newRecentPatch.data = {};
if (newRecentMod.rev == 0) newRecentMod.data = {};
let result = Object.assign({}, newRecent.data, newRecentPatch.data);
const md = newRecentMod.data;
if (md.key && result[md.key])
result[md.key] = utils.applyObjDiff(result[md.key], md.mod);
if (newRecent.rev != this.cachedRecent.rev)
await this.setCachedRecent(newRecent);
if (newRecentPatch.rev != this.cachedRecentPatch.rev)
await this.setCachedRecentPatch(newRecentPatch);
if (newRecentMod.rev != this.cachedRecentMod.rev)
await this.setCachedRecentMod(newRecentMod);
if (!bookManager.loaded) {
this.warning('Ожидание загрузки списка книг перед синхронизацией');
while (!bookManager.loaded) await utils.sleep(100);
}
await bookManager.setRecent(newRecent);
await bookManager.setRecentRev(recent.rev);
await bookManager.setRecentDeltaRev(recentDelta.rev);
await bookManager.setRecent(result);
} else {
this.warning(`Неверный ответ сервера: ${recent.state}`);
return;
}
if (doNotifySuccess)
@@ -436,6 +445,7 @@ class ServerStorage extends Vue {
} finally {
this.loadingRecent = false;
}
return true;
}
async saveRecent(itemKey, recurse) {
@@ -444,91 +454,87 @@ class ServerStorage extends Vue {
const bm = bookManager;
//вычисление критерия сохранения целиком
if (!this.sameKeyCount)
this.sameKeyCount = 0;
if (this.prevItemKey == itemKey) {
this.sameKeyCount++;
} else {
this.sameKeyCount = 0;
let needSaveRecent = false;
let needSaveRecentPatch = false;
let needSaveRecentMod = false;
//newRecentMod
let newRecentMod = {};
if (itemKey && this.cachedRecentPatch.data[itemKey] && this.prevItemKey == itemKey) {
newRecentMod = _.cloneDeep(this.cachedRecentMod);
newRecentMod.rev++;
newRecentMod.data.key = itemKey;
newRecentMod.data.mod = utils.getObjDiff(this.cachedRecentPatch.data[itemKey], bm.recent[itemKey]);
needSaveRecentMod = true;
}
const l = Object.keys(this.recentDelta).length - (1*(!!this.recentDelta.diff));
this.makeDeltaDiff = (l == 1 && this.prevItemKey == itemKey ? this.makeDeltaDiff : false);
const forceSaveRecent = l > 20 || (this.sameKeyCount > 5 && (l > 1)) || (l == 1 && this.sameKeyCount > 10 && !this.makeDeltaDiff);
this.sameKeyCount = (!forceSaveRecent ? this.sameKeyCount : 0);
this.prevItemKey = itemKey;
//дифф от дельты для уменьшения размера передаваемых данных в частном случае
if (this.makeDeltaDiff) {
this.recentDelta.diff = utils.getObjDiff(this.prevSavedItem, bm.recent[itemKey]);
this.recentDelta.diff.key = itemKey;
delete this.recentDelta[itemKey];
} else if (this.recentDelta.diff) {
const key = this.recentDelta.diff.key;
if (!this.prevSavedItem && bm.recent[key])
this.prevSavedItem = _.cloneDeep(bm.recent[key]);
if (this.prevSavedItem) {
this.recentDelta[key] = utils.applyObjDiff(this.prevSavedItem, this.recentDelta.diff);
}
delete this.recentDelta.diff;
//newRecentPatch
let newRecentPatch = {};
if (itemKey && !needSaveRecentMod) {
newRecentPatch = _.cloneDeep(this.cachedRecentPatch);
newRecentPatch.rev++;
newRecentPatch.data[itemKey] = bm.recent[itemKey];
let applyMod = this.cachedRecentMod.data;
if (applyMod && applyMod.key && newRecentPatch.data[applyMod.key])
newRecentPatch.data[applyMod.key] = utils.applyObjDiff(newRecentPatch.data[applyMod.key], applyMod.mod);
newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}};
needSaveRecentPatch = true;
needSaveRecentMod = true;
}
//newRecent
let newRecent = {};
if (!itemKey || (needSaveRecentPatch && Object.keys(newRecentPatch.data).length > 10)) {
newRecent = {rev: this.cachedRecent.rev + 1, data: bm.recent};
newRecentPatch = {rev: this.cachedRecentPatch.rev + 1, data: {}};
newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}};
needSaveRecent = true;
needSaveRecentPatch = true;
needSaveRecentMod = true;
}
//query
let query = {};
if (needSaveRecent) {
query = {recent: newRecent, recentPatch: newRecentPatch, recentMod: newRecentMod};
} else if (needSaveRecentPatch) {
query = {recentPatch: newRecentPatch, recentMod: newRecentMod};
} else {
query = {recentMod: newRecentMod};
}
//сохранение
this.savingRecent = true;
try {
if (forceSaveRecent) {//сохраняем recent целиком
let result = {state: ''};
let result = {state: ''};
try {
result = await this.storageSet({recent: {rev: bm.recentRev + 1, data: bm.recent}, recentDelta: {rev: bm.recentDeltaRev + 1, data: {}}});
} catch(e) {
this.error(`Ошибка соединения с сервером (${e.message}). Данные не сохранены и могут быть перезаписаны.`);
}
if (result.state == 'reject') {
await this.loadRecent(true, false);
this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`);
if (!recurse) {
this.savingRecent = false;
this.recentDelta[itemKey] = _.cloneDeep(bm.recent[itemKey]);
this.saveRecent(itemKey, true);
return;
}
} else if (result.state == 'success') {
this.makeDeltaDiff = true;
this.prevSavedItem = _.cloneDeep(bm.recent[itemKey]);
this.recentDelta = {};
await bm.setRecentRev(bm.recentRev + 1);
await bm.setRecentDeltaRev(bm.recentDeltaRev + 1);
}
} else {//сохраняем только дифф
let result = {state: ''};
try {
result = await this.storageSet({recentDelta: {rev: bm.recentDeltaRev + 1, data: this.recentDelta}});
} catch(e) {
this.error(`Ошибка соединения с сервером (${e.message}). Данные не сохранены и могут быть перезаписаны.`);
}
if (result.state == 'reject') {
await this.loadRecent(true, false);
this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`);
if (!recurse) {
this.savingRecent = false;
this.recentDelta[itemKey] = _.cloneDeep(bm.recent[itemKey]);
this.saveRecent(itemKey, true);
return;
}
} else if (result.state == 'success') {
await bm.setRecentDeltaRev(bm.recentDeltaRev + 1);
try {
result = await this.storageSet(query);
} catch(e) {
this.error(`Ошибка соединения с сервером (${e.message}). Данные не сохранены и могут быть перезаписаны.`);
}
if (result.state == 'reject') {
await this.loadRecent(false, false);
this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`);
if (!recurse && itemKey) {
this.savingRecent = false;
this.saveRecent(itemKey, true);
return;
}
} else if (result.state == 'success') {
if (needSaveRecent && newRecent.rev)
await this.setCachedRecent(newRecent);
if (needSaveRecentPatch && newRecentPatch.rev)
await this.setCachedRecentPatch(newRecentPatch);
if (needSaveRecentMod && newRecentMod.rev)
await this.setCachedRecentMod(newRecentMod);
}
} finally {
this.savingRecent = false;

View File

@@ -304,6 +304,19 @@
</el-col>
<el-input-number v-model="addEmptyParagraphs" :min="0" :max="2"></el-input-number>
</el-form-item>
<el-form-item label="">
<el-tooltip :open-delay="500" effect="light" placement="top">
<template slot="content">
Html-фильтр вырезает лишние элементы со<br>
страницы для определенных сайтов, таких как:<br>
samlib.ru<br>
www.fanfiction.net<br>
archiveofourown.org<br>
и других
</template>
<el-checkbox v-model="enableSitesFilter" @change="needTextReload">Включить html-фильтр для сайтов</el-checkbox>
</el-tooltip>
</el-form-item>
<el-form-item label="Изображения">
<el-col :span="11">
@@ -688,6 +701,10 @@ class SettingsPage extends Vue {
this.$notify.warning({message: 'Необходимо обновить страницу (F5), чтобы изменения возымели эффект'});
}
needTextReload() {
this.$notify.warning({message: 'Необходимо обновить книгу в обход кэша, чтобы изменения возымели эффект'});
}
close() {
this.$emit('settings-toggle');
}

View File

@@ -36,9 +36,6 @@ class BookManager {
}
}
this.recentRev = await bmRecentStore.getItem('recent-rev') || 0;
this.recentDeltaRev = await bmRecentStore.getItem('recent-delta-rev') || 0;
this.recentChanged = true;
this.loadStored();//no await
@@ -440,7 +437,7 @@ class BookManager {
Object.assign(mergedRecent, value);
//подстраховка
//подстраховка от hotReload
for (let i of Object.keys(mergedRecent)) {
if (!mergedRecent[i].key || mergedRecent[i].key !== i)
delete mergedRecent[i];
@@ -466,16 +463,6 @@ class BookManager {
this.emit('recent-changed');
}
async setRecentRev(value) {
await bmRecentStore.setItem('recent-rev', value);
this.recentRev = value;
}
async setRecentDeltaRev(value) {
await bmRecentStore.setItem('recent-delta-rev', value);
this.recentDeltaRev = value;
}
addEventListener(listener) {
if (this.eventListeners.indexOf(listener) < 0)
this.eventListeners.push(listener);

View File

@@ -1,10 +1,13 @@
export const versionHistory = [
{
showUntil: '2019-10-17',
header: '0.7.2 (2019-10-18)',
header: '0.7.3 (2019-10-18)',
content:
`
<ul>
<li>внутренние переделки механизма синхронизации с сервером</li>
<li>добавлен html-фильтр для сайтов www.fanfiction.net, archiveofourown.org</li>
<li>добавлен параметр "Включить html-фильтр для сайтов" в раздел "Вид"->"Текст" в настройках</li>
<li>исправления багов</li>
</ul>
`

View File

@@ -183,6 +183,7 @@ const settingDefaults = {
showServerStorageMessages: true,
showWhatsNewDialog: true,
showMigrationDialog: true,
enableSitesFilter: true,
fontShifts: {},
showToolButton: {},

View File

@@ -1,6 +1,6 @@
{
"name": "Liberama",
"version": "0.7.2",
"version": "0.7.3",
"engines": {
"node": ">=10.0.0"
},

View File

@@ -15,7 +15,10 @@ class ReaderController extends BaseController {
try {
if (!request.url)
throw new Error(`key 'url' is empty`);
const workerId = this.readerWorker.loadBookUrl(request.url);
const workerId = this.readerWorker.loadBookUrl({
url: request.url,
enableSitesFilter: (request.hasOwnProperty('enableSitesFilter') ? request.enableSitesFilter : true)
});
const state = workerState.getState(workerId);
return (state ? state : {});
} catch (e) {

View File

@@ -20,6 +20,9 @@ class ConvertSamlib extends ConvertBase {
}
async run(data, opts) {
if (!opts.enableSitesFilter)
return false;
const checkResult = this.check(data, opts);
if (!checkResult)
return false;

View File

@@ -0,0 +1,71 @@
const URL = require('url').URL;
const ConvertHtml = require('./ConvertHtml');
const sitesFilter = {
'www.fanfiction.net': {
converter: 'cutter',
begin: `<div class='storytext xcontrast_txt nocopy' id='storytext'>`,
end: `<div style='height:5px'></div><div style='clear:both;text-align:right;'>`,
},
'archiveofourown.org': {
converter: 'cutter',
begin: `<!-- BEGIN section where work skin applies -->`,
end: `<!-- END work skin -->`,
}
};
class ConvertSites extends ConvertHtml {
check(data, opts) {
const {url, dataType} = opts;
const parsedUrl = new URL(url);
if (dataType && dataType.ext == 'html') {
if (sitesFilter[parsedUrl.hostname])
return {hostname: parsedUrl.hostname};
}
return false;
}
async run(data, opts) {
if (!opts.enableSitesFilter)
return false;
const checkResult = this.check(data, opts);
if (!checkResult)
return false;
const {hostname} = checkResult;
let text = this.decode(data).toString();
text = this[sitesFilter[hostname].converter](text, sitesFilter[hostname]);
if (text === false)
return false;
return await super.run(Buffer.from(text), {skipCheck: true, cutTitle: true});
}
getTitle(text) {
let title = '';
const m = text.match(/<title>([\s\S]*?)<\/title>/);
if (m)
title = m[1];
return `<title>${title.trim()}</title>`;
}
cutter(text, opts) {
const title = this.getTitle(text);
const l = text.indexOf(opts.begin);
const r = text.indexOf(opts.end);
if (l < 0 || r < 0 || r <= l)
return false;
return text.substring(l, r) + title;
}
}
module.exports = ConvertSites;

View File

@@ -11,6 +11,7 @@ const convertClassFactory = [
require('./ConvertMobi'),
require('./ConvertFb2'),
require('./ConvertSamlib'),
require('./ConvertSites'),
require('./ConvertHtml'),
];
@@ -24,13 +25,14 @@ class BookConverter {
}
}
async convertToFb2(inputFiles, outputFile, url, callback) {
async convertToFb2(inputFiles, outputFile, opts, callback) {
const selectedFileType = await this.detector.detectFile(inputFiles.selectedFile);
const data = await fs.readFile(inputFiles.selectedFile);
const convertOpts = Object.assign({}, opts, {inputFiles, callback, dataType: selectedFileType});
let result = false;
for (const convert of this.convertFactory) {
result = await convert.run(data, {inputFiles, url, callback, dataType: selectedFileType});
result = await convert.run(data, convertOpts);
if (result) {
await fs.writeFile(outputFile, result);
break;
@@ -38,14 +40,14 @@ class BookConverter {
}
if (!result && inputFiles.nesting) {
result = await this.convertToFb2(inputFiles.nesting, outputFile, url, callback);
result = await this.convertToFb2(inputFiles.nesting, outputFile, opts, callback);
}
if (!result) {
if (selectedFileType)
throw new Error(`Этот формат файла не поддерживается: ${selectedFileType.mime}`);
else {
throw new Error(`Не удалось определить формат файла: ${url}`);
throw new Error(`Не удалось определить формат файла: ${opts.url}`);
}
}

View File

@@ -32,7 +32,8 @@ class ReaderWorker {
}
}
async loadBook(url, wState) {
async loadBook(opts, wState) {
const url = opts.url;
let errMes = '';
let decompDir = '';
let downloadedFilename = '';
@@ -77,7 +78,7 @@ class ReaderWorker {
//конвертирование в fb2
wState.set({state: 'convert', step: 3, progress: 0});
convertFilename = `${this.config.tempDownloadDir}/${tempFilename2}`;
await this.bookConverter.convertToFb2(decompFiles, convertFilename, url, progress => {
await this.bookConverter.convertToFb2(decompFiles, convertFilename, opts, progress => {
wState.set({progress});
});
@@ -105,12 +106,12 @@ class ReaderWorker {
}
}
loadBookUrl(url) {
loadBookUrl(opts) {
const workerId = workerState.generateWorkerId();
const wState = workerState.getControl(workerId);
wState.set({state: 'start'});
this.loadBook(url, wState);
this.loadBook(opts, wState);
return workerId;
}

View File

@@ -20,7 +20,7 @@ function webpackDevMiddleware(app) {
function logQueries(app) {
app.use(function(req, res, next) {
const start = Date.now();
log(`${req.method} ${req.originalUrl} ${JSON.stringify(req.body).substr(0, 2000)}`);
log(`${req.method} ${req.originalUrl} ${JSON.stringify(req.body).substr(0, 4000)}`);
//log(`${JSON.stringify(req.headers, null, 2)}`)
res.once('finish', () => {
log(`${Date.now() - start}ms`);