From 7853a14ce685f087317ef91f2025fbd3f55ee598 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Oct 2019 13:53:44 +0700 Subject: [PATCH 1/9] =?UTF-8?q?=D0=9D=D0=B0=D1=87=D0=B0=D0=BB=D0=BE=20?= =?UTF-8?q?=D0=BE=D1=87=D0=B5=D1=80=D0=B5=D0=B4=D0=BD=D0=BE=D0=B9=20=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D0=B4=D0=B5=D0=BB=D0=BA=D0=B8=20ServerStorag?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Reader/ServerStorage/ServerStorage.vue | 255 ++++++++---------- client/components/Reader/share/bookManager.js | 13 - 2 files changed, 113 insertions(+), 155 deletions(-) diff --git a/client/components/Reader/ServerStorage/ServerStorage.vue b/client/components/Reader/ServerStorage/ServerStorage.vue index 9c55ec3b..768196e9 100644 --- a/client/components/Reader/ServerStorage/ServerStorage.vue +++ b/client/components/Reader/ServerStorage/ServerStorage.vue @@ -13,6 +13,11 @@ import readerApi from '../../../api/reader'; import * as utils from '../../../share/utils'; import * as cryptoUtils from '../../../share/cryptoUtils'; +import localForage from 'localforage'; +const ssCacheStore = localForage.createInstance({ + name: 'ssCacheStore' +}); + export default @Component({ watch: { serverSyncEnabled: function() { @@ -58,35 +63,52 @@ class ServerStorage extends Vue { async init() { try { + this.cachedRecent = await ssCacheStore.getItem('recent'); + if (!this.cachedRecent) + await this.setCachedRecent({rev: 0, data: {}}); + + this.cachedRecentPatch = await ssCacheStore.getItem('recent-patch'); + if (!this.cachedRecentPatch) + await this.setCachedRecentPatch({rev: 0, data: {}}); + + this.cachedRecentMod = await ssCacheStore.getItem('recent-mod'); + if (!this.cachedRecentMod) + await this.setCachedRecentMod({rev: 0, data: {}}); + if (!this.serverStorageKey) { //генерируем новый ключ await this.generateNewServerStorageKey(); } else { await this.serverStorageKeyChanged(); } + bookManager.addEventListener(this.bookManagerEvent); } finally { this.inited = true; } } + async setCachedRecent(value) { + await ssCacheStore.setItem('recent', value); + this.cachedRecent = value; + } + + async setCachedRecentPatch(value) { + await ssCacheStore.setItem('recent-patch', value); + this.cachedRecentPatch = value; + } + + async setCachedRecentMod(value) { + await ssCacheStore.setItem('recent-mod', value); + this.cachedRecentMod = value; + } + async bookManagerEvent(eventName, itemKey) { if (!this.serverSyncEnabled) return; if (eventName == 'recent-changed') { if (itemKey) { - if (!this.recentDeltaInited) { - await this.loadRecent(); - this.warning('Функции сохранения на сервер пока недоступны'); - return; - } - - if (!this.recentDelta) - this.recentDelta = {}; - - this.recentDelta[itemKey] = _.cloneDeep(bookManager.recent[itemKey]); - this.debouncedSaveRecent(itemKey); } } @@ -340,46 +362,26 @@ class ServerStorage extends Vue { } } - async initRecentDelta() { - let recentDelta = null; - try { - recentDelta = await this.storageGet({recentDelta: {}}); - } catch(e) { - this.error(`Ошибка соединения с сервером: ${e.message}`); - return; - } - - if (recentDelta.state == 'success') { - recentDelta = recentDelta.items.recentDelta; - - if (recentDelta.rev == 0) - recentDelta.data = {}; - - this.recentDelta = recentDelta.data; - this.recentDeltaInited = true; - } else { - this.warning(`Неверный ответ сервера: ${recentDelta.state}`); - } - } - async loadRecent(skipRevCheck = false, doNotifySuccess = true) { if (!this.keyInited || !this.serverSyncEnabled || this.loadingRecent) return; - this.loadingRecent = true; try { - const oldRecentRev = bookManager.recentRev; - const oldRecentDeltaRev = bookManager.recentDeltaRev; //проверим ревизию на сервере + let query = {recent: {}, recentPatch: {}, recentMod: {}}; let revs = null; if (!skipRevCheck) { try { - revs = await this.storageCheck({recent: {}, recentDelta: {}}); - if (revs.state == 'success' && revs.items.recent.rev == oldRecentRev && - revs.items.recentDelta.rev == oldRecentDeltaRev) { - if (!this.recentDeltaInited) - await this.initRecentDelta(); - return; + revs = await this.storageCheck(query); + if (revs.state == 'success') { + if (revs.items.recent.rev != this.cachedRecent.rev) { + //no changes + } else if (revs.items.recentPatch.rev != this.cachedRecentPatch.rev) { + query = {recentPatch: {}, recentMod: {}}; + } else if (revs.items.recentMod.rev != this.cachedRecentMod.rev) { + query = {recentMod: {}}; + } else + return; } } catch(e) { this.error(`Ошибка соединения с сервером: ${e.message}`); @@ -389,44 +391,45 @@ class ServerStorage extends Vue { let recent = null; try { - recent = await this.storageGet({recent: {}, recentDelta: {}}); + recent = await this.storageGet(query); } catch(e) { this.error(`Ошибка соединения с сервером: ${e.message}`); return; } if (recent.state == 'success') { - let recentDelta = recent.items.recentDelta; - recent = recent.items.recent; + let newRecent = recent.items.recent; + let newRecentPatch = recent.items.recentPatch; + let newRecentMod = recent.items.recentMod; - if (recent.rev == 0) - recent.data = {}; + if (!newRecent) + newRecent = _.cloneDeep(this.cachedRecent); + if (!newRecentPatch) + newRecentPatch = _.cloneDeep(this.cachedRecentPatch); + if (!newRecentMod) + newRecentMod = _.cloneDeep(this.cachedRecentMod); - let newRecent = {}; - if (recentDelta && recentDelta.data) { - if (recentDelta.data.diff) { - newRecent = recent.data; - const key = recentDelta.data.diff.key; - if (newRecent[key]) - newRecent[key] = utils.applyObjDiff(newRecent[key], recentDelta.data.diff); - } else { - newRecent = Object.assign(recent.data, recentDelta.data); - } - this.recentDelta = recentDelta.data; - } else { - newRecent = recent.data; - this.recentDelta = {}; - } + if (newRecent.rev == 0) newRecent.data = {}; + if (newRecentPatch.rev == 0) newRecentPatch.data = {}; + if (newRecentMod.rev == 0) newRecentMod.data = {}; - this.recentDeltaInited = true; + let result = Object.assign({}, newRecent.data, newRecentPatch.data); + if (newRecentMod.key && result[newRecentMod.key]) + result[newRecentMod.key] = utils.applyObjDiff(result[newRecentMod.key], newRecentMod.mod); + + if (newRecent.rev != this.cachedRecent.rev) + await this.setCachedRecent(newRecent); + if (newRecentPatch.rev != this.cachedRecentPatch.rev) + await this.setCachedRecentPatch(newRecentPatch); + if (newRecentMod.rev != this.cachedRecentMod.rev) + await this.setCachedRecentMod(newRecentMod); if (!bookManager.loaded) { this.warning('Ожидание загрузки списка книг перед синхронизацией'); while (!bookManager.loaded) await utils.sleep(100); } - await bookManager.setRecent(newRecent); - await bookManager.setRecentRev(recent.rev); - await bookManager.setRecentDeltaRev(recentDelta.rev); + + await bookManager.setRecent(result); } else { this.warning(`Неверный ответ сервера: ${recent.state}`); } @@ -444,91 +447,59 @@ class ServerStorage extends Vue { const bm = bookManager; - //вычисление критерия сохранения целиком - if (!this.sameKeyCount) - this.sameKeyCount = 0; - if (this.prevItemKey == itemKey) { - this.sameKeyCount++; + let needSaveRecent = false; + let needSaveRecentPatch = false; + let needSaveRecentMod = true; + + let newRecentMod = _.cloneDeep(this.cachedRecentPatch); + newRecentMod.rev++; + + let newRecentPatch = _.cloneDeep(this.cachedRecentPatch); + newRecentPatch.data[itemKey] = bm.recent[itemKey]; + newRecentPatch.rev++; + needSaveRecentPatch = true; + + let newRecent = {rev: this.cachedRecent.rev + 1, data: bm.recent}; + + let query = {}; + if (needSaveRecent) { + query = {recent: newRecent, recentPatch: newRecentPatch, recentMod: newRecentMod}; + } else if (needSaveRecentPatch) { + query = {recentPatch: newRecentPatch, recentMod: newRecentMod}; } else { - this.sameKeyCount = 0; - } - - const l = Object.keys(this.recentDelta).length - (1*(!!this.recentDelta.diff)); - this.makeDeltaDiff = (l == 1 && this.prevItemKey == itemKey ? this.makeDeltaDiff : false); - const forceSaveRecent = l > 20 || (this.sameKeyCount > 5 && (l > 1)) || (l == 1 && this.sameKeyCount > 10 && !this.makeDeltaDiff); - - this.sameKeyCount = (!forceSaveRecent ? this.sameKeyCount : 0); - this.prevItemKey = itemKey; - - //дифф от дельты для уменьшения размера передаваемых данных в частном случае - if (this.makeDeltaDiff) { - this.recentDelta.diff = utils.getObjDiff(this.prevSavedItem, bm.recent[itemKey]); - this.recentDelta.diff.key = itemKey; - delete this.recentDelta[itemKey]; - } else if (this.recentDelta.diff) { - const key = this.recentDelta.diff.key; - if (!this.prevSavedItem && bm.recent[key]) - this.prevSavedItem = _.cloneDeep(bm.recent[key]); - if (this.prevSavedItem) { - this.recentDelta[key] = utils.applyObjDiff(this.prevSavedItem, this.recentDelta.diff); - } - delete this.recentDelta.diff; + query = {recentMod: newRecentMod}; } //сохранение this.savingRecent = true; try { - if (forceSaveRecent) {//сохраняем recent целиком - let result = {state: ''}; + let result = {state: ''}; - try { - result = await this.storageSet({recent: {rev: bm.recentRev + 1, data: bm.recent}, recentDelta: {rev: bm.recentDeltaRev + 1, data: {}}}); - } catch(e) { - this.error(`Ошибка соединения с сервером (${e.message}). Данные не сохранены и могут быть перезаписаны.`); + try { + result = await this.storageSet(query); + } catch(e) { + this.error(`Ошибка соединения с сервером (${e.message}). Данные не сохранены и могут быть перезаписаны.`); + } + + if (result.state == 'reject') { + + await this.loadRecent(true, false); + + this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`); + if (!recurse) { + this.savingRecent = false; + this.saveRecent(itemKey, true); + return; } + } else if (result.state == 'success') { + //this.prevSavedItem = _.cloneDeep(bm.recent[itemKey]); - if (result.state == 'reject') { - - await this.loadRecent(true, false); - - this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`); - if (!recurse) { - this.savingRecent = false; - this.recentDelta[itemKey] = _.cloneDeep(bm.recent[itemKey]); - this.saveRecent(itemKey, true); - return; - } - } else if (result.state == 'success') { - this.makeDeltaDiff = true; - this.prevSavedItem = _.cloneDeep(bm.recent[itemKey]); - - this.recentDelta = {}; - await bm.setRecentRev(bm.recentRev + 1); - await bm.setRecentDeltaRev(bm.recentDeltaRev + 1); - } - } else {//сохраняем только дифф - let result = {state: ''}; - - try { - result = await this.storageSet({recentDelta: {rev: bm.recentDeltaRev + 1, data: this.recentDelta}}); - } catch(e) { - this.error(`Ошибка соединения с сервером (${e.message}). Данные не сохранены и могут быть перезаписаны.`); - } - - if (result.state == 'reject') { - - await this.loadRecent(true, false); - - this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`); - if (!recurse) { - this.savingRecent = false; - this.recentDelta[itemKey] = _.cloneDeep(bm.recent[itemKey]); - this.saveRecent(itemKey, true); - return; - } - } else if (result.state == 'success') { - await bm.setRecentDeltaRev(bm.recentDeltaRev + 1); - } + if (needSaveRecent) + await this.setCachedRecent(newRecent); + if (needSaveRecentPatch) + await this.setCachedRecentPatch(newRecentPatch); + if (needSaveRecentMod) + await this.setCachedRecentMod(newRecentMod); } } finally { this.savingRecent = false; diff --git a/client/components/Reader/share/bookManager.js b/client/components/Reader/share/bookManager.js index e58a2e0b..b8405574 100644 --- a/client/components/Reader/share/bookManager.js +++ b/client/components/Reader/share/bookManager.js @@ -36,9 +36,6 @@ class BookManager { } } - this.recentRev = await bmRecentStore.getItem('recent-rev') || 0; - this.recentDeltaRev = await bmRecentStore.getItem('recent-delta-rev') || 0; - this.recentChanged = true; this.loadStored();//no await @@ -466,16 +463,6 @@ class BookManager { this.emit('recent-changed'); } - async setRecentRev(value) { - await bmRecentStore.setItem('recent-rev', value); - this.recentRev = value; - } - - async setRecentDeltaRev(value) { - await bmRecentStore.setItem('recent-delta-rev', value); - this.recentDeltaRev = value; - } - addEventListener(listener) { if (this.eventListeners.indexOf(listener) < 0) this.eventListeners.push(listener); From 5184661652a1f7b1cecbb32ade1f33881dee2e1a Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Oct 2019 14:00:52 +0700 Subject: [PATCH 2/9] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B0=D1=8F=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/dev.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/dev.js b/server/dev.js index df75cc84..7782d385 100644 --- a/server/dev.js +++ b/server/dev.js @@ -20,7 +20,7 @@ function webpackDevMiddleware(app) { function logQueries(app) { app.use(function(req, res, next) { const start = Date.now(); - log(`${req.method} ${req.originalUrl} ${JSON.stringify(req.body).substr(0, 2000)}`); + log(`${req.method} ${req.originalUrl} ${JSON.stringify(req.body).substr(0, 4000)}`); //log(`${JSON.stringify(req.headers, null, 2)}`) res.once('finish', () => { log(`${Date.now() - start}ms`); From 51ebbbc569e682a7a12c2df5a8a0eb9533a6da62 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Oct 2019 15:19:53 +0700 Subject: [PATCH 3/9] =?UTF-8?q?=D0=97=D0=B0=D0=BA=D0=BE=D0=BD=D1=87=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D0=BE=D1=87=D0=B5=D1=80=D0=B5=D0=B4=D0=BD=D0=B0?= =?UTF-8?q?=D1=8F=20=D0=BF=D0=B5=D1=80=D0=B5=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=BA=D0=B0=20loadRecent=20&=20saveRecent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Reader/ServerStorage/ServerStorage.vue | 83 ++++++++++++++----- client/components/Reader/share/bookManager.js | 2 +- 2 files changed, 63 insertions(+), 22 deletions(-) diff --git a/client/components/Reader/ServerStorage/ServerStorage.vue b/client/components/Reader/ServerStorage/ServerStorage.vue index 768196e9..45e2a239 100644 --- a/client/components/Reader/ServerStorage/ServerStorage.vue +++ b/client/components/Reader/ServerStorage/ServerStorage.vue @@ -141,8 +141,8 @@ class ServerStorage extends Vue { await this.loadProfiles(force); this.checkCurrentProfile(); await this.currentProfileChanged(force); - await this.loadRecent(); - if (force) + const loadSuccess = await this.loadRecent(); + if (loadSuccess && force) await this.saveRecent(); } } @@ -381,7 +381,7 @@ class ServerStorage extends Vue { } else if (revs.items.recentMod.rev != this.cachedRecentMod.rev) { query = {recentMod: {}}; } else - return; + return true; } } catch(e) { this.error(`Ошибка соединения с сервером: ${e.message}`); @@ -402,20 +402,25 @@ class ServerStorage extends Vue { let newRecentPatch = recent.items.recentPatch; let newRecentMod = recent.items.recentMod; - if (!newRecent) + if (!newRecent) { newRecent = _.cloneDeep(this.cachedRecent); - if (!newRecentPatch) + } + if (!newRecentPatch) { newRecentPatch = _.cloneDeep(this.cachedRecentPatch); - if (!newRecentMod) + } + if (!newRecentMod) { newRecentMod = _.cloneDeep(this.cachedRecentMod); + } if (newRecent.rev == 0) newRecent.data = {}; if (newRecentPatch.rev == 0) newRecentPatch.data = {}; if (newRecentMod.rev == 0) newRecentMod.data = {}; let result = Object.assign({}, newRecent.data, newRecentPatch.data); - if (newRecentMod.key && result[newRecentMod.key]) - result[newRecentMod.key] = utils.applyObjDiff(result[newRecentMod.key], newRecentMod.mod); + + const md = newRecentMod.data; + if (md.key && result[md.key]) + result[md.key] = utils.applyObjDiff(result[md.key], md.mod); if (newRecent.rev != this.cachedRecent.rev) await this.setCachedRecent(newRecent); @@ -432,6 +437,7 @@ class ServerStorage extends Vue { await bookManager.setRecent(result); } else { this.warning(`Неверный ответ сервера: ${recent.state}`); + return; } if (doNotifySuccess) @@ -439,6 +445,7 @@ class ServerStorage extends Vue { } finally { this.loadingRecent = false; } + return true; } async saveRecent(itemKey, recurse) { @@ -449,18 +456,52 @@ class ServerStorage extends Vue { let needSaveRecent = false; let needSaveRecentPatch = false; - let needSaveRecentMod = true; + let needSaveRecentMod = false; - let newRecentMod = _.cloneDeep(this.cachedRecentPatch); - newRecentMod.rev++; + let applyMod = null; - let newRecentPatch = _.cloneDeep(this.cachedRecentPatch); - newRecentPatch.data[itemKey] = bm.recent[itemKey]; - newRecentPatch.rev++; - needSaveRecentPatch = true; + //newRecentMod + let newRecentMod = {}; - let newRecent = {rev: this.cachedRecent.rev + 1, data: bm.recent}; + if (itemKey && this.cachedRecentPatch.data[itemKey]) { + if (this.prevItemKey == itemKey) {//сохраняем только дифф + newRecentMod = _.cloneDeep(this.cachedRecentMod); + newRecentMod.rev++; + newRecentMod.data.key = itemKey; + newRecentMod.data.mod = utils.getObjDiff(this.cachedRecentPatch.data[itemKey], bm.recent[itemKey]); + needSaveRecentMod = true; + } else {//ключ не совпадает, надо сохранять патч + applyMod = newRecentMod.data; + } + } + this.prevItemKey = itemKey; + + //newRecentPatch + let newRecentPatch = {}; + if (itemKey && !needSaveRecentMod) { + newRecentPatch = _.cloneDeep(this.cachedRecentPatch); + newRecentPatch.rev++; + newRecentPatch.data[itemKey] = bm.recent[itemKey]; + if (applyMod && applyMod.key && newRecentPatch.data[applyMod.key]) + newRecentPatch.data[applyMod.key] = utils.applyObjDiff(newRecentPatch.data[applyMod.key], applyMod.mod); + newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}}; + needSaveRecentPatch = true; + needSaveRecentMod = true; + } + + //newRecent + let newRecent = {}; + if (!itemKey || (needSaveRecentPatch && Object.keys(newRecentPatch.data).length > 2)) { + newRecent = {rev: this.cachedRecent.rev + 1, data: bm.recent}; + newRecentPatch = {rev: this.cachedRecentPatch.rev + 1, data: {}}; + newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}}; + needSaveRecent = true; + needSaveRecentPatch = true; + needSaveRecentMod = true; + } + + //query let query = {}; if (needSaveRecent) { query = {recent: newRecent, recentPatch: newRecentPatch, recentMod: newRecentMod}; @@ -483,10 +524,10 @@ class ServerStorage extends Vue { if (result.state == 'reject') { - await this.loadRecent(true, false); + await this.loadRecent(false, false); this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`); - if (!recurse) { + if (!recurse && itemKey) { this.savingRecent = false; this.saveRecent(itemKey, true); return; @@ -494,11 +535,11 @@ class ServerStorage extends Vue { } else if (result.state == 'success') { //this.prevSavedItem = _.cloneDeep(bm.recent[itemKey]); - if (needSaveRecent) + if (needSaveRecent && newRecent.rev) await this.setCachedRecent(newRecent); - if (needSaveRecentPatch) + if (needSaveRecentPatch && newRecentPatch.rev) await this.setCachedRecentPatch(newRecentPatch); - if (needSaveRecentMod) + if (needSaveRecentMod && newRecentMod.rev) await this.setCachedRecentMod(newRecentMod); } } finally { diff --git a/client/components/Reader/share/bookManager.js b/client/components/Reader/share/bookManager.js index b8405574..efef4552 100644 --- a/client/components/Reader/share/bookManager.js +++ b/client/components/Reader/share/bookManager.js @@ -437,7 +437,7 @@ class BookManager { Object.assign(mergedRecent, value); - //подстраховка + //подстраховка от hotReload for (let i of Object.keys(mergedRecent)) { if (!mergedRecent[i].key || mergedRecent[i].key !== i) delete mergedRecent[i]; From db5650e276a953b8ea8801dbcdcbf48c46c868da Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Oct 2019 15:20:32 +0700 Subject: [PATCH 4/9] =?UTF-8?q?=D0=9C=D0=B5=D0=BB=D0=BA=D0=B0=D1=8F=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/ServerStorage/ServerStorage.vue | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/components/Reader/ServerStorage/ServerStorage.vue b/client/components/Reader/ServerStorage/ServerStorage.vue index 45e2a239..023afcdf 100644 --- a/client/components/Reader/ServerStorage/ServerStorage.vue +++ b/client/components/Reader/ServerStorage/ServerStorage.vue @@ -533,8 +533,6 @@ class ServerStorage extends Vue { return; } } else if (result.state == 'success') { - //this.prevSavedItem = _.cloneDeep(bm.recent[itemKey]); - if (needSaveRecent && newRecent.rev) await this.setCachedRecent(newRecent); if (needSaveRecentPatch && newRecentPatch.rev) From f6513d40c839f3f9878bcc310c03335834b2fafd Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Oct 2019 15:32:05 +0700 Subject: [PATCH 5/9] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C=D1=88?= =?UTF-8?q?=D0=B0=D1=8F=20=D0=BF=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/components/Reader/ServerStorage/ServerStorage.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/components/Reader/ServerStorage/ServerStorage.vue b/client/components/Reader/ServerStorage/ServerStorage.vue index 023afcdf..5b0d5d7c 100644 --- a/client/components/Reader/ServerStorage/ServerStorage.vue +++ b/client/components/Reader/ServerStorage/ServerStorage.vue @@ -492,7 +492,7 @@ class ServerStorage extends Vue { //newRecent let newRecent = {}; - if (!itemKey || (needSaveRecentPatch && Object.keys(newRecentPatch.data).length > 2)) { + if (!itemKey || (needSaveRecentPatch && Object.keys(newRecentPatch.data).length > 10)) { newRecent = {rev: this.cachedRecent.rev + 1, data: bm.recent}; newRecentPatch = {rev: this.cachedRecentPatch.rev + 1, data: {}}; newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}}; From 558fed31aa8272b01d2e743eac3c4c601a1cc344 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Oct 2019 17:07:28 +0700 Subject: [PATCH 6/9] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=20=D1=84=D0=B8=D0=BB=D1=8C=D1=82=D1=80=20=D0=B4=D0=BB?= =?UTF-8?q?=D1=8F=20=D0=BD=D0=B5=D0=BA=D0=BE=D1=82=D0=BE=D1=80=D1=8B=D1=85?= =?UTF-8?q?=20=D1=81=D0=B0=D0=B9=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/BookConverter/ConvertSites.js | 68 +++++++++++++++++++++++ server/core/BookConverter/index.js | 1 + 2 files changed, 69 insertions(+) create mode 100644 server/core/BookConverter/ConvertSites.js diff --git a/server/core/BookConverter/ConvertSites.js b/server/core/BookConverter/ConvertSites.js new file mode 100644 index 00000000..f829c665 --- /dev/null +++ b/server/core/BookConverter/ConvertSites.js @@ -0,0 +1,68 @@ +const URL = require('url').URL; + +const ConvertHtml = require('./ConvertHtml'); + +const sitesFilter = { + 'www.fanfiction.net': { + converter: 'cutter', + begin: `
`, + end: `
`, + }, + 'archiveofourown.org': { + converter: 'cutter', + begin: ``, + end: ``, + } +}; + +class ConvertSites extends ConvertHtml { + check(data, opts) { + const {url, dataType} = opts; + + const parsedUrl = new URL(url); + if (dataType && dataType.ext == 'html') { + if (sitesFilter[parsedUrl.hostname]) + return {hostname: parsedUrl.hostname}; + } + + return false; + } + + async run(data, opts) { + const checkResult = this.check(data, opts); + if (!checkResult) + return false; + + const {hostname} = checkResult; + + let text = this.decode(data).toString(); + + text = this[sitesFilter[hostname].converter](text, sitesFilter[hostname]); + + if (text === false) + return false; + + return await super.run(Buffer.from(text), {skipCheck: true, cutTitle: true}); + } + + getTitle(text) { + let title = ''; + const m = text.match(/([\s\S]*?)<\/title>/); + if (m) + title = m[1]; + + return `<title>${title.trim()}`; + } + + cutter(text, opts) { + const title = this.getTitle(text); + const l = text.indexOf(opts.begin); + const r = text.indexOf(opts.end); + if (l < 0 || r < 0 || r <= l) + return false; + + return text.substring(l, r) + title; + } +} + +module.exports = ConvertSites; diff --git a/server/core/BookConverter/index.js b/server/core/BookConverter/index.js index 433c94b8..4b719ec1 100644 --- a/server/core/BookConverter/index.js +++ b/server/core/BookConverter/index.js @@ -11,6 +11,7 @@ const convertClassFactory = [ require('./ConvertMobi'), require('./ConvertFb2'), require('./ConvertSamlib'), + require('./ConvertSites'), require('./ConvertHtml'), ]; From e2bea407ee1c3c14039f3560c9a5929ff01db203 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Oct 2019 17:12:30 +0700 Subject: [PATCH 7/9] 0.7.3 --- client/components/Reader/versionHistory.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/components/Reader/versionHistory.js b/client/components/Reader/versionHistory.js index 3cdb9d90..ae8c1ef0 100644 --- a/client/components/Reader/versionHistory.js +++ b/client/components/Reader/versionHistory.js @@ -1,10 +1,13 @@ export const versionHistory = [ { showUntil: '2019-10-17', - header: '0.7.2 (2019-10-18)', + header: '0.7.3 (2019-10-18)', content: `
    +
  • внутренние переделки механизма синхронизации с сервером
    • +
  • добавлен html-фильтр для сайтов www.fanfiction.net, archiveofourown.org
    • +
  • добавлен параметр "Включить html-фильтр для сайтов" в раздел "Вид"->"Текст" в настройках
  • исправления багов
` From d76f60639c23a278762f192970bc1d9d3b4d621e Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Oct 2019 19:13:17 +0700 Subject: [PATCH 8/9] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=20=D0=BF=D0=B0=D1=80=D0=B0=D0=BC=D0=B5=D1=82=D1=80=20"?= =?UTF-8?q?=D0=92=D0=BA=D0=BB=D1=8E=D1=87=D0=B8=D1=82=D1=8C=20html-=D1=84?= =?UTF-8?q?=D0=B8=D0=BB=D1=8C=D1=82=D1=80=20=D0=B4=D0=BB=D1=8F=20=D1=81?= =?UTF-8?q?=D0=B0=D0=B9=D1=82=D0=BE=D0=B2"=20=D0=B2=20=D1=80=D0=B0=D0=B7?= =?UTF-8?q?=D0=B4=D0=B5=D0=BB=20"=D0=92=D0=B8=D0=B4"->"=D0=A2=D0=B5=D0=BA?= =?UTF-8?q?=D1=81=D1=82"=20=D0=B2=20=D0=BD=D0=B0=D1=81=D1=82=D1=80=D0=BE?= =?UTF-8?q?=D0=B9=D0=BA=D0=B0=D1=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/api/reader.js | 4 ++-- client/components/Reader/Reader.vue | 3 ++- .../Reader/SettingsPage/SettingsPage.vue | 17 +++++++++++++++++ client/store/modules/reader.js | 1 + server/controllers/ReaderController.js | 5 ++++- server/core/BookConverter/ConvertSamlib.js | 3 +++ server/core/BookConverter/ConvertSites.js | 3 +++ server/core/BookConverter/index.js | 9 +++++---- server/core/ReaderWorker.js | 9 +++++---- 9 files changed, 42 insertions(+), 12 deletions(-) diff --git a/client/api/reader.js b/client/api/reader.js index 7d45eada..52bfbde6 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -11,11 +11,11 @@ const workerApi = axios.create({ }); class Reader { - async loadBook(url, callback) { + async loadBook(opts, callback) { const refreshPause = 300; if (!callback) callback = () => {}; - let response = await api.post('/load-book', {type: 'url', url}); + let response = await api.post('/load-book', opts); const workerId = response.data.workerId; if (!workerId) diff --git a/client/components/Reader/Reader.vue b/client/components/Reader/Reader.vue index b3055709..e227627d 100644 --- a/client/components/Reader/Reader.vue +++ b/client/components/Reader/Reader.vue @@ -356,6 +356,7 @@ class Reader extends Vue { this.showWhatsNewDialog = settings.showWhatsNewDialog; this.showMigrationDialog = settings.showMigrationDialog; this.showToolButton = settings.showToolButton; + this.enableSitesFilter = settings.enableSitesFilter; this.updateHeaderMinWidth(); } @@ -1003,7 +1004,7 @@ class Reader extends Vue { // не удалось, скачиваем книгу полностью с конвертацией let loadCached = true; if (!book) { - book = await readerApi.loadBook(url, (state) => { + book = await readerApi.loadBook({url, enableSitesFilter: this.enableSitesFilter}, (state) => { progress.setState(state); }); loadCached = false; diff --git a/client/components/Reader/SettingsPage/SettingsPage.vue b/client/components/Reader/SettingsPage/SettingsPage.vue index 301345bc..b407cd4a 100644 --- a/client/components/Reader/SettingsPage/SettingsPage.vue +++ b/client/components/Reader/SettingsPage/SettingsPage.vue @@ -304,6 +304,19 @@ + + + + Включить html-фильтр для сайтов + + @@ -688,6 +701,10 @@ class SettingsPage extends Vue { this.$notify.warning({message: 'Необходимо обновить страницу (F5), чтобы изменения возымели эффект'}); } + needTextReload() { + this.$notify.warning({message: 'Необходимо обновить книгу в обход кэша, чтобы изменения возымели эффект'}); + } + close() { this.$emit('settings-toggle'); } diff --git a/client/store/modules/reader.js b/client/store/modules/reader.js index 9ad4c8ae..154021e5 100644 --- a/client/store/modules/reader.js +++ b/client/store/modules/reader.js @@ -183,6 +183,7 @@ const settingDefaults = { showServerStorageMessages: true, showWhatsNewDialog: true, showMigrationDialog: true, + enableSitesFilter: true, fontShifts: {}, showToolButton: {}, diff --git a/server/controllers/ReaderController.js b/server/controllers/ReaderController.js index 7811dc94..1fd19366 100644 --- a/server/controllers/ReaderController.js +++ b/server/controllers/ReaderController.js @@ -15,7 +15,10 @@ class ReaderController extends BaseController { try { if (!request.url) throw new Error(`key 'url' is empty`); - const workerId = this.readerWorker.loadBookUrl(request.url); + const workerId = this.readerWorker.loadBookUrl({ + url: request.url, + enableSitesFilter: (request.hasOwnProperty('enableSitesFilter') ? request.enableSitesFilter : true) + }); const state = workerState.getState(workerId); return (state ? state : {}); } catch (e) { diff --git a/server/core/BookConverter/ConvertSamlib.js b/server/core/BookConverter/ConvertSamlib.js index 31fe9067..a690ade7 100644 --- a/server/core/BookConverter/ConvertSamlib.js +++ b/server/core/BookConverter/ConvertSamlib.js @@ -20,6 +20,9 @@ class ConvertSamlib extends ConvertBase { } async run(data, opts) { + if (!opts.enableSitesFilter) + return false; + const checkResult = this.check(data, opts); if (!checkResult) return false; diff --git a/server/core/BookConverter/ConvertSites.js b/server/core/BookConverter/ConvertSites.js index f829c665..cba41d7e 100644 --- a/server/core/BookConverter/ConvertSites.js +++ b/server/core/BookConverter/ConvertSites.js @@ -29,6 +29,9 @@ class ConvertSites extends ConvertHtml { } async run(data, opts) { + if (!opts.enableSitesFilter) + return false; + const checkResult = this.check(data, opts); if (!checkResult) return false; diff --git a/server/core/BookConverter/index.js b/server/core/BookConverter/index.js index 4b719ec1..66657a25 100644 --- a/server/core/BookConverter/index.js +++ b/server/core/BookConverter/index.js @@ -25,13 +25,14 @@ class BookConverter { } } - async convertToFb2(inputFiles, outputFile, url, callback) { + async convertToFb2(inputFiles, outputFile, opts, callback) { const selectedFileType = await this.detector.detectFile(inputFiles.selectedFile); const data = await fs.readFile(inputFiles.selectedFile); + const convertOpts = Object.assign({}, opts, {inputFiles, callback, dataType: selectedFileType}); let result = false; for (const convert of this.convertFactory) { - result = await convert.run(data, {inputFiles, url, callback, dataType: selectedFileType}); + result = await convert.run(data, convertOpts); if (result) { await fs.writeFile(outputFile, result); break; @@ -39,14 +40,14 @@ class BookConverter { } if (!result && inputFiles.nesting) { - result = await this.convertToFb2(inputFiles.nesting, outputFile, url, callback); + result = await this.convertToFb2(inputFiles.nesting, outputFile, opts, callback); } if (!result) { if (selectedFileType) throw new Error(`Этот формат файла не поддерживается: ${selectedFileType.mime}`); else { - throw new Error(`Не удалось определить формат файла: ${url}`); + throw new Error(`Не удалось определить формат файла: ${opts.url}`); } } diff --git a/server/core/ReaderWorker.js b/server/core/ReaderWorker.js index 1a2868b2..24038693 100644 --- a/server/core/ReaderWorker.js +++ b/server/core/ReaderWorker.js @@ -32,7 +32,8 @@ class ReaderWorker { } } - async loadBook(url, wState) { + async loadBook(opts, wState) { + const url = opts.url; let errMes = ''; let decompDir = ''; let downloadedFilename = ''; @@ -77,7 +78,7 @@ class ReaderWorker { //конвертирование в fb2 wState.set({state: 'convert', step: 3, progress: 0}); convertFilename = `${this.config.tempDownloadDir}/${tempFilename2}`; - await this.bookConverter.convertToFb2(decompFiles, convertFilename, url, progress => { + await this.bookConverter.convertToFb2(decompFiles, convertFilename, opts, progress => { wState.set({progress}); }); @@ -105,12 +106,12 @@ class ReaderWorker { } } - loadBookUrl(url) { + loadBookUrl(opts) { const workerId = workerState.generateWorkerId(); const wState = workerState.getControl(workerId); wState.set({state: 'start'}); - this.loadBook(url, wState); + this.loadBook(opts, wState); return workerId; } From 6a7cbc70d6dd012ee282cfa69e1f746a5017c2e6 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Oct 2019 19:30:21 +0700 Subject: [PATCH 9/9] 0.7.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ef406575..5b6bbe10 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.7.2", + "version": "0.7.3", "engines": { "node": ">=10.0.0" },