Merge branch 'release/0.12.0-2'

0.12.0-1

build/linux.js

@@ -23,24 +23,6 @@ async function main() {
 
     await fs.ensureDir(tempDownloadDir);
 
-    //sqlite3
-    const sqliteRemoteUrl = 'https://mapbox-node-binary.s3.amazonaws.com/sqlite3/v5.0.2/napi-v3-linux-x64.tar.gz';
-    const sqliteDecompressedFilename = `${tempDownloadDir}/napi-v3-linux-x64/node_sqlite3.node`;
-
-    if (!await fs.pathExists(sqliteDecompressedFilename)) {
-        // Скачиваем node_sqlite3.node для винды, т.к. pkg не включает его в сборку
-        const res = await axios.get(sqliteRemoteUrl, {responseType: 'stream'})
-        await pipeline(res.data, fs.createWriteStream(`${tempDownloadDir}/sqlite.tar.gz`));
-        console.log(`done downloading ${sqliteRemoteUrl}`);
-
-        //распаковываем
-        console.log(await decomp.unpackTarZZ(`${tempDownloadDir}/sqlite.tar.gz`, tempDownloadDir));
-        console.log('files decompressed');
-    }
-    // копируем в дистрибутив
-    await fs.copy(sqliteDecompressedFilename, `${outDir}/node_sqlite3.node`);
-    console.log(`copied ${sqliteDecompressedFilename} to ${outDir}/node_sqlite3.node`);
-
     //ipfs
     const ipfsDecompressedFilename = `${tempDownloadDir}/go-ipfs/ipfs`;
     if (!await fs.pathExists(ipfsDecompressedFilename)) {

build/win.js

@@ -23,24 +23,6 @@ async function main() {
 
     await fs.ensureDir(tempDownloadDir);
 
-    //sqlite3
-    const sqliteRemoteUrl = 'https://mapbox-node-binary.s3.amazonaws.com/sqlite3/v5.0.2/napi-v3-win32-x64.tar.gz';
-    const sqliteDecompressedFilename = `${tempDownloadDir}/napi-v3-win32-x64/node_sqlite3.node`;
-
-    if (!await fs.pathExists(sqliteDecompressedFilename)) {
-        // Скачиваем node_sqlite3.node для винды, т.к. pkg не включает его в сборку
-        const res = await axios.get(sqliteRemoteUrl, {responseType: 'stream'})
-        await pipeline(res.data, fs.createWriteStream(`${tempDownloadDir}/sqlite.tar.gz`));
-        console.log(`done downloading ${sqliteRemoteUrl}`);
-
-        //распаковываем
-        console.log(await decomp.unpackTarZZ(`${tempDownloadDir}/sqlite.tar.gz`, tempDownloadDir));
-        console.log('files decompressed');
-    }
-    // копируем в дистрибутив
-    await fs.copy(sqliteDecompressedFilename, `${outDir}/node_sqlite3.node`);
-    console.log(`copied ${sqliteDecompressedFilename} to ${outDir}/node_sqlite3.node`);
-
     //ipfs
     const ipfsDecompressedFilename = `${tempDownloadDir}/go-ipfs/ipfs.exe`;
     if (!await fs.pathExists(ipfsDecompressedFilename)) {

client/api/misc.js

@@ -9,7 +9,7 @@ class Misc {
     async loadConfig() {
 
         const query = {params: [
-            'name', 'version', 'mode', 'maxUploadFileSize', 'useExternalBookConverter', 'acceptFileExt', 'branch',
+            'name', 'version', 'mode', 'maxUploadFileSize', 'useExternalBookConverter', 'acceptFileExt', 'bucEnabled', 'branch',
         ]};
 
         try {

client/api/reader.js

@@ -229,6 +229,17 @@ class Reader {
         return (await axios.get(url)).data;
     }
 
+    async checkBuc(bookUrls) {
+        const response = await wsc.message(await wsc.send({action: 'check-buc', bookUrls}));
+
+        if (response.error)
+            throw new Error(response.error);
+
+        if (!response.data)
+            throw new Error(`response.data is empty`);
+
+        return response.data;
+    }
 }
 
 export default new Reader();

client/components/Reader/Reader.vue

@@ -100,6 +100,12 @@
                         </q-tooltip>
                     </button>
                     <button v-show="showToolButton['recentBooks']" ref="recentBooks" v-ripple class="tool-button" :class="buttonActiveClass('recentBooks')" @click="buttonClick('recentBooks')">
+                        <div v-show="bothBucEnabled && needBookUpdateCount > 0" style="position: absolute">
+                            <div class="need-book-update-count">
+                                {{ needBookUpdateCount }}
+                            </div>
+                        </div>
+
                         <q-icon name="la la-book-open" size="32px" />
                         <q-tooltip :delay="1500" anchor="bottom middle" content-style="font-size: 80%">
                             {{ rstore.readerActions['recentBooks'] }}
@@ -156,7 +162,7 @@
             ></SearchPage>
             <CopyTextPage v-if="copyTextActive" ref="copyTextPage" @do-action="doAction"></CopyTextPage>
             <LibsPage v-show="hidden" ref="libsPage" @load-book="loadBook" @libs-close="libsClose" @do-action="doAction"></LibsPage>
-            <RecentBooksPage v-show="recentBooksActive" ref="recentBooksPage" @load-book="loadBook" @recent-books-close="recentBooksClose"></RecentBooksPage>
+            <RecentBooksPage v-show="recentBooksActive" ref="recentBooksPage" @load-book="loadBook" @recent-books-close="recentBooksClose" @update-count-changed="updateCountChanged"></RecentBooksPage>
             <SettingsPage v-show="settingsActive" ref="settingsPage" @do-action="doAction"></SettingsPage>
             <HelpPage v-if="helpActive" ref="helpPage" @do-action="doAction"></HelpPage>
             <ClickMapPage v-show="clickMapActive" ref="clickMapPage"></ClickMapPage>
@@ -309,6 +315,10 @@ class Reader {
     donationVisible = false;
     dualPageMode = false;
 
+    bucEnabled = false;
+    bucSetOnNew = false;
+    needBookUpdateCount = 0;
+
     created() {
         this.rstore = rstore;
         this.loading = true;
@@ -357,6 +367,32 @@ class Reader {
             }
         }, 200);
 
+        this.debouncedRecentBooksPageUpdate = _.debounce(async() => {
+            if (this.recentBooksActive) {
+                await this.$refs.recentBooksPage.updateTableData();
+            }
+        }, 100);
+
+        this.recentItemKeys = [];
+        this.debouncedSaveRecent = _.debounce(async() => {
+            let timer = setTimeout(() => {
+                if (!this.offlineModeActive)
+                    this.$root.notify.error('Таймаут соединения');
+            }, 10000);
+
+            try {
+                const itemKeys = this.recentItemKeys;
+                this.recentItemKeys = [];
+                //сохранение в удаленном хранилище
+                await this.$refs.serverStorage.saveRecent(itemKeys);
+            } catch (e) {
+                if (!this.offlineModeActive)
+                    this.$root.notify.error(e.message);
+            } finally {
+                clearTimeout(timer);
+            }
+        }, 500, {maxWait: 1000});
+
         document.addEventListener('fullscreenchange', () => {
             this.fullScreenActive = (document.fullscreenElement !== null);
         });
@@ -394,16 +430,30 @@ class Reader {
             this.updateRoute();
 
             await this.$refs.dialogs.init();
+
+            this.$refs.recentBooksPage.init();
         })();
 
+        //проверки обновлений читалки
         (async() => {
             this.isFirstNeedUpdateNotify = true;
             //вечный цикл, запрашиваем периодически конфиг для проверки выхода новой версии читалки
-            while (true) {// eslint-disable-line no-constant-condition
+            while (1) {// eslint-disable-line no-constant-condition
                 await this.checkNewVersionAvailable();
-                await utils.sleep(3600*1000); //каждый час
+                await utils.sleep(60*60*1000); //каждый час
             }
-            //дальше кода нет
+            //дальше хода нет
+        })();
+
+        //проверки обновлений книг
+        (async() => {
+            await utils.sleep(15*1000); //подождем неск. секунд перед первым запросом
+            //вечный цикл, запрашиваем периодически обновления
+            while (1) {// eslint-disable-line no-constant-condition
+                await this.checkBuc();
+                await utils.sleep(70*60*1000); //каждые 70 минут
+            }
+            //дальше хода нет
         })();
     }
 
@@ -425,6 +475,11 @@ class Reader {
         this.pdfQuality = settings.pdfQuality;
         this.dualPageMode = settings.dualPageMode;
         this.userWallpapers = settings.userWallpapers;
+        this.bucEnabled = settings.bucEnabled;
+        this.bucSizeDiff = settings.bucSizeDiff;
+        this.bucSetOnNew = settings.bucSetOnNew;
+        this.bucCancelEnabled = settings.bucCancelEnabled;
+        this.bucCancelDays = settings.bucCancelDays;
 
         this.readerActionByKeyCode = utils.userHotKeysObjectSwap(settings.userHotKeys);
         this.$root.readerActionByKeyEvent = (event) => {
@@ -522,6 +577,92 @@ class Reader {
         }
     }
 
+    async checkBuc() {
+        if (!this.bothBucEnabled)
+            return;
+
+        try {
+            const sorted = bookManager.getSortedRecent();
+
+            //выберем все кандидиаты на обновление
+            const updateUrls = new Set();
+            for (const book of sorted) {
+                if (!book.deleted && book.checkBuc && book.url && book.url.indexOf('disk://') !== 0)
+                    updateUrls.add(book.url);
+            }
+
+            //теперь по кусочкам запросим сервер
+            const arr = Array.from(updateUrls);
+            const bucSize = {};
+            const chunkSize = 100;
+            for (let i = 0; i < arr.length; i += chunkSize) {
+                const chunk = arr.slice(i, i + chunkSize);
+
+                const data = await readerApi.checkBuc(chunk);
+
+                for (const item of data) {
+                    bucSize[item.id] = item.size;
+                }
+
+                await utils.sleep(1000);//чтобы не ддосить сервер
+            }
+
+            const checkSetTime = {};
+            //проставим новые размеры у книг
+            for (const book of sorted) {
+                if (book.deleted)
+                    continue;
+                
+                //размер 0 считаем отсутствующим
+                if (book.url && bucSize[book.url] && bucSize[book.url] !== book.bucSize) {
+                    book.bucSize = bucSize[book.url];
+                    await bookManager.recentSetItem(book);
+                }
+
+                //подготовка к следующему шагу, ищем книгу по url с максимальной датой установки checkBucTime/loadTime
+                //от этой даты будем потом отсчитывать bucCancelDays
+                if (updateUrls.has(book.url)) {
+                    let rec = checkSetTime[book.url] || {time: 0, loadTime: 0};
+
+                    const time = (book.checkBucTime ? book.checkBucTime : (rec.loadTime || 0));
+                    if (time > rec.time || (time == rec.time && (book.loadTime > rec.loadTime)))
+                        rec = {time, loadTime: book.loadTime, key: book.key};
+
+                    checkSetTime[book.url] = rec;
+                }
+            }
+
+            //bucCancelEnabled и bucCancelDays
+            //снимем флаг checkBuc у необновлявшихся bucCancelDays
+            if (this.bucCancelEnabled) {
+                for (const rec of Object.values(checkSetTime)) {
+                    if (rec.time && Date.now() - rec.time > this.bucCancelDays*24*3600*1000) {
+                        const book = await bookManager.getRecentBook({key: rec.key});
+                        const needBookUpdate = 
+                            book.checkBuc
+                            && book.bucSize
+                            && utils.hasProp(book, 'downloadSize')
+                            && book.bucSize !== book.downloadSize
+                            && (book.bucSize - book.downloadSize >= this.bucSizeDiff)
+                        ;
+
+                        if (book && !needBookUpdate) {
+                            await bookManager.setCheckBuc(book, undefined);//!!!
+                        }
+                    }
+                }
+            }
+
+            await this.$refs.recentBooksPage.updateTableData();
+        } catch (e) {
+            console.error(e);
+        }
+    }
+
+    updateCountChanged(event) {
+        this.needBookUpdateCount = event.needBookUpdateCount;
+    }
+
     checkSetStorageAccessKey() {
         const q = this.$route.query;
 
@@ -600,6 +741,10 @@ class Reader {
         return versionHistory[0].version;
     }
 
+    get bothBucEnabled() {
+        return this.$store.state.config.bucEnabled && this.bucEnabled;
+    }
+
     get routeParamUrl() {
         let result = '';
         const path = this.$route.fullPath;
@@ -648,27 +793,12 @@ class Reader {
         }
 
         if (eventName == 'recent-changed') {
-            if (this.recentBooksActive) {
-                await this.$refs.recentBooksPage.updateTableData();
-            }
+            this.debouncedRecentBooksPageUpdate();
 
             //сохранение в serverStorage
-            if (value) {
-                await utils.sleep(500);
-                
-                let timer = setTimeout(() => {
-                    if (!this.offlineModeActive)
-                        this.$root.notify.error('Таймаут соединения');
-                }, 10000);
-
-                try {
-                    await this.$refs.serverStorage.saveRecent(value);
-                } catch (e) {
-                    if (!this.offlineModeActive)
-                        this.$root.notify.error(e.message);
-                } finally {
-                    clearTimeout(timer);
-                }
+            if (value && this.recentItemKeys.indexOf(value) < 0) {
+                this.recentItemKeys.push(value);
+                this.debouncedSaveRecent();
             }
         }
     }
@@ -1161,6 +1291,7 @@ class Reader {
 
                     this.checkBookPosPercent();
                     this.activateClickMapPage();//no await
+                    this.$refs.recentBooksPage.updateTableData();//no await
                     return;
                 }
 
@@ -1237,9 +1368,13 @@ class Reader {
                 delete wasOpened.loadTime;
 
             // добавляем в историю
-            await bookManager.setRecentBook(Object.assign(wasOpened, addedBook));
+            const recentBook = await bookManager.setRecentBook(Object.assign(wasOpened, addedBook));
+            if (this.bucSetOnNew) {
+                await bookManager.setCheckBuc(recentBook, true);
+            }
+
             this.mostRecentBook();
-            this.addAction(wasOpened.bookPos);
+            this.addAction(recentBook.bookPos);
             this.updateRoute(true);
 
             this.loaderActive = false;
@@ -1251,6 +1386,7 @@ class Reader {
 
             this.checkBookPosPercent();
             this.activateClickMapPage();//no await
+            this.$refs.recentBooksPage.updateTableData();//no await
         } catch (e) {
             progress.hide(); this.progressActive = false;
             this.loaderActive = true;
@@ -1601,4 +1737,16 @@ export default vueComponent(Reader);
 .clear {
     color: rgba(0,0,0,0);
 }
+
+.need-book-update-count {
+    position: relative;
+    padding: 2px 6px 2px 6px;
+    left: 27px;
+    top: 22px;
+    background-color: blue;
+    border-radius: 10px;
+    color: white;
+    z-index: 10;
+    font-size: 80%;
+}
 </style>

client/components/Reader/RecentBooksPage/RecentBooksPage.vue

@@ -9,14 +9,26 @@
 
         <template #buttons>
             <div
+                v-show="needBookUpdateCount > 0"
                 class="row justify-center items-center"
-                :class="{'header-button': !archive, 'header-button-pressed': archive}" 
-                @mousedown.stop @click="archiveToggle"
+                :class="{'header-button-update': !showNeedBookUpdateOnly, 'header-button-update-pressed': showNeedBookUpdateOnly}" 
+                @mousedown.stop @click="showNeedBookUpdateOnlyToggle"
+            >
+                <span style="font-size: 90%">{{ needBookUpdateCount }} обновлен{{ wordEnding(needBookUpdateCount, 3) }}</span>
+                <q-tooltip :delay="1500" anchor="bottom middle" content-style="font-size: 80%">
+                    {{ (needBookUpdateCount ? 'Скрыть обновления' : 'Показать обновления') }}
+                </q-tooltip>
+            </div>
+
+            <div
+                class="row justify-center items-center"
+                :class="{'header-button': !showArchive, 'header-button-pressed': showArchive}" 
+                @mousedown.stop @click="showArchiveToggle"
             >
                 <q-icon class="q-mr-xs" name="la la-archive" size="20px" />
                 <span style="font-size: 90%">Архив</span>
                 <q-tooltip :delay="1500" anchor="bottom middle" content-style="font-size: 80%">
-                    {{ (archive ? 'Скрыть архивные' : 'Показать архивные') }}
+                    {{ (showArchive ? 'Скрыть архивные' : 'Показать архивные') }}
                 </q-tooltip>
             </div>
         </template>
@@ -105,9 +117,17 @@
                     </div>
 
                     <div class="row-part column justify-center items-stretch" style="width: 80px">
-                        <div class="col row justify-center items-center clickable" style="padding: 0 2px 0 2px" @click="loadBook(item)">
+                        <div class="col row justify-center items-center clickable" style="padding: 0 2px 0 2px" @click="loadBook(item, bothBucEnabled && item.needBookUpdate)">
                             <div v-show="isLoadedCover(item.coverPageUrl)" style="height: 80px" v-html="getCoverHtml(item.coverPageUrl)" />
                             <q-icon v-show="!isLoadedCover(item.coverPageUrl)" name="la la-book" size="40px" style="color: #dddddd" />
+                        
+                            <div
+                                v-show="bothBucEnabled && item.needBookUpdate"
+                                class="column justify-center"
+                                style="position: absolute; background-color: rgba(255, 255, 255, 0.5); border-radius: 40px;"
+                            >
+                                <q-icon name="la la-sync" size="60px" style="color: blue" />
+                            </div>
                         </div>
 
                         <div v-show="!showSameBook && item.group && item.group.length > 0" class="row justify-center" style="font-size: 70%">
@@ -126,6 +146,10 @@
                             <div style="font-size: 75%">
                                 {{ item.desc.title }}
                             </div>
+                            <div v-show="bothBucEnabled && item.needBookUpdate" style="font-size: 75%; color: blue;">
+                                Размер: {{ item.bucSize - item.downloadSize > 0 ? '+' : '' }}{{ item.bucSize - item.downloadSize }}
+                                ({{ item.downloadSize }} &rarr; {{ item.bucSize }})
+                            </div>
                         </div>
 
                         <div class="row" style="font-size: 10px">
@@ -169,7 +193,7 @@
                             class="col column justify-center" 
                             style="font-size: 75%; padding-left: 6px; border: 1px solid #cccccc; border-left: 0;"
                         >
-                            <div :style="`margin-top: ${(archive ? 20 : 0)}px`">
+                            <div style="margin: 25px 0 0 5px">
                                 <a v-show="isUrl(item.url)" :href="item.url" target="_blank">Оригинал</a><br><br>
                                 <a :href="item.path" @click.prevent="downloadBook(item.path, item.fullTitle)">Скачать FB2</a>
                             </div>
@@ -181,12 +205,12 @@
                         >
                             <q-icon class="la la-times" size="12px" />
                             <q-tooltip :delay="1500" anchor="bottom middle" content-style="font-size: 80%">
-                                {{ (archive ? 'Удалить окончательно' : 'Перенести в архив') }}
+                                {{ (showArchive ? 'Удалить окончательно' : 'Перенести в архив') }}
                             </q-tooltip>
                         </div>
 
                         <div
-                            v-show="archive"
+                            v-show="showArchive"
                             class="restore-button self-start row justify-center items-center clickable"
                             @click="handleRestore(item.key)"
                         >
@@ -195,6 +219,27 @@
                                 Восстановить из архива
                             </q-tooltip>
                         </div>
+
+                        <div
+                            v-show="bothBucEnabled && item.showCheckBuc"
+                            class="buc-checkbox self-start"
+                        >
+                            <q-checkbox
+                                v-model="item.checkBuc"
+                                size="xs"
+                                style="position: relative; top: -3px; left: -3px;"
+                                @update:model-value="checkBucChange(item)"
+                            >
+                                <q-tooltip :delay="1500" anchor="bottom middle" content-style="font-size: 80%">
+                                    <div v-if="item.checkBuc === undefined">
+                                        Проверка обновлений отключена автоматически<br>т.к. книга не обновлялась {{ bucCancelDays }} дней
+                                    </div>
+                                    <div v-else>
+                                        {{ (item.checkBuc ? 'Проверка обновлений книги включена' : 'Проверка обновлений книги отключена') }}
+                                    </div>
+                                </q-tooltip>
+                            </q-checkbox>
+                        </div>
                     </div>
                 </div>
             </q-virtual-scroll>
@@ -230,6 +275,12 @@ const componentOptions = {
         settings() {
             this.loadSettings();
         },
+        needBookUpdateCount() {
+            if (this.needBookUpdateCount == 0)
+                this.showNeedBookUpdateOnly = false;
+
+            this.$emit('update-count-changed', {needBookUpdateCount: this.needBookUpdateCount});
+        }
     },
 };
 class RecentBooksPage {
@@ -240,7 +291,14 @@ class RecentBooksPage {
     tableData = [];
     sortMethod = '';
     showSameBook = false;
-    archive = false;
+    bucEnabled = false;
+    bucSizeDiff = 0;
+    bucSetOnNew = false;
+    bucCancelDays = 0;
+    needBookUpdateCount = 0;
+
+    showArchive = false;
+    showNeedBookUpdateOnly = false;
 
     covers = {};
     coversLoadFunc = {};
@@ -277,12 +335,20 @@ class RecentBooksPage {
         const settings = this.settings;
         this.showSameBook = settings.recentShowSameBook;
         this.sortMethod = settings.recentSortMethod || 'loadTimeDesc';
+        this.bucEnabled = settings.bucEnabled;
+        this.bucSizeDiff = settings.bucSizeDiff;
+        this.bucSetOnNew = settings.bucSetOnNew;
+        this.bucCancelDays = settings.bucCancelDays;
     }
 
     get settings() {
         return this.$store.state.reader.settings;
     }
 
+    get bothBucEnabled() {
+        return this.$store.state.config.bucEnabled && this.bucEnabled;
+    }
+
     async updateTableData() {
         if (!this.inited)
             return;
@@ -296,7 +362,7 @@ class RecentBooksPage {
 
             //подготовка полей
             for (const book of sorted) {
-                if ((!this.archive && book.deleted) || (this.archive && book.deleted != 1))
+                if ((!this.showArchive && book.deleted) || (this.showArchive && book.deleted != 1))
                     continue;
 
                 let d = new Date();
@@ -320,7 +386,7 @@ class RecentBooksPage {
 
                 let title = bt.bookTitle;
                 title = (title ? `"${title}"`: '');
-                const author = (bt.author ? bt.author : (bt.bookTitle ? bt.bookTitle : (book.uploadFileName ? book.uploadFileName : book.url)));
+                const author = (bt.author ? bt.author : (bt.bookTitle ? bt.bookTitle : (book.uploadFileName ? book.uploadFileName : book.url))) || '';
 
                 result.push({
                     key: book.key,
@@ -344,6 +410,19 @@ class RecentBooksPage {
                     inGroup: false,
                     coverPageUrl: book.coverPageUrl,
 
+                    showCheckBuc: !this.showArchive && utils.hasProp(book, 'downloadSize') && book.url.indexOf('disk://') !== 0,
+                    checkBuc: book.checkBuc,
+                    needBookUpdate: (
+                        !this.showArchive
+                        && book.checkBuc
+                        && book.bucSize
+                        && utils.hasProp(book, 'downloadSize')
+                        && book.bucSize !== book.downloadSize
+                        && (book.bucSize - book.downloadSize >= this.bucSizeDiff)
+                    ),
+                    bucSize: book.bucSize,
+                    downloadSize: book.downloadSize,
+
                     //для сортировки
                     loadTimeRaw,
                     touchTimeRaw: book.touchTime,
@@ -361,12 +440,15 @@ class RecentBooksPage {
             //фильтрация
             const search = this.search;
             if (search) {
+                const lowerSearch = search.toLowerCase();
+
                 result = result.filter(item => {
-                    return !search ||
-                        item.touchTime.includes(search) ||
-                        item.loadTime.includes(search) ||
-                        item.desc.title.toLowerCase().includes(search.toLowerCase()) ||
-                        item.desc.author.toLowerCase().includes(search.toLowerCase())
+                    return !search
+                        || item.touchTime.includes(search)
+                        || item.loadTime.includes(search)
+                        || item.desc.title.toLowerCase().includes(lowerSearch)
+                        || item.desc.author.toLowerCase().includes(lowerSearch)
+                    ;
                 });
             }
 
@@ -399,6 +481,7 @@ class RecentBooksPage {
             }
 
             //группировка
+            let nbuCount = 0;
             const groups = {};
             const parents = {};
             let newResult = [];
@@ -415,13 +498,20 @@ class RecentBooksPage {
                         if (book.active)
                             parents[book.sameBookKey].activeParent = true;
 
+                        book.showCheckBuc = false;
+                        book.needBookUpdate = false;
+
                         groups[book.sameBookKey].push(book);
                     }
                 } else {
                     newResult.push(book);
                 }
+
+                if (book.needBookUpdate)
+                    nbuCount++;
             }
             result = newResult;
+            this.needBookUpdateCount = nbuCount;
 
             //showSameBook
             if (this.showSameBook) {
@@ -438,6 +528,11 @@ class RecentBooksPage {
                 result = newResult;
             }
 
+            //showNeedBookUpdateOnly
+            if (this.showNeedBookUpdateOnly) {
+                result = result.filter(item => item.needBookUpdate);
+            }
+
             //другие стадии
             //.....
 
@@ -456,7 +551,8 @@ class RecentBooksPage {
         const endings = [
             ['ов', '', 'а', 'а', 'а', 'ов', 'ов', 'ов', 'ов', 'ов'],
             ['й', 'я', 'и', 'и', 'и', 'й', 'й', 'й', 'й', 'й'],
-            ['о', '', 'о', 'о', 'о', 'о', 'о', 'о', 'о', 'о']
+            ['о', '', 'о', 'о', 'о', 'о', 'о', 'о', 'о', 'о'],
+            ['ий', 'ие', 'ия', 'ия', 'ия', 'ий', 'ий', 'ий', 'ий', 'ий']
         ];
         const deci = num % 100;
         if (deci > 10 && deci < 20) {
@@ -468,7 +564,7 @@ class RecentBooksPage {
 
     get header() {
         const len = (this.tableData ? this.tableData.length : 0);
-        return `${(this.search ? `Найден${this.wordEnding(len, 2)}` : 'Всего')} ${len} файл${this.wordEnding(len)}${this.archive ? ' в архиве' : ''}`;
+        return `${(this.search || this.showNeedBookUpdateOnly ? `Найден${this.wordEnding(len, 2)}` : 'Всего')} ${len} файл${this.wordEnding(len)}${this.showArchive ? ' в архиве' : ''}`;
     }
 
     async downloadBook(fb2path, fullTitle) {
@@ -494,7 +590,7 @@ class RecentBooksPage {
     }
 
     async handleDel(key) {
-        if (!this.archive) {
+        if (!this.showArchive) {
             await bookManager.delRecentBook({key});
             this.$root.notify.info('Перенесено в архив');
         } else {
@@ -510,14 +606,11 @@ class RecentBooksPage {
         this.$root.notify.info('Восстановлено из архива');
     }
 
-    async loadBook(item) {
-        //чтобы не обновлять лишний раз updateTableData
-        this.inited = false;
-
+    async loadBook(item, force = false) {
         if (item.deleted)
             await this.handleRestore(item.key);
 
-        this.$emit('load-book', {url: item.url, path: item.path});
+        this.$emit('load-book', {url: item.url, path: item.path, force});
         this.close();
     }
 
@@ -645,8 +738,10 @@ class RecentBooksPage {
         ];
     }
 
-    archiveToggle() {
-        this.archive = !this.archive;
+    showArchiveToggle() {
+        this.showArchive = !this.showArchive;
+        this.showNeedBookUpdateOnly = false;
+
         this.updateTableData();
     }
 
@@ -713,6 +808,21 @@ class RecentBooksPage {
         else
             return '';
     }
+
+    async checkBucChange(item) {
+        const book = await bookManager.getRecentBook(item);
+        if (book) {
+            await bookManager.setCheckBuc(book, item.checkBuc);
+        }
+    }
+
+    showNeedBookUpdateOnlyToggle() {
+        this.showNeedBookUpdateOnly = !this.showNeedBookUpdateOnly;
+        this.showArchive = false;
+
+        this.updateTableData();
+    }
+
 }
 
 export default vueComponent(RecentBooksPage);
@@ -842,17 +952,24 @@ export default vueComponent(RecentBooksPage);
     color: #555555;
 }
 
-.header-button:hover {
+.header-button-update, .header-button-update-pressed {
+    width: 120px;
+    height: 30px;
+    cursor: pointer;
+    color: white;
+}
+
+.header-button:hover, .header-button-update:hover {
     color: white;
     background-color: #39902F;
 }
 
-.header-button-pressed {
+.header-button-pressed, .header-button-update-pressed {
     color: black;
     background-color: yellow;
 }
 
-.header-button-pressed:hover {
-    color: black;
+.buc-checkbox {
+    position: absolute;
 }
 </style>

client/components/Reader/ServerStorage/ServerStorage.vue

@@ -12,6 +12,7 @@ import bookManager from '../share/bookManager';
 import readerApi from '../../../api/reader';
 import * as utils from '../../../share/utils';
 import * as cryptoUtils from '../../../share/cryptoUtils';
+import LockQueue from '../../../share/LockQueue';
 
 import localForage from 'localforage';
 const ssCacheStore = localForage.createInstance({
@@ -48,6 +49,8 @@ class ServerStorage {
         this.keyInited = false;
         this.commit = this.$store.commit;
         this.prevServerStorageKey = null;
+        this.lock = new LockQueue(100);
+
         this.$root.generateNewServerStorageKey = () => {this.generateNewServerStorageKey()};
 
         this.debouncedSaveSettings = _.debounce(() => {
@@ -542,14 +545,16 @@ class ServerStorage {
         return true;
     }
 
-    async saveRecent(itemKey, recurse) {
-        while (!this.inited || this.savingRecent)
+    async saveRecent(itemKeys, recurse) {
+        while (!this.inited)
             await utils.sleep(100);
 
-        if (!this.keyInited || !this.serverSyncEnabled || this.savingRecent)
+        if (!this.keyInited || !this.serverSyncEnabled)
             return;
 
-        this.savingRecent = true;
+        let needRecurseCall = false;
+
+        await this.lock.get();
         try {        
             const bm = bookManager;
 
@@ -559,22 +564,29 @@ class ServerStorage {
 
             //newRecentMod
             let newRecentMod = {};
-            if (itemKey && this.cachedRecentPatch.data[itemKey] && this.prevItemKey == itemKey) {
+            let oneItemKey = null;
+            if (itemKeys && itemKeys.length == 1)
+                oneItemKey = itemKeys[0];
+
+            if (oneItemKey && this.cachedRecentPatch.data[oneItemKey] && this.prevItemKey == oneItemKey) {
                 newRecentMod = _.cloneDeep(this.cachedRecentMod);
                 newRecentMod.rev++;
 
-                newRecentMod.data.key = itemKey;
-                newRecentMod.data.mod = utils.getObjDiff(this.cachedRecentPatch.data[itemKey], bm.recent[itemKey]);
+                newRecentMod.data.key = oneItemKey;
+                newRecentMod.data.mod = utils.getObjDiff(this.cachedRecentPatch.data[oneItemKey], bm.recent[oneItemKey]);
                 needSaveRecentMod = true;
             }
-            this.prevItemKey = itemKey;
+            this.prevItemKey = oneItemKey;
 
             //newRecentPatch
             let newRecentPatch = {};
-            if (itemKey && !needSaveRecentMod) {
+            if (itemKeys && !needSaveRecentMod) {
                 newRecentPatch = _.cloneDeep(this.cachedRecentPatch);
                 newRecentPatch.rev++;
-                newRecentPatch.data[itemKey] = _.cloneDeep(bm.recent[itemKey]);
+
+                for (const key of itemKeys) {
+                    newRecentPatch.data[key] = _.cloneDeep(bm.recent[key]);
+                }
 
                 const applyMod = this.cachedRecentMod.data;
                 if (applyMod && applyMod.key && newRecentPatch.data[applyMod.key])
@@ -587,11 +599,7 @@ class ServerStorage {
 
             //newRecent
             let newRecent = {};
-            if (!itemKey || (needSaveRecentPatch && Object.keys(newRecentPatch.data).length > 10)) {
-                //ждем весь bm.recent
-                /*while (!bookManager.loaded)
-                    await utils.sleep(100);*/
-
+            if (!itemKeys || (needSaveRecentPatch && Object.keys(newRecentPatch.data).length > 10)) {
                 newRecent = {rev: this.cachedRecent.rev + 1, data: _.cloneDeep(bm.recent)};
                 newRecentPatch = {rev: this.cachedRecentPatch.rev + 1, data: {}};
                 newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}};
@@ -625,10 +633,8 @@ class ServerStorage {
 
                 if (res)
                     this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`);
-                if (!recurse && itemKey) {
-                    this.savingRecent = false;
-                    await this.saveRecent(itemKey, true);
-                    return;
+                if (!recurse && itemKeys) {
+                    needRecurseCall = true;
                 }
             } else if (result.state == 'success') {
                 if (needSaveRecent && newRecent.rev)
@@ -639,8 +645,11 @@ class ServerStorage {
                     await this.setCachedRecentMod(newRecentMod);
             }
         } finally {
-            this.savingRecent = false;
+            this.lock.ret();
         }
+
+        if (needRecurseCall)
+            await this.saveRecent(itemKeys, true);
     }
 
     async storageCheck(items) {

client/components/Reader/SettingsPage/OthersTab.inc

@@ -41,17 +41,6 @@
     </q-checkbox>
 </div>
 
-<div class="item row">
-    <div class="label-6">Уведомление</div>
-    <q-checkbox size="xs" v-model="showNeedUpdateNotify">
-        Показывать уведомление о новой версии
-        <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
-            Напоминать о необходимости обновления страницы<br>
-            при появлении новой версии читалки
-        </q-tooltip>
-    </q-checkbox>
-</div>
-
 <!--div class="item row">
     <div class="label-6">Уведомление</div>
     <q-checkbox size="xs" v-model="showDonationDialog2020">

client/components/Reader/SettingsPage/SettingsPage.vue

@@ -30,6 +30,7 @@
                     <q-tab class="tab" name="keys" icon="la la-gamepad" label="Управление" />
                     <q-tab class="tab" name="pagemove" icon="la la-school" label="Листание" />
                     <q-tab class="tab" name="convert" icon="la la-magic" label="Конвертир." />
+                    <q-tab class="tab" name="update" icon="la la-sync" label="Обновление" />
                     <q-tab class="tab" name="others" icon="la la-list-ul" label="Прочее" />
                     <q-tab class="tab" name="reset" icon="la la-broom" label="Сброс" />
                     <div v-show="tabsScrollable" class="q-pt-lg" />
@@ -99,6 +100,10 @@
                 <div v-if="selectedTab == 'convert'" class="fit tab-panel">
                     @@include('./ConvertTab.inc');
                 </div>
+                <!-- Обновление ------------------------------------------------------------------>
+                <div v-if="selectedTab == 'update'" class="fit tab-panel">
+                    @@include('./UpdateTab.inc');
+                </div>
                 <!-- Прочее ---------------------------------------------------------------------->
                 <div v-if="selectedTab == 'others'" class="fit tab-panel">
                     @@include('./OthersTab.inc');
@@ -313,6 +318,10 @@ class SettingsPage {
         return this.$store.state.reader.profiles;
     }
 
+    get configBucEnabled() {
+        return this.$store.state.config.bucEnabled;
+    }
+
     get currentProfileOptions() {
         const profNames = Object.keys(this.profiles)
         profNames.sort();

client/components/Reader/SettingsPage/UpdateTab.inc

@@ -0,0 +1,76 @@
+<!---------------------------------------------->
+<div class="part-header">Обновление читалки</div>
+<div class="item row">
+    <div class="label-6"></div>
+    <q-checkbox size="xs" v-model="showNeedUpdateNotify">
+        Проверять наличие новой версии
+        <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+            Напоминать о необходимости обновления страницы<br>
+            при появлении новой версии читалки
+        </q-tooltip>
+    </q-checkbox>
+</div>
+
+<!---------------------------------------------->
+<div class="part-header">Обновление книг</div>
+<div v-show="!configBucEnabled" class="item row">
+    <div class="label-6"></div>
+    <div>Сервер обновлений временно не работает</div>
+</div>
+
+<div v-show="configBucEnabled" class="item row">
+    <div class="label-6"></div>
+    <q-checkbox size="xs" v-model="bucEnabled">
+        Проверять обновления книг
+    </q-checkbox>
+</div>
+
+<div v-show="configBucEnabled && bucEnabled" class="item row">
+    <div class="label-6"></div>
+    <div class="col-5 column justify-center items-end q-pr-xs">Разница размеров</div>
+    <div class="col row">
+        <NumInput class="col-left" v-model="bucSizeDiff" />
+
+        <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+            Уведомлять о наличии обновления книги в списке загруженных<br>
+            при указанной разнице в размерах старого и нового файлов.<br>
+            Разница указывается в байтах и может быть отрицательной.
+        </q-tooltip>
+    </div>
+</div>
+
+<div v-show="configBucEnabled && bucEnabled" class="item row">
+    <div class="label-6"></div>
+    <q-checkbox size="xs" v-model="bucSetOnNew">
+        Автопроверка для вновь загружаемых
+        <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+            Автоматически устанавливать флаг проверки<br>
+            обновлений для всех вновь загружаемых книг
+        </q-tooltip>
+    </q-checkbox>
+</div>
+
+<div v-show="configBucEnabled && bucEnabled" class="item row">
+    <div class="label-6"></div>
+    <q-checkbox size="xs" v-model="bucCancelEnabled">
+        Отменять проверку через {{ bucCancelDays }} дней{{ (bucCancelEnabled ? ':' : '') }}
+        <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+            Снимать флаг проверки с книги, если не было<br>
+            обновлений в течение {{ bucCancelDays }} дней
+        </q-tooltip>
+    </q-checkbox>
+</div>
+
+<div v-show="configBucEnabled && bucEnabled && bucCancelEnabled" class="item row">
+    <div class="label-6"></div>
+    <div class="col-5"></div>
+    <div class="col row">
+        <NumInput class="col-left" v-model="bucCancelDays" :min="1" :max="10000"/>
+
+        <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+            Снимать флаг проверки с книги, если не было<br>
+            обновлений в течение {{ bucCancelDays }} дней
+        </q-tooltip>
+    </div>
+</div>
+

client/components/Reader/share/bookManager.js

@@ -234,6 +234,10 @@ class BookManager {
 
     async addBook(newBook, callback) {        
         let meta = {url: newBook.url, path: newBook.path};
+
+        if (newBook.downloadSize !== undefined && newBook.downloadSize >= 0)
+            meta.downloadSize = newBook.downloadSize;
+
         meta.key = this.keyFromPath(meta.path);
         meta.addTime = Date.now();//время добавления в кеш
 
@@ -483,6 +487,31 @@ class BookManager {
         await this.recentSetItem(item);
     }
 
+    async setCheckBuc(value, checkBuc) {
+        const item = this.recent[value.key];
+
+        const updateItems = [];
+        if (item) {
+            if (item.sameBookKey !== undefined) {
+                const sorted = this.getSortedRecent();
+                for (const book of sorted) {
+                    if (!book.deleted && book.sameBookKey === item.sameBookKey)
+                        updateItems.push(book);
+                }
+            } else {
+                updateItems.push(item);
+            }
+        }
+
+        const now = Date.now();
+        for (const book of updateItems) {
+            book.checkBuc = checkBuc;
+            if (checkBuc)
+                book.checkBucTime = now;
+            await this.recentSetItem(book);
+        }
+    }
+
     async cleanRecentBooks() {
         const sorted = this.getSortedRecent();
 

client/components/Reader/versionHistory.js

@@ -1,4 +1,22 @@
 export const versionHistory = [
+{
+    version: '0.12.0',
+    releaseDate: '2022-07-27',
+    showUntil: '2022-08-03',
+    content:
+`
+<ul>
+    <li>запущен сервер проверки обновлений книг:</li>
+        <ul>
+            <li>проверка обновления той или иной книги настраивается в списке загруженных (чекбокс)</li>
+            <li>для того, чтобы чекбокс появился у ранее загруженной, необходимо принудительно обновить книгу</li>
+            <li>в настройках можно указать разницу размеров, при которой требуется делать уведомление</li>
+        </ul>
+</ul>
+
+`
+},
+
 {
     version: '0.11.8',
     releaseDate: '2022-07-14',

client/store/index.js

@@ -1,5 +1,6 @@
 import { createStore } from 'vuex';
-import createPersistedState from 'vuex-persistedstate';
+//import createPersistedState from 'vuex-persistedstate';
+import VuexPersistence from 'vuex-persist';
 
 import root from './root.js';
 import uistate from './modules/uistate';
@@ -8,6 +9,8 @@ import reader from './modules/reader';
 
 const debug = process.env.NODE_ENV !== 'production';
 
+const vuexLocal = new VuexPersistence();
+
 export default createStore(Object.assign({}, root, {
     modules: {
         uistate,
@@ -15,5 +18,5 @@ export default createStore(Object.assign({}, root, {
         reader,
     },
     strict: debug,
-    plugins: [createPersistedState()]
+    plugins: [vuexLocal.plugin]
 }));

client/store/modules/reader.js

@@ -185,14 +185,23 @@ const settingDefaults = {
 
     fontShifts: {},
     showToolButton: {},
-    toolBarHideOnScroll: true,
+    toolBarHideOnScroll: false,
     userHotKeys: {},
     userWallpapers: [],
 
     recentShowSameBook: false,
     recentSortMethod: '',
 
+    //Book Update Checker
+    bucEnabled: true, // общее включение/выключение проверки обновлений
+    bucSizeDiff: 1, // разница в размерах файла, при которой показывать наличие обновления
+    bucSetOnNew: true, // автоматически включать проверку обновлений для вновь загружаемых файлов
+    bucCancelEnabled: true, // вкл/выкл отмену проверки книг через bucCancelDays
+    bucCancelDays: 90, // количество дней, через которое отменяется проверка книги, при условии отсутствия обновлений за это время
+
+    //для SettingsPage
     needUpdateSettingsView: 0,
+
 };
 
 for (const font of fonts)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "Liberama",
-  "version": "0.11.8",
+  "version": "0.12.0",
   "author": "Book Pauk <bookpauk@gmail.com>",
   "license": "CC0-1.0",
   "repository": "bookpauk/liberama",
@@ -21,67 +21,64 @@
     "scripts": "server/config/*.js"
   },
   "devDependencies": {
-    "@babel/core": "^7.16.0",
-    "@babel/eslint-parser": "^7.16.3",
-    "@babel/eslint-plugin": "^7.14.5",
-    "@babel/plugin-proposal-decorators": "^7.16.0",
-    "@babel/preset-env": "^7.16.0",
+    "@babel/core": "^7.18.9",
+    "@babel/eslint-parser": "^7.18.9",
+    "@babel/eslint-plugin": "^7.17.7",
+    "@babel/plugin-proposal-decorators": "^7.18.9",
+    "@babel/preset-env": "^7.18.9",
     "@vue/compiler-sfc": "^3.2.22",
-    "babel-loader": "^8.2.3",
+    "babel-loader": "^8.2.5",
     "copy-webpack-plugin": "^11.0.0",
-    "css-loader": "^6.5.1",
+    "css-loader": "^6.7.1",
     "css-minimizer-webpack-plugin": "^4.0.0",
-    "eslint": "^8.19.0",
-    "eslint-plugin-vue": "^9.2.0",
+    "eslint": "^8.20.0",
+    "eslint-plugin-vue": "^9.3.0",
     "html-webpack-plugin": "^5.5.0",
-    "mini-css-extract-plugin": "^2.4.4",
-    "pkg": "^5.5.1",
-    "terser-webpack-plugin": "^5.2.5",
+    "mini-css-extract-plugin": "^2.6.1",
+    "pkg": "^5.8.0",
+    "terser-webpack-plugin": "^5.3.3",
     "vue-eslint-parser": "^9.0.3",
     "vue-loader": "^17.0.0",
     "vue-style-loader": "^4.1.3",
-    "webpack": "^5.64.1",
-    "webpack-cli": "^4.9.1",
-    "webpack-dev-middleware": "^5.2.1",
+    "webpack": "^5.74.0",
+    "webpack-cli": "^4.10.0",
+    "webpack-dev-middleware": "^5.3.3",
     "webpack-hot-middleware": "^2.25.1",
     "webpack-merge": "^5.8.0",
-    "workbox-webpack-plugin": "^6.4.1"
+    "workbox-webpack-plugin": "^6.5.3"
   },
   "dependencies": {
-    "@quasar/extras": "^1.12.0",
-    "@vue/compat": "^3.2.21",
+    "@quasar/extras": "^1.15.0",
+    "@vue/compat": "^3.2.37",
     "axios": "^0.27.2",
     "base-x": "^4.0.0",
     "chardet": "^1.4.0",
     "compression": "^1.7.4",
-    "express": "^4.17.1",
+    "express": "^4.18.1",
     "fg-loadcss": "^3.1.0",
     "fs-extra": "^10.1.0",
     "he": "^1.2.0",
     "iconv-lite": "^0.6.3",
-    "jembadb": "^3.0.8",
+    "jembadb": "^3.0.9",
     "localforage": "^1.10.0",
     "lodash": "^4.17.21",
-    "minimist": "^1.2.5",
+    "minimist": "^1.2.6",
     "multer": "^1.4.5-lts.1",
     "pako": "^2.0.4",
     "path-browserify": "^1.0.1",
     "pidusage": "^3.0.0",
     "quasar": "^2.7.5",
     "safe-buffer": "^5.2.1",
-    "sanitize-html": "^2.5.3",
+    "sanitize-html": "^2.7.1",
     "sjcl": "^1.0.8",
-    "sql-template-strings": "^2.2.2",
-    "sqlite": "^4.0.23",
-    "sqlite3": "^5.0.2",
     "tar-fs": "^2.1.1",
     "unbzip2-stream": "^1.4.3",
     "vue": "^3.2.37",
-    "vue-router": "^4.1.1",
+    "vue-router": "^4.1.2",
     "vuex": "^4.0.2",
-    "vuex-persistedstate": "^4.1.0",
-    "webdav": "^4.7.0",
-    "ws": "^8.2.3",
+    "vuex-persist": "^3.1.3",
+    "webdav": "^4.10.0",
+    "ws": "^8.8.1",
     "zip-stream": "^4.1.0"
   }
 }

server/config/base.js

@@ -23,32 +23,27 @@ module.exports = {
 
     useExternalBookConverter: false,
     acceptFileExt: '.fb2, .fb3, .html, .txt, .zip, .bz2, .gz, .rar, .epub, .mobi, .rtf, .doc, .docx, .pdf, .djvu, .jpg, .jpeg, .png',
-    webConfigParams: ['name', 'version', 'mode', 'maxUploadFileSize', 'useExternalBookConverter', 'acceptFileExt', 'branch'],
-
-    db: [
-        {
-            poolName: 'app',
-            connCount: 20,
-            fileName: 'app.sqlite',
-        },
-        {
-            poolName: 'readerStorage',
-            connCount: 20,
-            fileName: 'reader-storage.sqlite',            
-        }
-    ],
+    webConfigParams: ['name', 'version', 'mode', 'maxUploadFileSize', 'useExternalBookConverter', 'acceptFileExt', 'bucEnabled', 'branch'],
 
     jembaDb: [
         {
+            serverMode: ['reader', 'omnireader', 'liberama.top'],
             dbName: 'app',
             thread: true,
             openAll: true,
         },
         {
+            serverMode: ['reader', 'omnireader', 'liberama.top'],
             dbName: 'reader-storage',
             thread: true,
             openAll: true,
-        }
+        },
+        {
+            serverMode: 'book_update_checker',
+            dbName: 'book-update-server',
+            thread: true,
+            openAll: true,            
+        },
     ],
 
     servers: [
@@ -58,23 +53,31 @@ module.exports = {
             ip: '0.0.0.0',
             port: '33080',
         },
+        /*{
+            serverName: '2',
+            mode: 'book_update_checker', //'none', 'normal', 'site', 'reader', 'omnireader', 'liberama.top', 'book_update_checker'
+            isHttps: true,
+            keysFile: 'server',
+            ip: '0.0.0.0',
+            port: '33443',
+            accessToken: '',
+        }*/
     ],
 
-    /*
-    remoteWebDavStorage: false,
-    remoteWebDavStorage: {
-        url: '127.0.0.1:1900',
-        username: '',
-        password: '',
-    },
-    */
-
     remoteStorage: false,
     /*
     remoteStorage: {
-        url: 'https://127.0.0.1:11900',
+        url: 'wss://127.0.0.1:11900',
         accessToken: '',
     },
     */
+    bucEnabled: false,
+    bucServer: false,
+    /*
+    bucServer: {
+        url: 'wss://127.0.0.1:33443',
+        accessToken: '',
+    }
+    */
 };
 

server/config/index.js

@@ -10,7 +10,9 @@ const propsToSave = [
     'useExternalBookConverter',
     
     'servers',
-    'remoteWebDavStorage',
+    'remoteStorage',
+    'bucEnabled',
+    'bucServer',
 ];
 
 let instance = null;

server/controllers/BookUpdateCheckerController.js

@@ -1,6 +1,7 @@
-const WebSocket = require ('ws');
+const WebSocket = require('ws');
 //const _ = require('lodash');
 
+const BUCServer = require('../core/BookUpdateChecker/BUCServer');
 const log = new (require('../core/AppLogger'))().log;//singleton
 //const utils = require('../core/utils');
 
@@ -12,7 +13,8 @@ class BookUpdateCheckerController {
         this.config = config;
         this.isDevelopment = (config.branch == 'development');
 
-        //this.readerStorage = new JembaReaderStorage();
+        this.accessToken = config.accessToken;
+        this.bucServer = new BUCServer(config);
 
         this.wss = wss;
 
@@ -46,7 +48,7 @@ class BookUpdateCheckerController {
         let req = {};
         try {
             if (this.isDevelopment) {
-                log(`WebSocket-IN:  ${message.substr(0, 4000)}`);
+                log(`BUC-WebSocket-IN:  ${message.substr(0, 4000)}`);
             }
 
             req = JSON.parse(message);
@@ -56,9 +58,16 @@ class BookUpdateCheckerController {
             //pong for WebSocketConnection
             this.send({_rok: 1}, req, ws);
 
+            if (req.accessToken !== this.accessToken)
+                throw new Error('Access denied');
+
             switch (req.action) {
                 case 'test':
                     await this.test(req, ws); break;
+                case 'get-buc':
+                    await this.getBuc(req, ws); break;
+                case 'update-buc':
+                    await this.updateBuc(req, ws); break;
 
                 default:
                     throw new Error(`Action not found: ${req.action}`);
@@ -79,7 +88,7 @@ class BookUpdateCheckerController {
             ws.send(message);
 
             if (this.isDevelopment) {
-                log(`WebSocket-OUT: ${message.substr(0, 4000)}`);
+                log(`BUC-WebSocket-OUT: ${message.substr(0, 4000)}`);
             }
 
         }
@@ -90,6 +99,28 @@ class BookUpdateCheckerController {
         this.send({message: 'Liberama project is awesome'}, req, ws);
     }
 
+    async getBuc(req, ws) {
+        if (!req.fromCheckTime)
+            throw new Error(`key 'fromCheckTime' is empty`);
+
+        await this.bucServer.getBuc(req.fromCheckTime, (rows) => {
+            this.send({state: 'get', rows}, req, ws);
+        });
+
+        this.send({state: 'finish'}, req, ws);
+    }
+
+    async updateBuc(req, ws) {
+        if (!req.bookUrls)
+            throw new Error(`key 'bookUrls' is empty`);
+
+        if (!Array.isArray(req.bookUrls))
+            throw new Error(`key 'bookUrls' must be array`);
+
+        await this.bucServer.updateBuc(req.bookUrls);
+
+        this.send({state: 'success'}, req, ws);
+    }
 }
 
 module.exports = BookUpdateCheckerController;

server/controllers/WebSocketController.js

@@ -4,6 +4,7 @@ const _ = require('lodash');
 const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton
 const JembaReaderStorage = require('../core/Reader/JembaReaderStorage');//singleton
 const WorkerState = require('../core/WorkerState');//singleton
+const BUCClient = require('../core/BookUpdateChecker/BUCClient');//singleton
 const log = new (require('../core/AppLogger'))().log;//singleton
 const utils = require('../core/utils');
 
@@ -19,6 +20,10 @@ class WebSocketController {
         this.readerWorker = new ReaderWorker(config);
         this.workerState = new WorkerState();
 
+        if (config.bucEnabled) {
+            this.bucClient = new BUCClient(config);
+        }
+
         this.wss = wss;
 
         wss.on('connection', (ws) => {
@@ -76,6 +81,8 @@ class WebSocketController {
                     await this.uploadFileBuf(req, ws); break;
                 case 'upload-file-touch':
                     await this.uploadFileTouch(req, ws); break;
+                case 'check-buc':
+                    await this.checkBuc(req, ws); break;
 
                 default:
                     throw new Error(`Action not found: ${req.action}`);
@@ -179,6 +186,21 @@ class WebSocketController {
         
         this.send({url: await this.readerWorker.uploadFileTouch(req.url)}, req, ws);
     }
+
+    async checkBuc(req, ws) {
+        if (!this.config.bucEnabled)
+            throw new Error('BookUpdateChecker disabled');
+
+        if (!req.bookUrls)
+            throw new Error(`key 'bookUrls' is empty`);
+
+        if (!Array.isArray(req.bookUrls))
+            throw new Error(`key 'bookUrls' must be array`);
+
+        const data = await this.bucClient.checkBuc(req.bookUrls);
+
+        this.send({state: 'success', data}, req, ws);
+    }
 }
 
 module.exports = WebSocketController;

server/core/BookUpdateChecker/BUCClient.js

@@ -0,0 +1,260 @@
+const WebSocketConnection = require('../WebSocketConnection');
+const JembaConnManager = require('../../db/JembaConnManager');//singleton
+
+const ayncExit = new (require('../AsyncExit'))();
+const utils = require('../utils');
+const log = new (require('../AppLogger'))().log;//singleton
+
+const minuteMs = 60*1000;
+const hourMs = 60*minuteMs;
+const dayMs = 24*hourMs;
+
+let instance = null;
+
+//singleton
+class BUCClient {
+    constructor(config) {
+        if (!instance) {
+            this.config = config;
+
+            this.connManager = new JembaConnManager();
+            this.appDb = this.connManager.db['app'];
+
+            this.wsc = new WebSocketConnection(config.bucServer.url, 10, 30, {rejectUnauthorized: false});
+            this.accessToken = config.bucServer.accessToken;
+
+            //константы
+            if (this.config.branch !== 'development') {
+                this.cleanQueryInterval = 300*dayMs;//интервал очистки устаревших
+                this.syncPeriod = 1*hourMs;//период синхронизации с сервером BUC
+                this.sendBookUrlsPeriod = 1*minuteMs;//период отправки BookUrls на сервер BUC
+            } else {
+                this.cleanQueryInterval = 300*dayMs;//интервал очистки устаревших
+                this.syncPeriod = 1*minuteMs;//период синхронизации с сервером BUC
+                this.sendBookUrlsPeriod = 1*1000;//период отправки BookUrls на сервер BUC
+            }
+
+            this.fromCheckTime = 1;
+            this.bookUrls = new Set();
+
+            this.main();//no await
+
+            instance = this;
+        }
+
+        return instance;
+    }
+
+    async wsRequest(query) {
+        const response = await this.wsc.message(
+            await this.wsc.send(Object.assign({accessToken: this.accessToken}, query), 60),
+            60
+        );
+        if (response.error)
+            throw new Error(response.error);
+        return response;
+    }
+
+    async wsGetBuc(fromCheckTime, callback) {
+        const requestId = await this.wsc.send({accessToken: this.accessToken, action: 'get-buc', fromCheckTime}, 60);
+        while (1) {//eslint-disable-line
+            const res = await this.wsc.message(requestId, 60);
+
+            if (res.state == 'get') {
+                await callback(res.rows);
+            } else {
+                break;
+            }
+        }
+    }
+
+    async wsUpdateBuc(bookUrls) {
+        return await this.wsRequest({action: 'update-buc', bookUrls});
+    }
+
+    async checkBuc(bookUrls) {
+        const db = this.appDb;
+
+        for (const url of bookUrls)
+            this.bookUrls.add(url);
+
+        const rows = await db.select({
+            table: 'buc',
+            map: `(r) => ({id: r.id, size: r.size})`,
+            where: `@@id(${db.esc(bookUrls)})`,
+        });
+
+        return rows;
+    }
+
+    async findMaxCheckTime() {
+        const db = this.appDb;
+
+        let result = 1;
+
+        //одним куском, возможно будет жрать память
+        const rows = await db.select({
+            table: 'buc',
+            where: `
+                const result = new Set();
+                let max = 0;
+                let maxId = null;
+
+                @iter(@all(), (row) => {
+                    if (row.checkTime > max) {
+                        max = row.checkTime;
+                        maxId = row.id;
+                    }
+                });
+
+                if (maxId)
+                    result.add(maxId);
+
+                return result;
+            `
+        });
+
+        if (rows.length)
+            result = rows[0].checkTime;
+
+        return result;
+    }
+
+    async periodicSendBookUrls() {
+        while (1) {//eslint-disable-line
+            try {
+                //отправим this.bookUrls
+                if (this.bookUrls.size) {
+                    log(`client: remote update buc begin`);
+
+                    const arr = Array.from(this.bookUrls);
+                    this.bookUrls = new Set();
+
+                    const chunkSize = 100;
+                    let updated = 0;
+                    for (let i = 0; i < arr.length; i += chunkSize) {
+                        const chunk = arr.slice(i, i + chunkSize);
+                        
+                        const res = await this.wsUpdateBuc(chunk);
+                        if (!res.error && res.state == 'success') {
+                            //update success
+                            updated += chunk.length;
+                        } else {
+                            for (const url of chunk) {
+                                this.bookUrls.add(url);
+                            }
+                            log(LM_ERR, `update-buc error: ${(res.error ? res.error : `wrong state "${res.state}"`)}`);
+                        }
+                    }
+                    log(`client: remote update buc end, updated ${updated} urls`);
+                }
+            } catch (e) {
+                log(LM_ERR, e.stack);
+            }
+
+            await utils.sleep(this.sendBookUrlsPeriod);
+        }
+    }
+
+    async periodicSync() {
+        const db = this.appDb;
+
+        while (1) {//eslint-disable-line
+            try {
+                //почистим нашу таблицу 'buc'
+                log(`client: clean 'buc' table begin`);
+                const cleanTime = Date.now() - this.cleanQueryInterval;
+                while (1) {//eslint-disable-line
+                    //выборка всех по кусочкам
+                    const rows = await db.select({
+                        table: 'buc',
+                        where: `
+                            let iter = @getItem('clean');
+                            if (!iter) {
+                                iter = @all();
+                                @setItem('clean', iter);
+                            }
+
+                            const ids = new Set();
+                            let id = iter.next();
+                            while (!id.done && ids.size < 1000) {
+                                ids.add(id.value);
+                                id = iter.next();
+                            }
+
+                            return ids;
+                        `
+                    });
+
+                    if (rows.length) {
+                        const toDelIds = [];
+                        for (const row of rows)
+                            if (row.queryTime <= cleanTime)
+                                toDelIds.push(row.id);
+
+                        //удаление
+                        const res = await db.delete({
+                            table: 'buc',
+                            where: `@@id(${db.esc(toDelIds)})`,
+                        });
+
+                        log(`client: clean 'buc' deleted ${res.deleted}`);
+                    } else {
+                        break;
+                    }
+                }
+                await db.select({
+                    table: 'buc',
+                    where: `
+                        @delItem('clean');
+                        return new Set();
+                    `
+                });
+
+                log(`client: clean 'buc' table end`);
+
+                //синхронизация с сервером BUC
+                log(`client: sync 'buc' table begin`);
+                this.fromCheckTime -= 30*minuteMs;//минус полчаса на всякий случай
+                await this.wsGetBuc(this.fromCheckTime, async(rows) => {
+                    for (const row of rows) {
+                        if (row.checkTime > this.fromCheckTime)
+                            this.fromCheckTime = row.checkTime;
+                    }
+
+                    const res = await db.insert({
+                        table: 'buc',
+                        replace: true,
+                        rows
+                    });
+    
+                    log(`client: sync 'buc' table, inserted ${res.inserted} rows, replaced ${res.replaced}`);
+                });
+                log(`client: sync 'buc' table end`);
+            } catch (e) {
+                log(LM_ERR, e.stack);
+            }
+
+            await utils.sleep(this.syncPeriod);
+        }
+    }
+
+    async main() {
+        try {
+            if (!this.config.bucEnabled)
+                throw new Error('BookUpdateChecker disabled');
+
+            this.fromCheckTime = await this.findMaxCheckTime();
+            
+            this.periodicSendBookUrls();//no await
+            this.periodicSync();//no await
+
+            log(`BUC Client Worker started`);
+        } catch (e) {
+            log(LM_FATAL, e.stack);
+            ayncExit.exit(1);
+        }
+    }
+}
+
+module.exports = BUCClient;

server/core/BookUpdateChecker/BUCServer.js

@@ -1,24 +1,350 @@
+const fs = require('fs-extra');
+
+const FileDownloader = require('../FileDownloader');
+const JembaConnManager = require('../../db/JembaConnManager');//singleton
+
+const ayncExit = new (require('../AsyncExit'))();
+const utils = require('../utils');
+const log = new (require('../AppLogger'))().log;//singleton
+
+const minuteMs = 60*1000;
+const hourMs = 60*minuteMs;
+const dayMs = 24*hourMs;
+
 let instance = null;
 
 //singleton
 class BUCServer {
     constructor(config) {
         if (!instance) {
-            this.config = Object.assign({}, config);
+            this.config = config;
+
+            //константы
+            if (this.config.branch !== 'development') {
+                this.maxCheckQueueLength = 10000;//максимальная длина checkQueue
+                this.fillCheckQueuePeriod = 1*minuteMs;//период пополнения очереди
+                this.periodicCheckWait = 500;//пауза, если нечего делать
+
+                this.cleanQueryInterval = 300*dayMs;//интервал очистки устаревших
+                this.oldQueryInterval = 14*dayMs;//интервал устаревания запроса на обновление
+                this.checkingInterval = 5*hourMs;//интервал проверки обновления одного и того же файла
+                this.sameHostCheckInterval = 1000;//интервал проверки файла на том же сайте, не менее
+            } else {
+                this.maxCheckQueueLength = 10;//максимальная длина checkQueue
+                this.fillCheckQueuePeriod = 10*1000;//период пополнения очереди
+                this.periodicCheckWait = 500;//пауза, если нечего делать
+
+                this.cleanQueryInterval = 300*dayMs;//интервал очистки устаревших
+                this.oldQueryInterval = 30*dayMs;//интервал устаревания запроса на обновление
+                this.checkingInterval = 30*1000;//интервал проверки обновления одного и того же файла
+                this.sameHostCheckInterval = 1000;//интервал проверки файла на том же сайте, не менее
+            }
+
             
             this.config.tempDownloadDir = `${config.tempDir}/download`;
             fs.ensureDirSync(this.config.tempDownloadDir);
 
-            this.down = new FileDownloader(config.maxUploadFileSize);
+            this.down = new FileDownloader(config.maxUploadFileSize);            
+
+            this.connManager = new JembaConnManager();
+            this.db = this.connManager.db['book-update-server'];
             
+            this.checkQueue = [];
+            this.hostChecking = {};
+
+            this.main(); //no await
+
             instance = this;
         }
 
         return instance;
-    }    
+    }
+
+    async getBuc(fromCheckTime, callback) {
+        const db = this.db;
+
+        const iterName = utils.randomHexString(30);
+
+        while (1) {//eslint-disable-line
+            const rows = await db.select({
+                table: 'buc',
+                where: `
+                    let iter = @getItem(${db.esc(iterName)});
+                    if (!iter) {
+                        iter = @dirtyIndexLR('checkTime', ${db.esc(fromCheckTime)});
+                        iter = iter.values();
+                        @setItem(${db.esc(iterName)}, iter);
+                    }
+
+                    const ids = new Set();
+                    let id = iter.next();
+                    while (!id.done && ids.size < 100) {
+                        ids.add(id.value);
+                        id = iter.next();
+                    }
+
+                    return ids;
+                `
+            });
+
+            if (rows.length)
+                callback(rows);
+            else
+                break;
+        }
+
+        await db.select({
+            table: 'buc',
+            where: `
+                @delItem(${db.esc(iterName)});
+                return new Set();
+            `
+        });
+    }
+
+    async updateBuc(bookUrls) {
+        const db = this.db;
+        const now = Date.now();
+
+        const rows = await db.select({
+            table: 'buc',
+            map: `(r) => ({id: r.id})`,
+            where: `@@id(${db.esc(bookUrls)})`
+        });
+
+        const exists = new Set();
+        for (const row of rows) {
+            exists.add(row.id);
+        }
+
+        const toUpdateIds = [];
+        const toInsertRows = [];
+        for (let id of bookUrls) {
+            if (!id)
+                continue;
+
+            if (id.length > 1000) {
+                id = id.substring(0, 1000);
+            }
+
+            if (exists.has(id)) {
+                toUpdateIds.push(id);
+            } else {
+                toInsertRows.push({
+                    id,
+                    queryTime: now,
+                    checkTime: 0, // 0 - never checked
+                    etag: '',
+                    modTime: '',
+                    size: 0,
+                    checkSum: '', //sha256
+                    state: 0, // 0 - not processing, 1 - processing
+                    error: '',
+                });
+            }
+        }
+
+        if (toUpdateIds.length) {
+            await db.update({
+                table: 'buc',
+                mod: `(r) => r.queryTime = ${db.esc(now)}`,
+                where: `@@id(${db.esc(toUpdateIds)})`
+            });
+        }
+
+        if (toInsertRows.length) {
+            await db.insert({
+                table: 'buc',
+                ignore: true,
+                rows: toInsertRows,
+            });
+        }
+    }
+
+    async fillCheckQueue() {
+        const db = this.db;
+
+        while (1) {//eslint-disable-line
+            try {
+                let now = Date.now();
+
+                //чистка совсем устаревших
+                let rows = await db.select({
+                    table: 'buc',
+                    where: `@@dirtyIndexLR('queryTime', undefined, ${db.esc(now - this.cleanQueryInterval)})`
+                });
+
+                if (rows.length) {
+                    const ids = rows.map((r) => r.id);
+                    const res = await db.delete({
+                        table: 'buc',
+                        where: `@@id(${db.esc(ids)})`,
+                    });
+
+                    log(LM_WARN, `clean 'buc' table: deleted ${res.deleted}`);
+                }
+
+                rows = await db.select({table: 'buc', count: true});
+                log(LM_WARN, `'buc' table size: ${rows[0].count}`);
+
+                now = Date.now();
+                //выборка кандидатов
+                rows = await db.select({
+                    table: 'buc',
+                    where: `
+                        @@and(
+                            @dirtyIndexLR('queryTime', ${db.esc(now - this.oldQueryInterval)}),
+                            @dirtyIndexLR('checkTime', undefined, ${db.esc(now - this.checkingInterval)}),
+                            @flag('notProcessing')
+                        );
+                    `
+                });
+
+                //формирование checkQueue
+                if (rows.length) {
+                    const ids = [];
+                    const rowsToPush = [];
+
+                    //сначала выберем сколько надо
+                    for (const row of rows) {
+                        if (this.checkQueue.length + rowsToPush.length >= this.maxCheckQueueLength)
+                            break;
+
+                        rowsToPush.push(row);
+                        ids.push(row.id);
+                    }
+
+                    //установим у них флаг "в обработке"
+                    await db.update({
+                        table: 'buc',
+                        mod: `(r) => r.state = 1`,
+                        where: `@@id(${db.esc(ids)})`
+                    });
+
+                    //пушим в очередь, после этого их обработает periodicCheck
+                    for (const row of rowsToPush)
+                        this.checkQueue.push(row);
+                    
+                    log(LM_WARN, `checkQueue: added ${ids.length} recs, total ${this.checkQueue.length}`);
+                }
+            } catch(e) {
+                log(LM_ERR, e.stack);
+            }
+
+            await utils.sleep(this.fillCheckQueuePeriod);
+        }
+    }
+
+    async periodicCheck() {
+        const db = this.db;
+
+        while (1) {//eslint-disable-line
+            try {
+                if (!this.checkQueue.length)
+                    await utils.sleep(this.periodicCheckWait);
+
+                if (!this.checkQueue.length)
+                    continue;
+
+                const row = this.checkQueue.shift();
+
+                const url = new URL(row.id);
+
+                //только если обращались к тому же хосту не ранее sameHostCheckInterval миллисекунд назад
+                if (!this.hostChecking[url.hostname]) {
+                    this.hostChecking[url.hostname] = true;
+
+                    try {
+                        let unchanged = true;
+                        let hash = '';
+
+                        const headers = await this.down.head(row.id);
+
+                        const etag = headers['etag'] || '';
+                        const modTime = headers['last-modified'] || '';
+                        let size = parseInt(headers['content-length'], 10) || 0;
+
+                        //log(row.id);
+                        //log(`etag: ${etag}, modTime: ${modTime}, size: ${size}`)
+
+                        if ((!etag || !row.etag || (etag !== row.etag))
+                            && (!modTime || !row.modTime || (modTime !== row.modTime))
+                            && (!size || !row.size || (size !== row.size))
+                            ) {
+                            const downdata = await this.down.load(row.id);
+
+                            size = downdata.length;
+                            hash = await utils.getBufHash(downdata, 'sha256', 'hex');
+                            unchanged = false;
+                        }
+
+                        await db.update({
+                            table: 'buc',
+                            mod: `(r) => {
+                                r.checkTime = ${db.esc(Date.now())};
+                                r.etag = ${(unchanged ? 'r.etag' : db.esc(etag))};
+                                r.modTime = ${(unchanged ? 'r.modTime' : db.esc(modTime))};
+                                r.size = ${(unchanged ? 'r.size' : db.esc(size))};
+                                r.checkSum = ${(unchanged ? 'r.checkSum' : db.esc(hash))};
+                                r.state = 0;
+                                r.error = '';
+                            }`,
+                            where: `@@id(${db.esc(row.id)})`
+                        });
+
+                        if (unchanged) {
+                            log(`checked ${row.id} > unchanged`);
+                        } else {
+                            log(`checked ${row.id} > size ${size}`);
+                        }
+                    } catch (e) {
+                        await db.update({
+                            table: 'buc',
+                            mod: `(r) => {
+                                r.checkTime = ${db.esc(Date.now())};
+                                r.state = 0;
+                                r.error = ${db.esc(e.message)};
+                            }`,
+                            where: `@@id(${db.esc(row.id)})`
+                        });
+
+                        log(LM_ERR, `error ${row.id} > ${e.stack ? e.stack : e.message}`);
+                    } finally {
+                        (async() => {
+                            await utils.sleep(this.sameHostCheckInterval);
+                            this.hostChecking[url.hostname] = false;
+                        })();
+                    }
+                } else {
+                    this.checkQueue.push(row);
+                }
+            } catch(e) {
+                log(LM_ERR, e.stack);
+            }
+
+            await utils.sleep(10);
+        }
+    }
 
     async main() {
+        try {
+            //обнуляем все статусы
+            await this.db.update({table: 'buc', mod: `(r) => r.state = 0`});
+
+            this.fillCheckQueue();//no await
+
+            //10 потоков
+            for (let i = 0; i < 10; i++)
+                this.periodicCheck();//no await
+
+            log(`-------------------------`);
+            log(`BUC Server Worker started`);
+            log(`-------------------------`);
+        } catch (e) {
+            log(LM_FATAL, e.stack);
+            ayncExit.exit(1);
+        }
     }
 }
 
-module.exports = BUCServer;
+module.exports = BUCServer;

server/core/FileDownloader.js

@@ -1,5 +1,7 @@
 const axios = require('axios');
 
+const userAgent = 'Mozilla/5.0 (X11; HasCodingOs 1.0; Linux x64) AppleWebKit/637.36 (KHTML, like Gecko) Chrome/70.0.3112.101 Safari/637.36 HasBrowser/5.0';
+
 class FileDownloader {
     constructor(limitDownloadSize = 0) {
         this.limitDownloadSize = limitDownloadSize;
@@ -10,7 +12,7 @@ class FileDownloader {
 
         const options = {
             headers: {
-                'user-agent': 'Mozilla/5.0 (X11; HasCodingOs 1.0; Linux x64) AppleWebKit/637.36 (KHTML, like Gecko) Chrome/70.0.3112.101 Safari/637.36 HasBrowser/5.0'
+                'user-agent': userAgent
             },
             responseType: 'stream',
         };
@@ -62,6 +64,17 @@ class FileDownloader {
         }
     }
 
+    async head(url) {
+        const options = {
+            headers: {
+                'user-agent': userAgent
+            },
+        };
+
+        const res = await axios.head(url, options);
+        return res.headers;
+    }
+
     streamToBuffer(stream, progress) {
         return new Promise((resolve, reject) => {
             

server/core/Reader/ReaderWorker.js

@@ -105,6 +105,7 @@ class ReaderWorker {
             const tempFilename2 = utils.randomHexString(30);
             const decompDirname = utils.randomHexString(30);
 
+            let downloadSize = -1;
             //download or use uploaded
             if (url.indexOf('disk://') != 0) {//download
                 const downdata = await this.down.load(url, (progress) => {
@@ -112,6 +113,8 @@ class ReaderWorker {
                 }, q.abort);
 
                 downloadedFilename = `${this.config.tempDownloadDir}/${tempFilename}`;
+
+                downloadSize = downdata.length;
                 await fs.writeFile(downloadedFilename, downdata);
             } else {//uploaded file
                 const fileHash = url.substr(7);
@@ -166,7 +169,12 @@ class ReaderWorker {
 
             //finish
             const finishFilename = path.basename(compFilename);
-            wState.finish({path: `/tmp/${finishFilename}`, size: stat.size});
+
+            const result = {path: `/tmp/${finishFilename}`, size: stat.size};
+            if (downloadSize >= 0)
+                result.downloadSize = downloadSize;
+
+            wState.finish(result);
 
             //асинхронно через 30 сек добавим в очередь на отправку
             //т.к. gzipFileIfNotExists может переупаковать файл

server/db/ConnManager.js

@@ -1,61 +0,0 @@
-//TODO: удалить модуль в 2023г
-const fs = require('fs-extra');
-
-const SqliteConnectionPool = require('./SqliteConnectionPool');
-const log = new (require('../core/AppLogger'))().log;//singleton
-
-const migrations = {
-    'app': require('./migrations/app'),
-    'readerStorage': require('./migrations/readerStorage'),
-};
-
-let instance = null;
-
-//singleton
-class ConnManager {
-    constructor() {
-        if (!instance) {
-            this.inited = false;
-
-            instance = this;
-        }
-
-        return instance;
-    }
-
-    async init(config) {
-        this.config = config;
-        this._pool = {};
-
-        const force = null;//(config.branch == 'development' ? 'last' : null);
-
-        for (const poolConfig of this.config.db) {
-            const dbFileName = this.config.dataDir + '/' + poolConfig.fileName;
-
-            //бэкап
-            if (!poolConfig.noBak && await fs.pathExists(dbFileName))
-                await fs.copy(dbFileName, `${dbFileName}.bak`);
-
-            const connPool = new SqliteConnectionPool();
-            await connPool.open(poolConfig, dbFileName);
-
-            log(`Opened database "${poolConfig.poolName}"`);
-            //миграции
-            const migs = migrations[poolConfig.poolName];
-            if (migs && migs.data.length) {
-                const applied = await connPool.migrate(migs.data, migs.table, force);
-                if (applied.length)
-                    log(`${applied.length} migrations applied to "${poolConfig.poolName}"`);
-            }
-
-            this._pool[poolConfig.poolName] = connPool;
-        }
-        this.inited = true;
-    }
-
-    get pool() {
-        return this._pool;
-    }
-}
-
-module.exports = ConnManager;

server/db/Converter.js

@@ -1,42 +0,0 @@
-//TODO: удалить модуль в 2023г
-const fs = require('fs-extra');
-const log = new (require('../core/AppLogger'))().log;//singleton
-
-class Converter {    
-    async run(config) {        
-        log('Converter start');
-
-        try {
-            const connManager = new (require('./ConnManager'))();//singleton
-            const storagePool = connManager.pool.readerStorage;
-
-            const jembaConnManager = new (require('./JembaConnManager'))();//singleton
-            const db = jembaConnManager.db['reader-storage'];
-
-            const srcDbPath = `${config.dataDir}/reader-storage.sqlite`;
-            if (!await fs.pathExists(srcDbPath)) {
-                log(LM_WARN, '  Source DB does not exist, nothing to do');
-                return;
-            }
-
-            const rows = await db.select({table: 'storage', count: true});
-            if (rows.length && rows[0].count != 0) {
-                log(LM_WARN, `  Destination table already exists (found ${rows[0].count} items), nothing to do`);
-                return;
-            }
-
-            const dbSrc = await storagePool.get();
-            try {
-                const rows = await dbSrc.all(`SELECT * FROM storage`);
-                await db.insert({table: 'storage', rows});
-                log(`  Inserted ${rows.length} items`);
-            } finally {
-                dbSrc.ret();
-            }
-        } finally {
-            log('Converter finish');
-        }
-    }
-}
-
-module.exports = Converter;

server/db/JembaConnManager.js

@@ -31,7 +31,29 @@ class JembaConnManager {
 
         ayncExit.add(this.close.bind(this));
 
+        const serverModes = new Set();
+        for (const serverCfg of this.config.servers) {
+            serverModes.add(serverCfg.mode);
+        }
+
         for (const dbConfig of this.config.jembaDb) {
+            //проверка, надо ли открывать базу, зависит от serverMode
+            if (dbConfig.serverMode) {
+                let serverMode = dbConfig.serverMode;
+                if (!Array.isArray(dbConfig.serverMode))
+                    serverMode = [dbConfig.serverMode];
+
+                let modePresent = false;
+                for (const mode of serverMode) {
+                    modePresent = serverModes.has(mode);
+                    if (modePresent)
+                        break;
+                }
+
+                if (!modePresent)
+                    continue;
+            }
+
             const dbPath = `${this.config.dataDir}/db/${dbConfig.dbName}`;
 
             //бэкап

server/db/SqliteConnectionPool.js

@@ -1,193 +0,0 @@
-//TODO: удалить модуль в 2023г
-const sqlite3 = require('sqlite3');
-const sqlite = require('sqlite');
-
-const SQL = require('sql-template-strings');
-
-class SqliteConnectionPool {
-    constructor() {
-        this.closed = true;
-    }
-
-    async open(poolConfig, dbFileName) {
-        const connCount = poolConfig.connCount || 1;
-        const busyTimeout = poolConfig.busyTimeout || 60*1000;
-        const cacheSize = poolConfig.cacheSize || 2000;
-
-        this.dbFileName = dbFileName;
-        this.connections = [];
-        this.freed = new Set();
-        this.waitingQueue = [];
-
-        for (let i = 0; i < connCount; i++) {
-            let client = await sqlite.open({
-                filename: dbFileName,
-                driver: sqlite3.Database
-            });
-
-            client.configure('busyTimeout', busyTimeout); //ms
-            await client.exec(`PRAGMA cache_size = ${cacheSize}`);
-
-            client.ret = () => {
-                this.freed.add(i);
-                if (this.waitingQueue.length) {
-                    this.waitingQueue.shift().onFreed(i);
-                }
-            };
-
-            this.freed.add(i);
-            this.connections[i] = client;
-        }
-        this.closed = false;
-    }
-
-    get() {
-        return new Promise((resolve) => {
-            if (this.closed)
-                throw new Error('Connection pool closed');
-
-            const freeConnIndex = this.freed.values().next().value;
-            if (freeConnIndex !== undefined) {
-                this.freed.delete(freeConnIndex);
-                resolve(this.connections[freeConnIndex]);
-                return;
-            }
-
-            this.waitingQueue.push({
-                onFreed: (connIndex) => {
-                    this.freed.delete(connIndex);
-                    resolve(this.connections[connIndex]);
-                },
-            });
-        });
-    }
-
-    async run(query) {
-        const dbh = await this.get();
-        try {
-            let result = await dbh.run(query);
-            dbh.ret();
-            return result;
-        } catch (e) {
-            dbh.ret();
-            throw e;
-        }
-    }
-
-    async all(query) {
-        const dbh = await this.get();
-        try {
-            let result = await dbh.all(query);
-            dbh.ret();
-            return result;
-        } catch (e) {
-            dbh.ret();
-            throw e;
-        }
-    }
-
-    async exec(query) {
-        const dbh = await this.get();
-        try {
-            let result = await dbh.exec(query);
-            dbh.ret();
-            return result;
-        } catch (e) {
-            dbh.ret();
-            throw e;
-        }
-    }
-
-    async close() {
-        for (let i = 0; i < this.connections.length; i++) {
-            await this.connections[i].close();
-        }
-        this.closed = true;
-    }
-
-     // Modified from node-sqlite/.../src/Database.js
-    async migrate(migs, table, force) {
-        const migrations = migs.sort((a, b) => Math.sign(a.id - b.id));
-
-        if (!migrations.length) {
-            throw new Error('No migration data');
-        }
-
-        migrations.map(migration => {
-            const data = migration.data;
-            const [up, down] = data.split(/^--\s+?down\b/mi);
-            if (!down) {
-                const message = `The ${migration.filename} file does not contain '-- Down' separator.`;
-                throw new Error(message);
-            } else {
-                /* eslint-disable no-param-reassign */
-                migration.up = up.replace(/^-- .*?$/gm, '').trim();// Remove comments
-                migration.down = down.trim(); // and trim whitespaces
-            }
-        });
-
-        // Create a database table for migrations meta data if it doesn't exist
-        await this.run(`CREATE TABLE IF NOT EXISTS "${table}" (
-    id   INTEGER PRIMARY KEY,
-    name TEXT    NOT NULL,
-    up   TEXT    NOT NULL,
-    down TEXT    NOT NULL
-)`);
-
-        // Get the list of already applied migrations
-        let dbMigrations = await this.all(
-            `SELECT id, name, up, down FROM "${table}" ORDER BY id ASC`,
-        );
-
-        // Undo migrations that exist only in the database but not in migs,
-        // also undo the last migration if the `force` option was set to `last`.
-        const lastMigration = migrations[migrations.length - 1];
-        for (const migration of dbMigrations.slice().sort((a, b) => Math.sign(b.id - a.id))) {
-            if (!migrations.some(x => x.id === migration.id) ||
-                (force === 'last' && migration.id === lastMigration.id)) {
-                const dbh = await this.get();
-                await dbh.run('BEGIN');
-                try {
-                    await dbh.exec(migration.down);
-                    await dbh.run(SQL`DELETE FROM "`.append(table).append(SQL`" WHERE id = ${migration.id}`));
-                    await dbh.run('COMMIT');
-                    dbMigrations = dbMigrations.filter(x => x.id !== migration.id);
-                } catch (err) {
-                    await dbh.run('ROLLBACK');
-                    throw err;
-                } finally {
-                    dbh.ret();
-                }
-            } else {
-                break;
-            }
-        }
-
-        // Apply pending migrations
-        let applied = [];
-        const lastMigrationId = dbMigrations.length ? dbMigrations[dbMigrations.length - 1].id : 0;
-        for (const migration of migrations) {
-            if (migration.id > lastMigrationId) {
-                const dbh = await this.get();
-                await dbh.run('BEGIN');
-                try {
-                    await dbh.exec(migration.up);
-                    await dbh.run(SQL`INSERT INTO "`.append(table).append(
-                        SQL`" (id, name, up, down) VALUES (${migration.id}, ${migration.name}, ${migration.up}, ${migration.down})`)
-                    );
-                    await dbh.run('COMMIT');
-                    applied.push(migration.id);
-                } catch (err) {
-                    await dbh.run('ROLLBACK');
-                    throw err;
-                } finally {
-                    dbh.ret();
-                }
-            }
-        }
-
-        return applied;
-    }
-}
-
-module.exports = SqliteConnectionPool;

server/db/jembaMigrations/app/002-create.js

@@ -0,0 +1,22 @@
+module.exports = {
+    up: [
+        ['create', {
+            /*{
+                id, // book URL
+                queryTime: Number,
+                checkTime: Number, // 0 - never checked
+                modTime: String,
+                size: Number,
+                checkSum: String, //sha256
+                state: Number, // 0 - not processing, 1 - processing
+                error: String,
+            }*/
+            table: 'buc'
+        }],
+    ],    
+    down: [
+        ['drop', {
+            table: 'buc'
+        }],
+    ]
+};

server/db/jembaMigrations/app/index.js

@@ -1,6 +1,7 @@
 module.exports = {
     table: 'migration1',
     data: [
-        {id: 1, name: 'create', data: require('./001-create')}
+        {id: 1, name: 'create', data: require('./001-create')},
+        {id: 2, name: 'create', data: require('./002-create')},
     ]
 }

server/db/jembaMigrations/book-update-server/001-create.js

@@ -1,7 +1,20 @@
 module.exports = {
     up: [
         ['create', {
-            table: 'checked',
+            /*{
+                id, // book URL
+                queryTime: Number,
+                checkTime: Number, // 0 - never checked
+                modTime: String,
+                size: Number,
+                checkSum: String, //sha256
+                state: Number, // 0 - not processing, 1 - processing
+                error: String,
+            }*/
+            table: 'buc',
+            flag: [
+                {name: 'notProcessing', check: `(r) => r.state === 0`},
+            ],
             index: [
                 {field: 'queryTime', type: 'number'},
                 {field: 'checkTime', type: 'number'},
@@ -10,7 +23,7 @@ module.exports = {
     ],    
     down: [
         ['drop', {
-            table: 'checked'
+            table: 'buc'
         }],
     ]
 };

server/db/migrations/app/index.js

@@ -1,5 +0,0 @@
-module.exports = {
-    table: 'migration1',
-    data: [
-    ]
-}

server/db/migrations/readerStorage/001-create.js

@@ -1,7 +0,0 @@
-module.exports = `
--- Up
-CREATE TABLE storage (id TEXT PRIMARY KEY, rev INTEGER, time INTEGER, data TEXT);
-
--- Down
-DROP TABLE storage;
-`;

server/db/migrations/readerStorage/index.js

@@ -1,6 +0,0 @@
-module.exports = {
-    table: 'migration1',
-    data: [
-        {id: 1, name: 'create', data: require('./001-create')}
-    ]
-}

server/index.js

@@ -5,6 +5,7 @@ const argv = require('minimist')(process.argv.slice(2));
 const express = require('express');
 const compression = require('compression');
 const http = require('http');
+const https = require('https');
 const WebSocket = require ('ws');
 
 const ayncExit = new (require('./core/AsyncExit'))();
@@ -45,15 +46,8 @@ async function init() {
     }
 
     //connections
-    const connManager = new (require('./db/ConnManager'))();//singleton
-    await connManager.init(config);
-
     const jembaConnManager = new (require('./db/JembaConnManager'))();//singleton
     await jembaConnManager.init(config, argv['auto-repair']);
-
-    //converter SQLITE => JembaDb
-    const converter = new  (require('./db/Converter'))();
-    await converter.run(config);
 }
 
 async function main() {
@@ -64,7 +58,15 @@ async function main() {
     for (let serverCfg of config.servers) {
         if (serverCfg.mode !== 'none') {
             const app = express();
-            const server = http.createServer(app);
+            let server;
+            if (serverCfg.isHttps) {
+                const key = fs.readFileSync(`${config.dataDir}/${serverCfg.keysFile}.key`);
+                const cert = fs.readFileSync(`${config.dataDir}/${serverCfg.keysFile}.crt`);
+
+                server = https.createServer({key, cert}, app);
+            } else {
+                server = http.createServer(app);
+            }
             const wss = new WebSocket.Server({ server, maxPayload: maxPayloadSize*1024*1024 });
 
             const serverConfig = Object.assign({}, config, serverCfg);
@@ -93,7 +95,7 @@ async function main() {
             }
 
             server.listen(serverConfig.port, serverConfig.ip, function() {
-                log(`Server-${serverConfig.serverName} is ready on ${serverConfig.ip}:${serverConfig.port}, mode: ${serverConfig.mode}`);
+                log(`Server "${serverConfig.serverName}" is ready on ${(serverConfig.isHttps ? 'https://' : 'http://')}${serverConfig.ip}:${serverConfig.port}, mode: ${serverConfig.mode}`);
             });
         }
     }