From a1cdd6b116f5baed1d9a9053612ac47552c907ff Mon Sep 17 00:00:00 2001
From: Book Pauk <bookpauk@gmail.com>
Date: Fri, 26 Aug 2022 17:37:41 +0700
Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20=D0=BD?=
 =?UTF-8?q?=D0=B0=D0=B4=20=D0=BF=D1=80=D0=BE=D0=B5=D0=BA=D1=82=D0=BE=D0=BC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitignore                                    |  1 +
 client/components/Api/Api.vue                 | 96 ++++++++++++++++++-
 client/components/Search/Search.vue           |  4 +-
 .../components/Search/authorBooksStorage.js   |  2 +-
 client/components/share/DivBtn.vue            |  3 +-
 client/quasar.js                              |  2 +
 server/core/DbCreator.js                      | 68 +++++++++----
 server/core/InpxParser.js                     |  2 +-
 8 files changed, 155 insertions(+), 23 deletions(-)

diff --git a/.gitignore b/.gitignore
index bec7510..121c230 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /node_modules
 /server/.inpx-web
+/server/.inpx-web-bak
 /dist
diff --git a/client/components/Api/Api.vue b/client/components/Api/Api.vue
index 8244677..ade4a3d 100644
--- a/client/components/Api/Api.vue
+++ b/client/components/Api/Api.vue
@@ -1,5 +1,28 @@
 <template>
     <div>
+        <q-dialog v-model="busyDialogVisible" no-route-dismiss no-esc-dismiss no-backdrop-dismiss>
+            <div class="q-pa-lg bg-white column" style="width: 400px">
+                <div style="font-weight: bold; font-size: 120%;">
+                    {{ mainMessage }}
+                </div>
+
+                <div v-show="jobMessage" class="q-mt-sm" style="width: 350px; white-space: nowrap; overflow: hidden">
+                    {{ jobMessage }}
+                </div>
+                <div v-show="jobMessage">
+                    <q-linear-progress stripe rounded size="30px" :value="progress" color="green">
+                        <div class="absolute-full flex flex-center">
+                            <div class="text-black bg-white" style="font-size: 10px; padding: 1px 4px 1px 4px; border-radius: 4px">
+                                {{ (progress*100).toFixed(2) }}%
+                            </div>
+                        </div>
+                    </q-linear-progress>
+                </div>
+                <!--div class="q-ml-sm">
+                    {{ jsonMessage }}
+                </div-->                
+            </div>
+        </q-dialog>
     </div>
 </template>
 
@@ -7,9 +30,27 @@
 //-----------------------------------------------------------------------------
 import vueComponent from '../vueComponent.js';
 
-import wsc from './webSocketConnection';
 //import _ from 'lodash';
 
+import wsc from './webSocketConnection';
+import * as utils from '../../share/utils';
+
+const rotor = '|/-\\';
+const stepBound = [
+    0,
+    0,//1
+    18,//2
+    20,//3
+    70,//4
+    82,//5
+    84,//6
+    88,//7
+    90,//8
+    98,//9
+    99,//10
+    100,//11
+];
+
 const componentOptions = {
     components: {
     },
@@ -18,6 +59,11 @@ const componentOptions = {
 };
 class Api {
     _options = componentOptions;
+    busyDialogVisible = false;
+    mainMessage = '';
+    jobMessage = '';
+    //jsonMessage = '';
+    progress = 0;
 
     created() {
         this.commit = this.$store.commit;
@@ -40,8 +86,54 @@ class Api {
         return this.$store.state.config;
     }
 
+    async showBusyDialog() {
+        this.mainMessage = '';
+        this.jobMessage = '';
+        this.busyDialogVisible = true;
+        try {
+            let ri = 0;
+            while (1) {// eslint-disable-line
+                const server = await wsc.message(await wsc.send({action: 'get-worker-state', workerId: 'server_state'}));
+
+                if (server.state != 'normal') {
+                    this.mainMessage = `${server.serverMessage} ${rotor[ri]}`;
+                    if (server.job == 'load inpx') {
+                        this.jobMessage = `${server.jobMessage} (${server.recsLoaded}): ${server.fileName}`;
+                    } else {
+                        this.jobMessage = server.jobMessage;
+                    }
+
+                    //this.jsonMessage = server;
+
+                    const jStep = server.jobStep;
+
+                    if (jStep && stepBound[jStep] !== undefined) {
+                        const sp = server.progress || 0;
+                        const delta = stepBound[jStep + 1] - stepBound[jStep];
+                        this.progress = (stepBound[jStep] + sp*delta)/100;
+                    }
+                } else {
+                    break;
+                }
+
+                await utils.sleep(300);
+                ri = (ri < rotor.length - 1 ? ri + 1 : 0);
+            }
+        } finally {
+            this.busyDialogVisible = false;
+        }
+    }
+
     async request(params) {
-        return await wsc.message(await wsc.send(params));
+        while (1) {// eslint-disable-line
+            const response = await wsc.message(await wsc.send(params));
+
+            if (response && response.error == 'server_busy') {
+                await this.showBusyDialog();
+            } else {
+                return response;
+            }
+        }
     }
 
     async search(query) {
diff --git a/client/components/Search/Search.vue b/client/components/Search/Search.vue
index 9405510..524229f 100644
--- a/client/components/Search/Search.vue
+++ b/client/components/Search/Search.vue
@@ -32,7 +32,7 @@
                         </div>
                     </div>
                     
-                    <DivBtn class="q-ml-md text-white bg-secondary" :size="32" :icon-size="24" icon="la la-cog" round @click="settingsDialogVisible = true" />
+                    <DivBtn class="q-ml-md text-white bg-secondary" :size="30" :icon-size="24" :imt="1" icon="la la-cog" round @click="settingsDialogVisible = true" />
                 </div>
                 <div class="row q-mx-md q-mb-sm items-center">
                     <q-input
@@ -63,7 +63,7 @@
                     />
                     <div class="q-mx-xs" />                
 
-                    <DivBtn class="text-white bg-grey-13" :size="32" :icon-size="24" icon="la la-question" round @click="showSearchHelp" />
+                    <DivBtn class="text-white bg-grey-13" :size="30" :icon-size="24" icon="la la-question" round @click="showSearchHelp" />
 
                     <div class="q-mx-xs" />
                     <div class="row items-center q-mt-xs">
diff --git a/client/components/Search/authorBooksStorage.js b/client/components/Search/authorBooksStorage.js
index b7a85bc..6e6be34 100644
--- a/client/components/Search/authorBooksStorage.js
+++ b/client/components/Search/authorBooksStorage.js
@@ -2,7 +2,7 @@ import localForage from 'localforage';
 //import _ from 'lodash';
 import * as utils from '../../share/utils';
 
-const maxDataSize = 100*1024*1024;
+const maxDataSize = 100*1024*1024;//100 Mb
 
 const abStore = localForage.createInstance({
     name: 'authorBooksStorage'
diff --git a/client/components/share/DivBtn.vue b/client/components/share/DivBtn.vue
index 84f916c..f558039 100644
--- a/client/components/share/DivBtn.vue
+++ b/client/components/share/DivBtn.vue
@@ -1,7 +1,7 @@
 <template>
     <div ref="btn" class="button clickable row justify-center items-center" @click="clickEffect">
         <div class="row justify-center items-center no-wrap" :class="{'button-pressed': pressed}">
-            <q-icon :name="icon" :size="`${iconSize}px`" />
+            <i :class="icon" :style="`font-size: ${iconSize}px; margin-top: ${imt}px`" />
             <slot></slot>
         </div>
     </div>
@@ -29,6 +29,7 @@ class DivBtn {
         icon: { type: String, default: '' },
         iconSize: { type: Number, default: 14 },
         round: { type: Boolean },
+        imt:  { type: Number, default: 0 },// icon margin top
     };
 
     pressed = false;
diff --git a/client/quasar.js b/client/quasar.js
index 73db3d3..a2a3243 100644
--- a/client/quasar.js
+++ b/client/quasar.js
@@ -10,6 +10,7 @@ const config = {};
 //import {QDrawer} from 'quasar/src/components/drawer';
 
 //import {QCircularProgress} from 'quasar/src/components/circular-progress';
+import {QLinearProgress} from 'quasar/src/components/linear-progress';
 import {QInput} from 'quasar/src/components/input';
 import {QBtn} from 'quasar/src/components/btn';
 //import {QBtnGroup} from 'quasar/src/components/btn-group';
@@ -41,6 +42,7 @@ const components = {
     //QDrawer,
 
     //QCircularProgress,
+    QLinearProgress,
     QInput,
     QBtn,
     //QBtnGroup,
diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js
index 457be2b..2e750a1 100644
--- a/server/core/DbCreator.js
+++ b/server/core/DbCreator.js
@@ -12,10 +12,8 @@ class DbCreator {
     async run(db, callback) {
         const config = this.config;
 
-        callback({job: 'load inpx', jobMessage: 'Загрузка INPX'});
-        const readFileCallback = async(readState) => {
-            callback(readState);
-        };
+        callback({jobStepCount: 5});
+        callback({job: 'load inpx', jobMessage: 'Загрузка INPX', jobStep: 1, progress: 0});
 
         //временная таблица
         await db.create({
@@ -43,6 +41,7 @@ class DbCreator {
 
         //stuff
         let recsLoaded = 0;
+        callback({recsLoaded});
         let chunkNum = 0;
 
         const splitAuthor = (author) => {
@@ -57,6 +56,17 @@ class DbCreator {
             return result;
         }
 
+        let totalFiles = 0;
+        const readFileCallback = async(readState) => {
+            callback(readState);
+
+            if (readState.totalFiles)
+                totalFiles = readState.totalFiles;
+
+            if (totalFiles)
+                callback({progress: (readState.current || 0)/totalFiles});
+        };
+
         let id = 0;
         const parsedCallback = async(chunk) => {
             for (const rec of chunk) {
@@ -123,7 +133,8 @@ class DbCreator {
 
         //отсортируем авторов и выдадим им правильные id
         //порядок id соответствует ASC-сортировке по author.toLowerCase
-        callback({job: 'author sort', jobMessage: 'Сортировка'});
+        callback({job: 'author sort', jobMessage: 'Сортировка авторов', jobStep: 2, progress: 0});
+        await utils.sleep(100);
         authorArr.sort((a, b) => a.value.localeCompare(b.value));
 
         id = 0;
@@ -137,7 +148,9 @@ class DbCreator {
         utils.freeMemory();
 
         //подготовка к сохранению author_book
-        const saveBookChunk = async(authorChunk) => {
+        const saveBookChunk = async(authorChunk, callback) => {
+            callback(0);
+
             const ids = [];
             for (const a of authorChunk) {
                 for (const id of a.bookId) {
@@ -147,7 +160,11 @@ class DbCreator {
 
             ids.sort();// обязательно, иначе будет тормозить - особенности JembaDb
 
+            callback(0.1);
             const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
+            callback(0.6);
+            await utils.sleep(100);
+
             const bookArr = new Map();
             for (const row of rows)
                 bookArr.set(row.id, row);
@@ -165,13 +182,15 @@ class DbCreator {
                 delete a.bookId;//в дальнейшем не понадобится, authorArr сохраняем без него
             }
 
+            callback(0.7);
             await db.insert({
                 table: 'author_book',
                 rows: abRows,
             });
+            callback(1);
         };
 
-        callback({job: 'search tables create', jobMessage: 'Создание поисковых таблиц'});        
+        callback({job: 'book sort', jobMessage: 'Сортировка книг', jobStep: 3, progress: 0});
 
         //сохранение author_book
         await db.create({
@@ -180,12 +199,19 @@ class DbCreator {
 
         let idsLen = 0;
         let aChunk = [];
-        for (const author of authorArr) {// eslint-disable-line
+        let prevI = 0;
+        for (let i = 0; i < authorArr.length; i++) {// eslint-disable-line
+            const author = authorArr[i];
+
             aChunk.push(author);
             idsLen += author.bookId.length;
 
             if (idsLen > 50000) {//константа выяснена эмпирическим путем "память/скорость"
-                await saveBookChunk(aChunk);
+                await saveBookChunk(aChunk, (p) => {
+                    callback({progress: (prevI + (i - prevI)*p)/authorArr.length});
+                });
+
+                prevI = i;
                 idsLen = 0;
                 aChunk = [];
                 await utils.sleep(100);
@@ -194,10 +220,12 @@ class DbCreator {
             }
         }
         if (aChunk.length) {
-            await saveBookChunk(aChunk);
+            await saveBookChunk(aChunk, () => {});
             aChunk = null;
         }
 
+        callback({progress: 1});
+
         //чистка памяти, ибо жрет как не в себя
         await db.drop({table: 'book'});
         await db.freeMemory();
@@ -267,7 +295,10 @@ class DbCreator {
             parseField(rec.lang, langMap, langArr, authorIds);
         };
 
+        callback({job: 'search tables create', jobMessage: 'Создание поисковых таблиц', jobStep: 4, progress: 0});
+
         //парсинг 2, теперь можно создавать остальные поисковые таблицы
+        let proc = 0;
         while (1) {// eslint-disable-line
             const rows = await db.select({
                 table: 'author_book',
@@ -295,6 +326,9 @@ class DbCreator {
                     for (const rec of books)
                         parseBookRec(rec);
                 }
+
+                proc += rows.length;
+                callback({progress: proc/authorArr.length});
             } else
                 break;
 
@@ -312,7 +346,7 @@ class DbCreator {
         utils.freeMemory();
 
         //config
-        callback({job: 'config save', jobMessage: 'Сохранение конфигурации'});
+        callback({job: 'config save', jobMessage: 'Сохранение конфигурации', jobStep: 5, progress: 0});
         await db.create({
             table: 'config'
         });
@@ -367,6 +401,8 @@ class DbCreator {
                     await db.freeMemory();
                     await utils.sleep(100);
                 }
+
+                callback({progress: i/arr.length});                
             }
 
             nullArr();
@@ -375,23 +411,23 @@ class DbCreator {
         };
 
         //author
-        callback({job: 'author save', jobMessage: 'Сохранение индекса авторов'});
+        callback({job: 'author save', jobMessage: 'Сохранение индекса авторов', jobStep: 6, progress: 0});
         await saveTable('author', authorArr, () => {authorArr = null}, false);
 
         //series
-        callback({job: 'series save', jobMessage: 'Сохранение индекса серий'});
+        callback({job: 'series save', jobMessage: 'Сохранение индекса серий', jobStep: 7, progress: 0});
         await saveTable('series', seriesArr, () => {seriesArr = null});
 
         //title
-        callback({job: 'title save', jobMessage: 'Сохранение индекса названий'});
+        callback({job: 'title save', jobMessage: 'Сохранение индекса названий', jobStep: 8, progress: 0});
         await saveTable('title', titleArr, () => {titleArr = null});
 
         //genre
-        callback({job: 'genre save', jobMessage: 'Сохранение индекса жанров'});
+        callback({job: 'genre save', jobMessage: 'Сохранение индекса жанров', jobStep: 9, progress: 0});
         await saveTable('genre', genreArr, () => {genreArr = null});
 
         //lang
-        callback({job: 'lang save', jobMessage: 'Сохранение индекса языков'});
+        callback({job: 'lang save', jobMessage: 'Сохранение индекса языков', jobStep: 10, progress: 0});
         await saveTable('lang', langArr, () => {langArr = null});
 
         //кэш-таблицы запросов
diff --git a/server/core/InpxParser.js b/server/core/InpxParser.js
index 7f968e9..ae455fa 100644
--- a/server/core/InpxParser.js
+++ b/server/core/InpxParser.js
@@ -46,7 +46,7 @@ class InpxParser {
             }            
 
             //плюс 3 файла .info
-            await readFileCallback({total: inpFiles.length + 3});
+            await readFileCallback({totalFiles: inpFiles.length + 3});
 
             let current = 0;
             //info