Работа над проектом

2022-08-26 17:37:41 +07:00
parent 2c51bfaf96
commit a1cdd6b116
8 changed files with 155 additions and 23 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /node_modules
 /server/.inpx-web
+/server/.inpx-web-bak
 /dist
--- a/client/components/Api/Api.vue
+++ b/client/components/Api/Api.vue
@@ -1,5 +1,28 @@
 <template>
    <div>
+        <q-dialog v-model="busyDialogVisible" no-route-dismiss no-esc-dismiss no-backdrop-dismiss>
+            <div class="q-pa-lg bg-white column" style="width: 400px">
+                <div style="font-weight: bold; font-size: 120%;">
+                    {{ mainMessage }}
+                </div>
+
+                <div v-show="jobMessage" class="q-mt-sm" style="width: 350px; white-space: nowrap; overflow: hidden">
+                    {{ jobMessage }}
+                </div>
+                <div v-show="jobMessage">
+                    <q-linear-progress stripe rounded size="30px" :value="progress" color="green">
+                        <div class="absolute-full flex flex-center">
+                            <div class="text-black bg-white" style="font-size: 10px; padding: 1px 4px 1px 4px; border-radius: 4px">
+                                {{ (progress*100).toFixed(2) }}%
+                            </div>
+                        </div>
+                    </q-linear-progress>
+                </div>
+                <!--div class="q-ml-sm">
+                    {{ jsonMessage }}
+                </div-->                
+            </div>
+        </q-dialog>
    </div>
 </template>

@@ -7,9 +30,27 @@
 //-----------------------------------------------------------------------------
 import vueComponent from '../vueComponent.js';

-import wsc from './webSocketConnection';
 //import _ from 'lodash';

+import wsc from './webSocketConnection';
+import * as utils from '../../share/utils';
+
+const rotor = '|/-\\';
+const stepBound = [
+    0,
+    0,//1
+    18,//2
+    20,//3
+    70,//4
+    82,//5
+    84,//6
+    88,//7
+    90,//8
+    98,//9
+    99,//10
+    100,//11
+];
+
 const componentOptions = {
    components: {
    },
@@ -18,6 +59,11 @@ const componentOptions = {
 };
 class Api {
    _options = componentOptions;
+    busyDialogVisible = false;
+    mainMessage = '';
+    jobMessage = '';
+    //jsonMessage = '';
+    progress = 0;

    created() {
        this.commit = this.$store.commit;
@@ -40,8 +86,54 @@ class Api {
        return this.$store.state.config;
    }

+    async showBusyDialog() {
+        this.mainMessage = '';
+        this.jobMessage = '';
+        this.busyDialogVisible = true;
+        try {
+            let ri = 0;
+            while (1) {// eslint-disable-line
+                const server = await wsc.message(await wsc.send({action: 'get-worker-state', workerId: 'server_state'}));
+
+                if (server.state != 'normal') {
+                    this.mainMessage = `${server.serverMessage} ${rotor[ri]}`;
+                    if (server.job == 'load inpx') {
+                        this.jobMessage = `${server.jobMessage} (${server.recsLoaded}): ${server.fileName}`;
+                    } else {
+                        this.jobMessage = server.jobMessage;
+                    }
+
+                    //this.jsonMessage = server;
+
+                    const jStep = server.jobStep;
+
+                    if (jStep && stepBound[jStep] !== undefined) {
+                        const sp = server.progress || 0;
+                        const delta = stepBound[jStep + 1] - stepBound[jStep];
+                        this.progress = (stepBound[jStep] + sp*delta)/100;
+                    }
+                } else {
+                    break;
+                }
+
+                await utils.sleep(300);
+                ri = (ri < rotor.length - 1 ? ri + 1 : 0);
+            }
+        } finally {
+            this.busyDialogVisible = false;
+        }
+    }
+
    async request(params) {
-        return await wsc.message(await wsc.send(params));
+        while (1) {// eslint-disable-line
+            const response = await wsc.message(await wsc.send(params));
+
+            if (response && response.error == 'server_busy') {
+                await this.showBusyDialog();
+            } else {
+                return response;
+            }
+        }
    }

    async search(query) {
--- a/client/components/Search/Search.vue
+++ b/client/components/Search/Search.vue
@@ -32,7 +32,7 @@
                        </div>
                    </div>
                    
-                    <DivBtn class="q-ml-md text-white bg-secondary" :size="32" :icon-size="24" icon="la la-cog" round @click="settingsDialogVisible = true" />
+                    <DivBtn class="q-ml-md text-white bg-secondary" :size="30" :icon-size="24" :imt="1" icon="la la-cog" round @click="settingsDialogVisible = true" />
                </div>
                <div class="row q-mx-md q-mb-sm items-center">
                    <q-input
@@ -63,7 +63,7 @@
                    />
                    <div class="q-mx-xs" />                

-                    <DivBtn class="text-white bg-grey-13" :size="32" :icon-size="24" icon="la la-question" round @click="showSearchHelp" />
+                    <DivBtn class="text-white bg-grey-13" :size="30" :icon-size="24" icon="la la-question" round @click="showSearchHelp" />

                    <div class="q-mx-xs" />
                    <div class="row items-center q-mt-xs">
--- a/client/components/Search/authorBooksStorage.js
+++ b/client/components/Search/authorBooksStorage.js
@@ -2,7 +2,7 @@ import localForage from 'localforage';
 //import _ from 'lodash';
 import * as utils from '../../share/utils';

-const maxDataSize = 100*1024*1024;
+const maxDataSize = 100*1024*1024;//100 Mb

 const abStore = localForage.createInstance({
    name: 'authorBooksStorage'
--- a/client/components/share/DivBtn.vue
+++ b/client/components/share/DivBtn.vue
@@ -1,7 +1,7 @@
 <template>
    <div ref="btn" class="button clickable row justify-center items-center" @click="clickEffect">
        <div class="row justify-center items-center no-wrap" :class="{'button-pressed': pressed}">
-            <q-icon :name="icon" :size="`${iconSize}px`" />
+            <i :class="icon" :style="`font-size: ${iconSize}px; margin-top: ${imt}px`" />
            <slot></slot>
        </div>
    </div>
@@ -29,6 +29,7 @@ class DivBtn {
        icon: { type: String, default: '' },
        iconSize: { type: Number, default: 14 },
        round: { type: Boolean },
+        imt:  { type: Number, default: 0 },// icon margin top
    };

    pressed = false;
--- a/client/quasar.js
+++ b/client/quasar.js
@@ -10,6 +10,7 @@ const config = {};
 //import {QDrawer} from 'quasar/src/components/drawer';

 //import {QCircularProgress} from 'quasar/src/components/circular-progress';
+import {QLinearProgress} from 'quasar/src/components/linear-progress';
 import {QInput} from 'quasar/src/components/input';
 import {QBtn} from 'quasar/src/components/btn';
 //import {QBtnGroup} from 'quasar/src/components/btn-group';
@@ -41,6 +42,7 @@ const components = {
    //QDrawer,

    //QCircularProgress,
+    QLinearProgress,
    QInput,
    QBtn,
    //QBtnGroup,
--- a/server/core/DbCreator.js
+++ b/server/core/DbCreator.js
@@ -12,10 +12,8 @@ class DbCreator {
    async run(db, callback) {
        const config = this.config;

-        callback({job: 'load inpx', jobMessage: 'Загрузка INPX'});
-        const readFileCallback = async(readState) => {
-            callback(readState);
-        };
+        callback({jobStepCount: 5});
+        callback({job: 'load inpx', jobMessage: 'Загрузка INPX', jobStep: 1, progress: 0});

        //временная таблица
        await db.create({
@@ -43,6 +41,7 @@ class DbCreator {

        //stuff
        let recsLoaded = 0;
+        callback({recsLoaded});
        let chunkNum = 0;

        const splitAuthor = (author) => {
@@ -57,6 +56,17 @@ class DbCreator {
            return result;
        }

+        let totalFiles = 0;
+        const readFileCallback = async(readState) => {
+            callback(readState);
+
+            if (readState.totalFiles)
+                totalFiles = readState.totalFiles;
+
+            if (totalFiles)
+                callback({progress: (readState.current || 0)/totalFiles});
+        };
+
        let id = 0;
        const parsedCallback = async(chunk) => {
            for (const rec of chunk) {
@@ -123,7 +133,8 @@ class DbCreator {

        //отсортируем авторов и выдадим им правильные id
        //порядок id соответствует ASC-сортировке по author.toLowerCase
-        callback({job: 'author sort', jobMessage: 'Сортировка'});
+        callback({job: 'author sort', jobMessage: 'Сортировка авторов', jobStep: 2, progress: 0});
+        await utils.sleep(100);
        authorArr.sort((a, b) => a.value.localeCompare(b.value));

        id = 0;
@@ -137,7 +148,9 @@ class DbCreator {
        utils.freeMemory();

        //подготовка к сохранению author_book
-        const saveBookChunk = async(authorChunk) => {
+        const saveBookChunk = async(authorChunk, callback) => {
+            callback(0);
+
            const ids = [];
            for (const a of authorChunk) {
                for (const id of a.bookId) {
@@ -147,7 +160,11 @@ class DbCreator {

            ids.sort();// обязательно, иначе будет тормозить - особенности JembaDb

+            callback(0.1);
            const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`});
+            callback(0.6);
+            await utils.sleep(100);
+
            const bookArr = new Map();
            for (const row of rows)
                bookArr.set(row.id, row);
@@ -165,13 +182,15 @@ class DbCreator {
                delete a.bookId;//в дальнейшем не понадобится, authorArr сохраняем без него
            }

+            callback(0.7);
            await db.insert({
                table: 'author_book',
                rows: abRows,
            });
+            callback(1);
        };

-        callback({job: 'search tables create', jobMessage: 'Создание поисковых таблиц'});        
+        callback({job: 'book sort', jobMessage: 'Сортировка книг', jobStep: 3, progress: 0});

        //сохранение author_book
        await db.create({
@@ -180,12 +199,19 @@ class DbCreator {

        let idsLen = 0;
        let aChunk = [];
-        for (const author of authorArr) {// eslint-disable-line
+        let prevI = 0;
+        for (let i = 0; i < authorArr.length; i++) {// eslint-disable-line
+            const author = authorArr[i];
+
            aChunk.push(author);
            idsLen += author.bookId.length;

            if (idsLen > 50000) {//константа выяснена эмпирическим путем "память/скорость"
-                await saveBookChunk(aChunk);
+                await saveBookChunk(aChunk, (p) => {
+                    callback({progress: (prevI + (i - prevI)*p)/authorArr.length});
+                });
+
+                prevI = i;
                idsLen = 0;
                aChunk = [];
                await utils.sleep(100);
@@ -194,10 +220,12 @@ class DbCreator {
            }
        }
        if (aChunk.length) {
-            await saveBookChunk(aChunk);
+            await saveBookChunk(aChunk, () => {});
            aChunk = null;
        }

+        callback({progress: 1});
+
        //чистка памяти, ибо жрет как не в себя
        await db.drop({table: 'book'});
        await db.freeMemory();
@@ -267,7 +295,10 @@ class DbCreator {
            parseField(rec.lang, langMap, langArr, authorIds);
        };

+        callback({job: 'search tables create', jobMessage: 'Создание поисковых таблиц', jobStep: 4, progress: 0});
+
        //парсинг 2, теперь можно создавать остальные поисковые таблицы
+        let proc = 0;
        while (1) {// eslint-disable-line
            const rows = await db.select({
                table: 'author_book',
@@ -295,6 +326,9 @@ class DbCreator {
                    for (const rec of books)
                        parseBookRec(rec);
                }
+
+                proc += rows.length;
+                callback({progress: proc/authorArr.length});
            } else
                break;

@@ -312,7 +346,7 @@ class DbCreator {
        utils.freeMemory();

        //config
-        callback({job: 'config save', jobMessage: 'Сохранение конфигурации'});
+        callback({job: 'config save', jobMessage: 'Сохранение конфигурации', jobStep: 5, progress: 0});
        await db.create({
            table: 'config'
        });
@@ -367,6 +401,8 @@ class DbCreator {
                    await db.freeMemory();
                    await utils.sleep(100);
                }
+
+                callback({progress: i/arr.length});                
            }

            nullArr();
@@ -375,23 +411,23 @@ class DbCreator {
        };

        //author
-        callback({job: 'author save', jobMessage: 'Сохранение индекса авторов'});
+        callback({job: 'author save', jobMessage: 'Сохранение индекса авторов', jobStep: 6, progress: 0});
        await saveTable('author', authorArr, () => {authorArr = null}, false);

        //series
-        callback({job: 'series save', jobMessage: 'Сохранение индекса серий'});
+        callback({job: 'series save', jobMessage: 'Сохранение индекса серий', jobStep: 7, progress: 0});
        await saveTable('series', seriesArr, () => {seriesArr = null});

        //title
-        callback({job: 'title save', jobMessage: 'Сохранение индекса названий'});
+        callback({job: 'title save', jobMessage: 'Сохранение индекса названий', jobStep: 8, progress: 0});
        await saveTable('title', titleArr, () => {titleArr = null});

        //genre
-        callback({job: 'genre save', jobMessage: 'Сохранение индекса жанров'});
+        callback({job: 'genre save', jobMessage: 'Сохранение индекса жанров', jobStep: 9, progress: 0});
        await saveTable('genre', genreArr, () => {genreArr = null});

        //lang
-        callback({job: 'lang save', jobMessage: 'Сохранение индекса языков'});
+        callback({job: 'lang save', jobMessage: 'Сохранение индекса языков', jobStep: 10, progress: 0});
        await saveTable('lang', langArr, () => {langArr = null});

        //кэш-таблицы запросов
--- a/server/core/InpxParser.js
+++ b/server/core/InpxParser.js
@@ -46,7 +46,7 @@ class InpxParser {
            }            

            //плюс 3 файла .info
-            await readFileCallback({total: inpFiles.length + 3});
+            await readFileCallback({totalFiles: inpFiles.length + 3});

            let current = 0;
            //info