diff --git a/client/components/Api/Api.vue b/client/components/Api/Api.vue index deae3aa..742eecc 100644 --- a/client/components/Api/Api.vue +++ b/client/components/Api/Api.vue @@ -44,14 +44,14 @@ const stepBound = [ 0,// jobStep = 1 18,// jobStep = 2 20,// jobStep = 3 - 60,// jobStep = 4 - 72,// jobStep = 5 - 72,// jobStep = 6 - 74,// jobStep = 7 - 75,// jobStep = 8 - 79,// jobStep = 9 - 79,// jobStep = 10 - 80,// jobStep = 11 + 50,// jobStep = 4 + 62,// jobStep = 5 + 62,// jobStep = 6 + 64,// jobStep = 7 + 65,// jobStep = 8 + 69,// jobStep = 9 + 69,// jobStep = 10 + 70,// jobStep = 11 95,// jobStep = 12 100,// jobStep = 13 ]; diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 0327858..b350afb 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -425,8 +425,8 @@ class DbCreator { } else break; - await utils.sleep(100); if (config.lowMemoryMode) { + await utils.sleep(100); utils.freeMemory(); await db.freeMemory(); } @@ -441,7 +441,7 @@ class DbCreator { utils.freeMemory(); //сортировка серий - callback({job: 'series sort', jobMessage: 'Сортировка серий', jobStep: 5, progress: 0}); + callback({job: 'sort', jobMessage: 'Сортировка', jobStep: 5, progress: 0}); await utils.sleep(100); seriesArr.sort((a, b) => a.value.localeCompare(b.value)); await utils.sleep(100); @@ -559,12 +559,12 @@ class DbCreator { callback({job: 'optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0}); await this.optimizeTable('series', 'series_book', 'series', db, (p) => { if (p.progress) - p.progress = 0.5*p.progress; + p.progress = 0.2*p.progress; callback(p); }); await this.optimizeTable('title', 'title_book', 'title', db, (p) => { if (p.progress) - p.progress = 0.5*(1 + p.progress); + p.progress = 0.2 + 0.8*p.progress; callback(p); }); @@ -649,33 +649,47 @@ class DbCreator { }); }; - const rows = await db.select({table: from}); + const rows = await db.select({table: from, count: true}); + const fromLength = rows[0].count; - let idsLen = 0; - let chunk = []; let processed = 0; - for (const row of rows) {// eslint-disable-line - chunk.push(row); - idsLen += row.bookId.length; - processed++; + while (1) {// eslint-disable-line + const chunk = await db.select({ + table: from, + where: ` + let iter = @getItem('optimize'); + if (!iter) { + iter = @all(); + @setItem('optimize', iter); + } - if (idsLen > 20000) {//константа выяснена эмпирическим путем "память/скорость" + const ids = new Set(); + let id = iter.next(); + while (!id.done) { + ids.add(id.value); + if (ids.size >= 20000) + break; + id = iter.next(); + } + + return ids; + ` + }); + + if (chunk.length) { await saveChunk(chunk); - idsLen = 0; - chunk = []; + processed += chunk.length; + callback({progress: processed/fromLength}); + } else + break; - callback({progress: processed/rows.length}); - - await utils.sleep(100); + if (this.config.lowMemoryMode) { + await utils.sleep(10); utils.freeMemory(); await db.freeMemory(); } } - if (chunk.length) { - await saveChunk(chunk); - chunk = null; - } await db.delete({table: to, where: `@@flag('toDel')`}); await db.close({table: to}); diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index bb44c57..42338cd 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -194,11 +194,14 @@ class WebWorker { }); //открываем почти все таблицы - await db.openAll({exclude: ['author']}); + await db.openAll({exclude: ['author', 'title_book']}); //откроем таблицу 'author' с бОльшим размером кеша блоков, для ускорения выборки await db.open({table: 'author', cacheSize: (config.dbCacheSize > 100 ? config.dbCacheSize : 100)}); + if (!config.extendedSearch) + await db.open({table: 'title_book'}); + this.dbSearcher = new DbSearcher(config, db); db.wwCache = {};