From 87369a00a2d810aecec94ff09ebf3405dd83e950 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 26 Oct 2022 16:37:04 +0700 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BE=D0=BF=D1=82=D0=B8=D0=BC=D0=B8=D0=B7=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbCreator.js | 126 ++++++--------------------------------- 1 file changed, 19 insertions(+), 107 deletions(-) diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 67653d0..0327858 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -557,12 +557,12 @@ class DbCreator { }); callback({job: 'optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0}); - await this.optimizeSeries(db, (p) => { + await this.optimizeTable('series', 'series_book', 'series', db, (p) => { if (p.progress) p.progress = 0.5*p.progress; callback(p); }); - await this.optimizeTitle(db, (p) => { + await this.optimizeTable('title', 'title_book', 'title', db, (p) => { if (p.progress) p.progress = 0.5*(1 + p.progress); callback(p); @@ -592,18 +592,18 @@ class DbCreator { callback({job: 'done', jobMessage: ''}); } - async optimizeSeries(db, callback) { - //оптимизация series, превращаем массив bookId в books, кладем все в series_book - await db.open({table: 'series'}); + async optimizeTable(from, to, restoreProp, db, callback) { + //оптимизация таблицы from, превращаем массив bookId в books, кладем все в таблицу to + await db.open({table: from}); await db.create({ - table: 'series_book', + table: to, flag: {name: 'toDel', check: 'r => r.toDel'}, }); - const saveSeriesChunk = async(seriesChunk) => { + const saveChunk = async(chunk) => { const ids = []; - for (const s of seriesChunk) { + for (const s of chunk) { for (const id of s.bookId) { ids.push(id); } @@ -617,7 +617,7 @@ class DbCreator { for (const row of rows) bookArr.set(row.id, row); - for (const s of seriesChunk) { + for (const s of chunk) { s.books = []; s.bookCount = 0; s.bookDelCount = 0; @@ -633,22 +633,23 @@ class DbCreator { } if (s.books.length) { - s.series = s.books[0].series; + s[restoreProp] = s.books[0][restoreProp]; } else { s.toDel = 1; } + delete s.value; delete s.authorId; delete s.bookId; } await db.insert({ - table: 'series_book', - rows: seriesChunk, + table: to, + rows: chunk, }); }; - const rows = await db.select({table: 'series'}); + const rows = await db.select({table: from}); let idsLen = 0; let chunk = []; @@ -659,7 +660,7 @@ class DbCreator { processed++; if (idsLen > 20000) {//константа выяснена эмпирическим путем "память/скорость" - await saveSeriesChunk(chunk); + await saveChunk(chunk); idsLen = 0; chunk = []; @@ -672,102 +673,13 @@ class DbCreator { } } if (chunk.length) { - await saveSeriesChunk(chunk); + await saveChunk(chunk); chunk = null; } - await db.delete({table: 'series_book', where: `@@flag('toDel')`}); - await db.close({table: 'series_book'}); - await db.close({table: 'series'}); - } - - async optimizeTitle(db, callback) { - //оптимизация title, превращаем массив bookId в books, кладем все в title_book - await db.open({table: 'title'}); - - await db.create({ - table: 'title_book', - flag: {name: 'toDel', check: 'r => r.toDel'}, - }); - - const saveTitleChunk = async(titleChunk) => { - const ids = []; - for (const s of titleChunk) { - for (const id of s.bookId) { - ids.push(id); - } - } - - ids.sort((a, b) => a - b);// обязательно, иначе будет тормозить - особенности JembaDb - - const rows = await db.select({table: 'book', where: `@@id(${db.esc(ids)})`}); - - const bookArr = new Map(); - for (const row of rows) - bookArr.set(row.id, row); - - for (const s of titleChunk) { - s.books = []; - s.bookCount = 0; - s.bookDelCount = 0; - for (const id of s.bookId) { - const rec = bookArr.get(id); - if (rec) {//на всякий случай - s.books.push(rec); - if (!rec.del) - s.bookCount++; - else - s.bookDelCount++; - } - } - - if (s.books.length) { - s.series = s.books[0].series; - } else { - s.toDel = 1; - } - - delete s.authorId; - delete s.bookId; - } - - await db.insert({ - table: 'title_book', - rows: titleChunk, - }); - }; - - const rows = await db.select({table: 'title'}); - - let idsLen = 0; - let chunk = []; - let processed = 0; - for (const row of rows) {// eslint-disable-line - chunk.push(row); - idsLen += row.bookId.length; - processed++; - - if (idsLen > 20000) {//константа выяснена эмпирическим путем "память/скорость" - await saveTitleChunk(chunk); - - idsLen = 0; - chunk = []; - - callback({progress: processed/rows.length}); - - await utils.sleep(100); - utils.freeMemory(); - await db.freeMemory(); - } - } - if (chunk.length) { - await saveTitleChunk(chunk); - chunk = null; - } - - await db.delete({table: 'title_book', where: `@@flag('toDel')`}); - await db.close({table: 'title_book'}); - await db.close({table: 'title'}); + await db.delete({table: to, where: `@@flag('toDel')`}); + await db.close({table: to}); + await db.close({table: from}); } async countStats(db, callback, stats) {