deadcxap/inpx-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Рефакторинг

Browse Source
This commit is contained in:
Book Pauk
2022-10-20 14:03:49 +07:00
parent a556a88ee5
commit 4ca56db142
2 changed files with 146 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
server/core/DbCreator.js
View File

@@ -539,7 +539,7 @@ class DbCreator {
//series
callback({job: 'series save', jobMessage: 'Сохранение индекса серий', jobStep: 7, progress: 0});
await saveTable('series_temporary', seriesArr, () => {seriesArr = null}, true, true);
await saveTable('series', seriesArr, () => {seriesArr = null}, true, true);
//title
callback({job: 'title save', jobMessage: 'Сохранение индекса названий', jobStep: 8, progress: 0});
@@ -561,21 +561,33 @@ class DbCreator {
await db.create({table: 'file_hash'});
//-- завершающие шаги --------------------------------
//оптимизация series, превращаем массив bookId в books
callback({job: 'series optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0});
await db.open({
table: 'book',
cacheSize: (config.lowMemoryMode ? 5 : 500),
});
await db.open({table: 'series_temporary'});
await db.create({
table: 'series',
index: {field: 'value', unique: true, depth: 1000000},
});
const count = await db.select({table: 'series_temporary', count: true});
const seriesCount = (count.length ? count[0].count : 0);
callback({job: 'series optimization', jobMessage: 'Оптимизация', jobStep: 11, progress: 0});
await this.optimizeSeries(db, callback);
callback({job: 'files count', jobMessage: 'Подсчет статистики', jobStep: 12, progress: 0});
await this.countStats(db, callback, stats);
//чистка памяти, ибо жрет как не в себя
await db.close({table: 'book'});
await db.freeMemory();
utils.freeMemory();
callback({job: 'done', jobMessage: ''});
}
async optimizeSeries(db, callback) {
//оптимизация series, превращаем массив bookId в books, кладем все в series_book
await db.open({table: 'series'});
await db.create({
table: 'series_book',
flag: {name: 'toDel', check: 'r => r.toDel'},
});
const saveSeriesChunk = async(seriesChunk) => {
const ids = [];
@@ -594,52 +606,62 @@ class DbCreator {
bookArr.set(row.id, row);
for (const s of seriesChunk) {
const sBooks = [];
s.books = [];
for (const id of s.bookId) {
const rec = bookArr.get(id);
sBooks.push(rec);
s.books.push(rec);
}
if (s.books.length) {
s.series = s.books[0].value;
} else {
s.toDel = 1;
}
s.books = JSON.stringify(sBooks);
delete s.bookId;
}
await db.insert({
table: 'series',
table: 'series_book',
rows: seriesChunk,
});
};
const rows = await db.select({table: 'series_temporary'});
const rows = await db.select({table: 'series'});
idsLen = 0;
aChunk = [];
proc = 0;
let idsLen = 0;
let chunk = [];
let processed = 0;
for (const row of rows) {// eslint-disable-line
aChunk.push(row);
chunk.push(row);
idsLen += row.bookId.length;
proc++;
processed++;
if (idsLen > 20000) {//константа выяснена эмпирическим путем "память/скорость"
await saveSeriesChunk(aChunk);
await saveSeriesChunk(chunk);
idsLen = 0;
aChunk = [];
chunk = [];
callback({progress: proc/seriesCount});
callback({progress: processed/rows.length});
await utils.sleep(100);
utils.freeMemory();
await db.freeMemory();
}
}
if (aChunk.length) {
await saveSeriesChunk(aChunk);
aChunk = null;
if (chunk.length) {
await saveSeriesChunk(chunk);
chunk = null;
}
await db.delete({table: 'series_book', where: `@@flag('toDel')`});
await db.close({table: 'series_book'});
await db.close({table: 'series'});
}
async countStats(db, callback, stats) {
//статистика по количеству файлов
callback({job: 'files count', jobMessage: 'Подсчет статистики', jobStep: 12, progress: 0});
//эмуляция прогресса
let countDone = false;
@@ -674,16 +696,6 @@ class DbCreator {
]});
}
countDone = true;
//чистка памяти, ибо жрет как не в себя
await db.drop({table: 'series_temporary'});//таблица больше не понадобится
await db.close({table: 'book'});
await db.close({table: 'series'});
await db.freeMemory();
utils.freeMemory();
callback({job: 'done', jobMessage: ''});
}
}

142
server/core/DbSearcher.js
View File

@@ -2,6 +2,8 @@
const utils = require('./utils');
const maxMemCacheSize = 100;
const maxUtf8Char = String.fromCodePoint(0xFFFFF);
const ruAlphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя';
const enAlphabet = 'abcdefghijklmnopqrstuvwxyz';
@@ -16,6 +18,11 @@ class DbSearcher {
this.timer = null;
this.closed = false;
db.searchCache = {
memCache: new Map(),
authorIdsAll: false,
};
this.periodicCleanCache();//no await
}
@@ -180,60 +187,82 @@ class DbSearcher {
return authorIds;
}
async getAuthorIds(query) {
queryKey(q) {
return JSON.stringify([q.author, q.series, q.title, q.genre, q.lang]);
}
async getCached(key) {
if (!this.config.queryCacheEnabled)
return null;
let result = null;
const db = this.db;
const memCache = db.searchCache.memCache;
if (!db.searchCache)
db.searchCache = {};
if (memCache.has(key)) {//есть в недавних
result = memCache.get(key);
let result;
//изменим порядок ключей, для последующей правильной чистки старых
memCache.delete(key);
memCache.set(key, result);
} else {//смотрим в таблице
const rows = await db.select({table: 'query_cache', where: `@@id(${db.esc(key)})`});
//сначала попробуем найти в кеше
const q = query;
const keyArr = [q.author, q.series, q.title, q.genre, q.lang];
const keyStr = `query-${keyArr.join('')}`;
if (!keyStr) {//пустой запрос
if (db.searchCache.authorIdsAll)
result = db.searchCache.authorIdsAll;
else
result = await this.selectAuthorIds(query);
if (rows.length) {//нашли в кеше
await db.insert({
table: 'query_time',
replace: true,
rows: [{id: key, time: Date.now()}],
});
} else {//непустой запрос
if (this.config.queryCacheEnabled) {
const key = JSON.stringify(keyArr);
const rows = await db.select({table: 'query_cache', where: `@@id(${db.esc(key)})`});
result = rows[0].value;
memCache.set(key, result);
if (rows.length) {//нашли в кеше
await db.insert({
table: 'query_time',
replace: true,
rows: [{id: key, time: Date.now()}],
});
result = rows[0].value;
} else {//не нашли в кеше, ищем в поисковых таблицах
result = await this.selectAuthorIds(query);
await db.insert({
table: 'query_cache',
replace: true,
rows: [{id: key, value: result}],
});
await db.insert({
table: 'query_time',
replace: true,
rows: [{id: key, time: Date.now()}],
});
if (memCache.size > maxMemCacheSize) {
//удаляем самый старый ключ-значение
for (const k of memCache.keys()) {
memCache.delete(k);
break;
}
}
} else {
result = await this.selectAuthorIds(query);
}
}
return result;
}
async putCached(key, value) {
if (!this.config.queryCacheEnabled)
return;
const db = this.db;
const memCache = db.searchCache.memCache;
memCache.set(key, value);
if (memCache.size > maxMemCacheSize) {
//удаляем самый старый ключ-значение
for (const k of memCache.keys()) {
memCache.delete(k);
break;
}
}
//кладем в таблицу
await db.insert({
table: 'query_cache',
replace: true,
rows: [{id: key, value}],
});
await db.insert({
table: 'query_time',
replace: true,
rows: [{id: key, time: Date.now()}],
});
}
async search(query) {
if (this.closed)
throw new Error('DbSearcher closed');
@@ -243,7 +272,15 @@ class DbSearcher {
try {
const db = this.db;
const authorIds = await this.getAuthorIds(query);
const key = `author-ids-${this.queryKey(query)}`;
//сначала попробуем найти в кеше
let authorIds = await this.getCached(key);
if (authorIds === null) {//не нашли в кеше, ищем в поисковых таблицах
authorIds = await this.selectAuthorIds(query);
await this.putCached(key, authorIds);
}
const totalFound = authorIds.length;
let limit = (query.limit ? query.limit : 100);
@@ -251,7 +288,7 @@ class DbSearcher {
const offset = (query.offset ? query.offset : 0);
//выборка найденных авторов
let result = await db.select({
const result = await db.select({
table: 'author',
map: `(r) => ({id: r.id, author: r.author, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`,
where: `@@id(${db.esc(authorIds.slice(offset, offset + limit))})`
@@ -272,7 +309,7 @@ class DbSearcher {
try {
const db = this.db;
//выборка автора по authorId
//выборка книг автора по authorId
const rows = await db.select({
table: 'author_book',
where: `@@id(${db.esc(authorId)})`
@@ -302,13 +339,26 @@ class DbSearcher {
const db = this.db;
series = series.toLowerCase();
//выборка серии по названию серии
const rows = await db.select({
let rows = await db.select({
table: 'series',
where: `@@dirtyIndexLR('value', ${db.esc(series)}, ${db.esc(series)})`
});
return {books: (rows.length ? rows[0].books : '')};
let books = [];
if (rows.length) {
//выборка книг серии
rows = await db.select({
table: 'series_book',
where: `@@id(${rows[0].id})`
});
if (rows.length)
books = rows[0].books;
}
return {books: (books && books.length ? JSON.stringify(books) : '')};
} finally {
this.searchFlag--;
}