From d6260e3433886ab3bbaec25fd6f740a30ced51cf Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 15 Nov 2022 00:03:10 +0700 Subject: [PATCH 01/42] =?UTF-8?q?=D0=9E=D0=BF=D1=82=D0=B8=D0=BC=D0=B8?= =?UTF-8?q?=D0=B7=D0=B0=D1=86=D0=B8=D1=8F=20=D0=B8=D1=81=D0=BF=D0=BE=D0=BB?= =?UTF-8?q?=D1=8C=D0=B7=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=BF=D0=B0?= =?UTF-8?q?=D0=BC=D1=8F=D1=82=D0=B8=20=D0=BF=D1=80=D0=B8=20=D0=B7=D0=B0?= =?UTF-8?q?=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B5=20=D0=BC=D0=B0=D0=BF=D0=BF?= =?UTF-8?q?=D0=B8=D0=BD=D0=B3=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/config/base.js | 2 +- server/core/DbCreator.js | 27 ++++++--------- server/core/DbSearcher.js | 71 ++++++++++++++++++--------------------- 3 files changed, 44 insertions(+), 56 deletions(-) diff --git a/server/config/base.js b/server/config/base.js index 923a53a..7eec68c 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -16,7 +16,7 @@ module.exports = { //поправить в случае, если были критические изменения в DbCreator или InpxParser //иначе будет рассинхронизация между сервером и клиентом на уровне БД - dbVersion: '7', + dbVersion: '8', dbCacheSize: 5, maxPayloadSize: 500,//in MB diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 3cfc00e..d1aba16 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -459,7 +459,6 @@ class DbCreator { const config = this.config; const to = `${from}_book`; - const toId = `${from}_id`; await db.open({table: from}); await db.create({table: to}); @@ -548,7 +547,7 @@ class DbCreator { await saveChunk(chunk); processed += chunk.length; - callback({progress: 0.5*processed/fromLength}); + callback({progress: 0.9*processed/fromLength}); } else break; @@ -562,24 +561,18 @@ class DbCreator { await db.close({table: to}); await db.close({table: from}); - await db.create({table: toId}); - - const chunkSize = 50000; - let idRows = []; - let proc = 0; + const idMap = {arr: [], map: []}; for (const [id, value] of bookId2RecId) { - idRows.push({id, value}); - if (idRows.length >= chunkSize) { - await db.insert({table: toId, rows: idRows}); - idRows = []; - - proc += chunkSize; - callback({progress: 0.5 + 0.5*proc/bookId2RecId.size}); + if (value.length > 1) { + idMap.map.push([id, value]); + idMap.arr[id] = 0; + } else { + idMap.arr[id] = value[0]; } } - if (idRows.length) - await db.insert({table: toId, rows: idRows}); - await db.close({table: toId}); + + callback({progress: 1}); + await fs.writeFile(`${this.config.dataDir}/db/${from}_id.map`, JSON.stringify(idMap)); bookId2RecId = null; utils.freeMemory(); diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index b198f59..a39262c 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -1,3 +1,4 @@ +const fs = require('fs-extra'); //const _ = require('lodash'); const LockQueue = require('./LockQueue'); const utils = require('./utils'); @@ -299,29 +300,13 @@ class DbSearcher { await this.lock.get(); try { - const db = this.db; - const map = new Map(); - const table = `${from}_id`; + const data = await fs.readFile(`${this.config.dataDir}/db/${from}_id.map`, 'utf-8'); - await db.open({table}); - let rows = await db.select({table}); - await db.close({table}); + const idMap = JSON.parse(data); + idMap.arr = new Uint32Array(idMap.arr); + idMap.map = new Map(idMap.map); - for (const row of rows) { - if (!row.value.length) - continue; - - if (row.value.length > 1) - map.set(row.id, row.value); - else - map.set(row.id, row.value[0]); - } - - this.bookIdMap[from] = map; - - rows = null; - await db.freeMemory(); - utils.freeMemory(); + this.bookIdMap[from] = idMap; return this.bookIdMap[from]; } finally { @@ -330,15 +315,21 @@ class DbSearcher { } async fillBookIdMapAll() { - await this.fillBookIdMap('author'); - await this.fillBookIdMap('series'); - await this.fillBookIdMap('title'); + try { + await this.fillBookIdMap('author'); + await this.fillBookIdMap('series'); + await this.fillBookIdMap('title'); + } catch (e) { + // + } } async filterTableIds(tableIds, from, query) { let result = tableIds; - //т.к. авторы у книги идут списком, то дополнительно фильтруем + //т.к. авторы у книги идут списком (т.е. одна книга относиться сразу к нескольким авторам), + //то в выборку по bookId могут попасть авторы, которые отсутствуют в критерии query.author, + //поэтому дополнительно фильтруем if (from == 'author' && query.author && query.author !== '*') { const key = `filter-ids-author-${query.author}`; let authorIds = await this.getCached(key); @@ -381,24 +372,25 @@ class DbSearcher { await this.putCached(bookKey, bookIds); } + //id книг (bookIds) нашли, теперь надо их смаппировать в id таблицы from (авторов, серий, названий) if (bookIds) { const tableIdsSet = new Set(); - const bookIdMap = await this.fillBookIdMap(from); + const idMap = await this.fillBookIdMap(from); let proc = 0; let nextProc = 0; for (const bookId of bookIds) { - const tableIdValue = bookIdMap.get(bookId); - if (!tableIdValue) - continue; - - if (Array.isArray(tableIdValue)) { - for (const tableId of tableIdValue) { - tableIdsSet.add(tableId); - proc++; - } - } else { - tableIdsSet.add(tableIdValue); + const tableId = idMap.arr[bookId]; + if (tableId) { + tableIdsSet.add(tableId); proc++; + } else { + const tableIdArr = idMap.map.get(bookId); + if (tableIdArr) { + for (const tableId of tableIdArr) { + tableIdsSet.add(tableId); + proc++; + } + } } //прерываемся иногда, чтобы не блокировать Event Loop @@ -409,7 +401,7 @@ class DbSearcher { } tableIds = Array.from(tableIdsSet); - } else { + } else {//bookIds пустой - критерии не заданы, значит берем все id из from const rows = await db.select({ table: from, rawResult: true, @@ -419,8 +411,11 @@ class DbSearcher { tableIds = rows[0].rawResult; } + //т.к. авторы у книги идут списком, то дополнительно фильтруем tableIds = await this.filterTableIds(tableIds, from, query); + //сортируем по id + //порядок id соответствует ASC-сортировке по строковому значению из from (имя автора, назание серии, название книги) tableIds.sort((a, b) => a - b); await this.putCached(tableKey, tableIds); From 044ab1ab267dd91932022525ea220f551aab967b Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 15 Nov 2022 00:16:45 +0700 Subject: [PATCH 02/42] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B0=20=D0=BE=D0=B1=D1=80=D0=B0=D0=B1=D0=BE?= =?UTF-8?q?=D1=82=D0=BA=D0=B0=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index a39262c..7bc8a81 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -320,7 +320,7 @@ class DbSearcher { await this.fillBookIdMap('series'); await this.fillBookIdMap('title'); } catch (e) { - // + throw new Error(`DbSearcher.fillBookIdMapAll error: ${e.message}`) } } From 64a301eda1e9fa36e5366a2040fa1dfcd8f2b5bb Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 16 Nov 2022 18:56:46 +0700 Subject: [PATCH 03/42] "jembadb": "^5.1.0" --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index e3f9747..a229404 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "express": "^4.18.1", "fs-extra": "^10.1.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.0.2", + "jembadb": "^5.1.0", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", @@ -5046,9 +5046,9 @@ } }, "node_modules/jembadb": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.2.tgz", - "integrity": "sha512-0309Qo4wSkyf154xTokxNl0DuBP5f2Q2MzWGUNX1JmMzlRypFsPY/9VDYV/htkxhT53f2prlQ2NUguQjG2lCRA==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.0.tgz", + "integrity": "sha512-CqgGIcpDNivom2i20Xrq4F9EGKsEM2G8BdRVxqC44UNXUpr6IkQb4Z0pRowoSYd15mfLo4VlPe9ncYNt43psgQ==", "engines": { "node": ">=16.16.0" } @@ -12521,9 +12521,9 @@ "dev": true }, "jembadb": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.2.tgz", - "integrity": "sha512-0309Qo4wSkyf154xTokxNl0DuBP5f2Q2MzWGUNX1JmMzlRypFsPY/9VDYV/htkxhT53f2prlQ2NUguQjG2lCRA==" + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.0.tgz", + "integrity": "sha512-CqgGIcpDNivom2i20Xrq4F9EGKsEM2G8BdRVxqC44UNXUpr6IkQb4Z0pRowoSYd15mfLo4VlPe9ncYNt43psgQ==" }, "jest-worker": { "version": "27.5.1", diff --git a/package.json b/package.json index 727a9bc..8270683 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "express": "^4.18.1", "fs-extra": "^10.1.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.0.2", + "jembadb": "^5.1.0", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", From 5630feba363597805827e2babddbaf6b9caa0a52 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 16 Nov 2022 19:32:51 +0700 Subject: [PATCH 04/42] "jembadb": "^5.1.1" --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index a229404..5f73260 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "express": "^4.18.1", "fs-extra": "^10.1.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.1.0", + "jembadb": "^5.1.1", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", @@ -5046,9 +5046,9 @@ } }, "node_modules/jembadb": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.0.tgz", - "integrity": "sha512-CqgGIcpDNivom2i20Xrq4F9EGKsEM2G8BdRVxqC44UNXUpr6IkQb4Z0pRowoSYd15mfLo4VlPe9ncYNt43psgQ==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.1.tgz", + "integrity": "sha512-HMz/EQug+PGfzV9JADHJFEEkxnA+BBoFALvPEtd1qFFa+gzusqW8WHYfp3ihFPVhkvEFz9VbXrlYs1TzKItnKw==", "engines": { "node": ">=16.16.0" } @@ -12521,9 +12521,9 @@ "dev": true }, "jembadb": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.0.tgz", - "integrity": "sha512-CqgGIcpDNivom2i20Xrq4F9EGKsEM2G8BdRVxqC44UNXUpr6IkQb4Z0pRowoSYd15mfLo4VlPe9ncYNt43psgQ==" + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.1.tgz", + "integrity": "sha512-HMz/EQug+PGfzV9JADHJFEEkxnA+BBoFALvPEtd1qFFa+gzusqW8WHYfp3ihFPVhkvEFz9VbXrlYs1TzKItnKw==" }, "jest-worker": { "version": "27.5.1", diff --git a/package.json b/package.json index 8270683..9e4b509 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "express": "^4.18.1", "fs-extra": "^10.1.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.1.0", + "jembadb": "^5.1.1", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", From 3d1385da6e3dd1f4d77e0985679d82d7cd925743 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 16 Nov 2022 20:08:28 +0700 Subject: [PATCH 05/42] "jembadb": "^5.1.2" --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5f73260..96a2a9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "express": "^4.18.1", "fs-extra": "^10.1.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.1.1", + "jembadb": "^5.1.2", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", @@ -5046,9 +5046,9 @@ } }, "node_modules/jembadb": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.1.tgz", - "integrity": "sha512-HMz/EQug+PGfzV9JADHJFEEkxnA+BBoFALvPEtd1qFFa+gzusqW8WHYfp3ihFPVhkvEFz9VbXrlYs1TzKItnKw==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.2.tgz", + "integrity": "sha512-0lLGyIrcY53oRQ5JercUysq+sld5mQl6Sw+ZNnTniUqsFKVp1vyYfa3HILYP0yLKwSFF336szlCwGZ+VsX4ufg==", "engines": { "node": ">=16.16.0" } @@ -12521,9 +12521,9 @@ "dev": true }, "jembadb": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.1.tgz", - "integrity": "sha512-HMz/EQug+PGfzV9JADHJFEEkxnA+BBoFALvPEtd1qFFa+gzusqW8WHYfp3ihFPVhkvEFz9VbXrlYs1TzKItnKw==" + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.2.tgz", + "integrity": "sha512-0lLGyIrcY53oRQ5JercUysq+sld5mQl6Sw+ZNnTniUqsFKVp1vyYfa3HILYP0yLKwSFF336szlCwGZ+VsX4ufg==" }, "jest-worker": { "version": "27.5.1", diff --git a/package.json b/package.json index 9e4b509..129a180 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "express": "^4.18.1", "fs-extra": "^10.1.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.1.1", + "jembadb": "^5.1.2", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", From d5931138e3a51e8867a3b556b7ac38dc19858aad Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 16 Nov 2022 20:27:53 +0700 Subject: [PATCH 06/42] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D1=85=D0=BE?= =?UTF-8?q?=D0=B4=20=D0=BD=D0=B0=20Uint32Array?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 7bc8a81..78c5e70 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -78,7 +78,7 @@ class DbSearcher { result.add(bookId); } - return Array.from(result); + return new Uint32Array(result); ` }); @@ -152,7 +152,7 @@ class DbSearcher { result.add(bookId); } - return Array.from(result); + return new Uint32Array(result); ` }); @@ -188,7 +188,7 @@ class DbSearcher { result.add(bookId); } - return Array.from(result); + return new Uint32Array(result); ` }); @@ -253,7 +253,7 @@ class DbSearcher { result.add(bookId); } - return Array.from(result); + return new Uint32Array(result); ` }); @@ -286,7 +286,7 @@ class DbSearcher { inter = newInter; } - return Array.from(inter); + return new Uint32Array(inter); } else if (idsArr.length == 1) { return idsArr[0]; } else { @@ -324,12 +324,11 @@ class DbSearcher { } } - async filterTableIds(tableIds, from, query) { - let result = tableIds; - + async tableIdsFilter(from, query) { //т.к. авторы у книги идут списком (т.е. одна книга относиться сразу к нескольким авторам), //то в выборку по bookId могут попасть авторы, которые отсутствуют в критерии query.author, //поэтому дополнительно фильтруем + let result = null; if (from == 'author' && query.author && query.author !== '*') { const key = `filter-ids-author-${query.author}`; let authorIds = await this.getCached(key); @@ -338,7 +337,7 @@ class DbSearcher { const rows = await this.db.select({ table: 'author', rawResult: true, - where: `return Array.from(${this.getWhere(query.author)})` + where: `return new Uint32Array(${this.getWhere(query.author)})` }); authorIds = rows[0].rawResult; @@ -346,12 +345,7 @@ class DbSearcher { await this.putCached(key, authorIds); } - //пересечение tableIds и authorIds - result = []; - const authorIdsSet = new Set(authorIds); - for (const id of tableIds) - if (authorIdsSet.has(id)) - result.push(id); + result = new Set(authorIds); } return result; @@ -374,6 +368,9 @@ class DbSearcher { //id книг (bookIds) нашли, теперь надо их смаппировать в id таблицы from (авторов, серий, названий) if (bookIds) { + //т.к. авторы у книги идут списком, то дополнительно фильтруем + const filter = await this.tableIdsFilter(from, query); + const tableIdsSet = new Set(); const idMap = await this.fillBookIdMap(from); let proc = 0; @@ -381,13 +378,15 @@ class DbSearcher { for (const bookId of bookIds) { const tableId = idMap.arr[bookId]; if (tableId) { - tableIdsSet.add(tableId); + if (!filter || filter.has(tableId)) + tableIdsSet.add(tableId); proc++; } else { const tableIdArr = idMap.map.get(bookId); if (tableIdArr) { for (const tableId of tableIdArr) { - tableIdsSet.add(tableId); + if (!filter || filter.has(tableId)) + tableIdsSet.add(tableId); proc++; } } @@ -411,9 +410,6 @@ class DbSearcher { tableIds = rows[0].rawResult; } - //т.к. авторы у книги идут списком, то дополнительно фильтруем - tableIds = await this.filterTableIds(tableIds, from, query); - //сортируем по id //порядок id соответствует ASC-сортировке по строковому значению из from (имя автора, назание серии, название книги) tableIds.sort((a, b) => a - b); From 4b4865b6edd45d4eb9b409e5072231cf538abe1e Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 16 Nov 2022 20:37:18 +0700 Subject: [PATCH 07/42] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D1=85=D0=BE?= =?UTF-8?q?=D0=B4=20=D0=BD=D0=B0=20Uint32Array?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 78c5e70..98a6778 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -399,12 +399,12 @@ class DbSearcher { } } - tableIds = Array.from(tableIdsSet); + tableIds = new Uint32Array(tableIdsSet); } else {//bookIds пустой - критерии не заданы, значит берем все id из from const rows = await db.select({ table: from, rawResult: true, - where: `return Array.from(@all())` + where: `return new Uint32Array(@all())` }); tableIds = rows[0].rawResult; @@ -500,11 +500,13 @@ class DbSearcher { limit = (limit > maxLimit ? maxLimit : limit); const offset = (query.offset ? query.offset : 0); + const slice = ids.slice(offset, offset + limit); + //выборка найденных значений const found = await db.select({ table: from, map: `(r) => ({id: r.id, ${from}: r.name, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, - where: `@@id(${db.esc(ids.slice(offset, offset + limit))})` + where: `@@id(${db.esc(Array.from(slice))})` }); //для title восстановим books From 6b91c436552e7e076f604e491e4cb41653456b0a Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 17 Nov 2022 14:01:40 +0700 Subject: [PATCH 08/42] "jembadb": "^5.1.3" --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 96a2a9f..cd83e95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "express": "^4.18.1", "fs-extra": "^10.1.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.1.2", + "jembadb": "^5.1.3", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", @@ -5046,9 +5046,9 @@ } }, "node_modules/jembadb": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.2.tgz", - "integrity": "sha512-0lLGyIrcY53oRQ5JercUysq+sld5mQl6Sw+ZNnTniUqsFKVp1vyYfa3HILYP0yLKwSFF336szlCwGZ+VsX4ufg==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.3.tgz", + "integrity": "sha512-HGl9d3/fcNNahOqEsb3ocpXRWEfmDwV2zgWvKXERwlsxOHqoEId2fHXPkjv97qRywEyE/n9U8WimIWsP2Evf4w==", "engines": { "node": ">=16.16.0" } @@ -12521,9 +12521,9 @@ "dev": true }, "jembadb": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.2.tgz", - "integrity": "sha512-0lLGyIrcY53oRQ5JercUysq+sld5mQl6Sw+ZNnTniUqsFKVp1vyYfa3HILYP0yLKwSFF336szlCwGZ+VsX4ufg==" + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.3.tgz", + "integrity": "sha512-HGl9d3/fcNNahOqEsb3ocpXRWEfmDwV2zgWvKXERwlsxOHqoEId2fHXPkjv97qRywEyE/n9U8WimIWsP2Evf4w==" }, "jest-worker": { "version": "27.5.1", diff --git a/package.json b/package.json index 129a180..236ea9a 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "express": "^4.18.1", "fs-extra": "^10.1.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.1.2", + "jembadb": "^5.1.3", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", From 412335c0f11a926a9fd0e0b930624d1a4157c857 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 17 Nov 2022 16:51:12 +0700 Subject: [PATCH 09/42] =?UTF-8?q?=D0=92=20=D0=BA=D0=BE=D0=BD=D1=84=D0=B8?= =?UTF-8?q?=D0=B3=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D1=8B?= =?UTF-8?q?=20=D0=BF=D0=B0=D1=80=D0=B0=D0=BC=D0=B5=D1=82=D1=80=D1=8B=20que?= =?UTF-8?q?ryCacheMemSize,=20queryCacheDiskSize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/config/base.js | 2 + server/config/index.js | 2 + server/core/DbSearcher.js | 125 +++++++++++++++++++++++++------------- 3 files changed, 87 insertions(+), 42 deletions(-) diff --git a/server/config/base.js b/server/config/base.js index 7eec68c..40a0801 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -22,6 +22,8 @@ module.exports = { maxPayloadSize: 500,//in MB maxFilesDirSize: 1024*1024*1024,//1Gb queryCacheEnabled: true, + queryCacheMemSize: 50, + queryCacheDiskSize: 500, cacheCleanInterval: 60,//minutes inpxCheckInterval: 60,//minutes lowMemoryMode: false, diff --git a/server/config/index.js b/server/config/index.js index c30381c..3552fce 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -11,6 +11,8 @@ const propsToSave = [ 'dbCacheSize', 'maxFilesDirSize', 'queryCacheEnabled', + 'queryCacheMemSize', + 'queryCacheDiskSize', 'cacheCleanInterval', 'inpxCheckInterval', 'lowMemoryMode', diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 98a6778..04d0698 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -3,7 +3,6 @@ const fs = require('fs-extra'); const LockQueue = require('./LockQueue'); const utils = require('./utils'); -const maxMemCacheSize = 100; const maxLimit = 1000; const emptyFieldValue = '?'; @@ -15,6 +14,11 @@ const enruArr = (ruAlphabet + enAlphabet).split(''); class DbSearcher { constructor(config, db) { this.config = config; + this.queryCacheMemSize = this.config.queryCacheMemSize; + this.queryCacheDiskSize = this.config.queryCacheDiskSize; + this.queryCacheEnabled = this.config.queryCacheEnabled + && (this.queryCacheMemSize > 0 || this.queryCacheDiskSize > 0); + this.db = db; this.lock = new LockQueue(); @@ -501,7 +505,7 @@ class DbSearcher { const offset = (query.offset ? query.offset : 0); const slice = ids.slice(offset, offset + limit); - + //выборка найденных значений const found = await db.select({ table: from, @@ -594,7 +598,7 @@ class DbSearcher { } async getCached(key) { - if (!this.config.queryCacheEnabled) + if (!this.queryCacheEnabled) return null; let result = null; @@ -602,13 +606,13 @@ class DbSearcher { const db = this.db; const memCache = this.memCache; - if (memCache.has(key)) {//есть в недавних + if (this.queryCacheMemSize > 0 && memCache.has(key)) {//есть в недавних result = memCache.get(key); //изменим порядок ключей, для последующей правильной чистки старых memCache.delete(key); memCache.set(key, result); - } else {//смотрим в таблице + } else if (this.queryCacheDiskSize > 0) {//смотрим в таблице const rows = await db.select({table: 'query_cache', where: `@@id(${db.esc(key)})`}); if (rows.length) {//нашли в кеше @@ -619,13 +623,17 @@ class DbSearcher { }); result = rows[0].value; - memCache.set(key, result); - if (memCache.size > maxMemCacheSize) { - //удаляем самый старый ключ-значение - for (const k of memCache.keys()) { - memCache.delete(k); - break; + //заполняем кеш в памяти + if (this.queryCacheMemSize > 0) { + memCache.set(key, result); + + if (memCache.size > this.queryCacheMemSize) { + //удаляем самый старый ключ-значение + for (const k of memCache.keys()) { + memCache.delete(k); + break; + } } } } @@ -635,40 +643,44 @@ class DbSearcher { } async putCached(key, value) { - if (!this.config.queryCacheEnabled) + if (!this.queryCacheEnabled) return; const db = this.db; - const memCache = this.memCache; - memCache.set(key, value); + if (this.queryCacheMemSize > 0) { + const memCache = this.memCache; + memCache.set(key, value); - if (memCache.size > maxMemCacheSize) { - //удаляем самый старый ключ-значение - for (const k of memCache.keys()) { - memCache.delete(k); - break; + if (memCache.size > this.queryCacheMemSize) { + //удаляем самый старый ключ-значение + for (const k of memCache.keys()) { + memCache.delete(k); + break; + } } } - //кладем в таблицу асинхронно - (async() => { - try { - await db.insert({ - table: 'query_cache', - replace: true, - rows: [{id: key, value}], - }); + if (this.queryCacheDiskSize > 0) { + //кладем в таблицу асинхронно + (async() => { + try { + await db.insert({ + table: 'query_cache', + replace: true, + rows: [{id: key, value}], + }); - await db.insert({ - table: 'query_time', - replace: true, - rows: [{id: key, time: Date.now()}], - }); - } catch(e) { - console.error(`putCached: ${e.message}`); - } - })(); + await db.insert({ + table: 'query_time', + replace: true, + rows: [{id: key, time: Date.now()}], + }); + } catch(e) { + console.error(`putCached: ${e.message}`); + } + })(); + } } async periodicCleanCache() { @@ -678,21 +690,50 @@ class DbSearcher { return; try { + if (!this.queryCacheEnabled || this.queryCacheDiskSize <= 0) + return; + const db = this.db; - const oldThres = Date.now() - cleanInterval; + let rows = await db.select({table: 'query_time', count: true}); + const delCount = rows[0].count - this.queryCacheDiskSize; + + if (delCount < 1) + return; //выберем всех кандидатов на удаление - const rows = await db.select({ + //находим delCount минимальных по time + rows = await db.select({ table: 'query_time', + rawResult: true, where: ` - @@iter(@all(), (r) => (r.time < ${db.esc(oldThres)})); + const res = Array(${db.esc(delCount)}).fill({time: Date.now()}); + + @unsafeIter(@all(), (r) => { + if (r.time >= res[${db.esc(delCount - 1)}].time) + return false; + + let ins = {id: r.id, time: r.time}; + + for (let i = 0; i < res.length; i++) { + if (!res[i].id || ins.time < res[i].time) { + const t = res[i]; + res[i] = ins; + ins = t; + } + + if (!ins.id) + break; + } + + return false; + }); + + return res.filter(r => r.id).map(r => r.id); ` }); - const ids = []; - for (const row of rows) - ids.push(row.id); + const ids = rows[0].rawResult; //удаляем await db.delete({table: 'query_cache', where: `@@id(${db.esc(ids)})`}); From 1ba54c12378cf64771eb381c35230a0cc55c72b3 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 17 Nov 2022 16:58:09 +0700 Subject: [PATCH 10/42] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=BE=20=D0=BE=D0=BF=D0=B8=D1=81=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D0=B5=20=D0=BF=D0=B0=D1=80=D0=B0=D0=BC=D0=B5=D1=82=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=BA=D0=BE=D0=BD=D1=84=D0=B8=D0=B3=D0=B0=20queryCach?= =?UTF-8?q?eMemSize,=20queryCacheDiskSize?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8737970..0ff9abd 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,17 @@ Options: // чистка каждый час "maxFilesDirSize": 1073741824, - // включить(true)/выключить(false) кеширование запросов на сервере + // включить(true)/выключить(false) серверное кеширование запросов на диске и в памяти "queryCacheEnabled": true, + // размер кеша запросов в оперативной памяти (количество) + // 0 - отключить кеширование запросов в оперативной памяти + "queryCacheMemSize": 50, + + // размер кеша запросов на диске (количество) + // 0 - отключить кеширование запросов на диске + "queryCacheDiskSize": 500, + // периодичность чистки кеша запросов на сервере, в минутах // 0 - отключить чистку "cacheCleanInterval": 60, From a840fb7233d7a356555dd7e1b24d578f9b3e1816 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 17 Nov 2022 17:08:06 +0700 Subject: [PATCH 11/42] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BE=D1=82=D0=BE=D0=B1=D1=80=D0=B0=D0=B6=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D1=8F=20Inpx=20=D0=B8=D0=BD=D1=84=D0=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/Search/BookInfoDialog/BookInfoDialog.vue | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client/components/Search/BookInfoDialog/BookInfoDialog.vue b/client/components/Search/BookInfoDialog/BookInfoDialog.vue index ee8b9bc..f20e3f6 100644 --- a/client/components/Search/BookInfoDialog/BookInfoDialog.vue +++ b/client/components/Search/BookInfoDialog/BookInfoDialog.vue @@ -174,7 +174,6 @@ class BookInfoDialog { {name: 'fileInfo', label: 'Информация о файле', value: [ {name: 'folder', label: 'Папка'}, {name: 'file', label: 'Файл'}, - {name: 'ext', label: 'Тип'}, {name: 'size', label: 'Размер'}, {name: 'date', label: 'Добавлен'}, {name: 'del', label: 'Удален'}, @@ -193,7 +192,10 @@ class BookInfoDialog { ]}, ]; - const valueToString = (value, nodePath) => {//eslint-disable-line no-unused-vars + const valueToString = (value, nodePath, b) => {//eslint-disable-line no-unused-vars + if (nodePath == 'fileInfo/file') + return `${value}.${b.ext}`; + if (nodePath == 'fileInfo/size') return `${this.formatSize(value)} (${value.toString().replace(/(\d)(?=(\d{3})+(?!\d))/g, '$1 ')} Bytes)`; @@ -230,7 +232,7 @@ class BookInfoDialog { const subItemOut = { name: subItem.name, label: subItem.label, - value: valueToString(book[subItem.name], `${item.name}/${subItem.name}`) + value: valueToString(book[subItem.name], `${item.name}/${subItem.name}`, book) }; if (subItemOut.value) itemOut.value.push(subItemOut); From 1b70259ea7aadb30ba943082c6c0604b4bbd3991 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 17 Nov 2022 17:53:28 +0700 Subject: [PATCH 12/42] "showdown": "^2.1.0" --- package-lock.json | 43 +++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 44 insertions(+) diff --git a/package-lock.json b/package-lock.json index cd83e95..6deab87 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49,6 +49,7 @@ "html-webpack-plugin": "^5.5.0", "mini-css-extract-plugin": "^2.6.1", "pkg": "^5.8.0", + "showdown": "^2.1.0", "terser-webpack-plugin": "^5.3.3", "vue-eslint-parser": "^9.0.3", "vue-loader": "^17.0.0", @@ -7477,6 +7478,31 @@ "node": ">=8" } }, + "node_modules/showdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz", + "integrity": "sha512-/6NVYu4U819R2pUIk79n67SYgJHWCce0a5xTP979WbNp0FL9MN1I1QK662IDU1b6JzKTvmhgI7T7JYIxBi3kMQ==", + "dev": true, + "dependencies": { + "commander": "^9.0.0" + }, + "bin": { + "showdown": "bin/showdown.js" + }, + "funding": { + "type": "individual", + "url": "https://www.paypal.me/tiviesantos" + } + }, + "node_modules/showdown/node_modules/commander": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.4.1.tgz", + "integrity": "sha512-5EEkTNyHNGFPD2H+c/dXXfQZYa/scCKasxWcXJaWnNJ99pnQN9Vnmqow+p+PlFPE63Q6mThaZws1T+HxfpgtPw==", + "dev": true, + "engines": { + "node": "^12.20.0 || >=14" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -14257,6 +14283,23 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, + "showdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz", + "integrity": "sha512-/6NVYu4U819R2pUIk79n67SYgJHWCce0a5xTP979WbNp0FL9MN1I1QK662IDU1b6JzKTvmhgI7T7JYIxBi3kMQ==", + "dev": true, + "requires": { + "commander": "^9.0.0" + }, + "dependencies": { + "commander": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.4.1.tgz", + "integrity": "sha512-5EEkTNyHNGFPD2H+c/dXXfQZYa/scCKasxWcXJaWnNJ99pnQN9Vnmqow+p+PlFPE63Q6mThaZws1T+HxfpgtPw==", + "dev": true + } + } + }, "side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", diff --git a/package.json b/package.json index 236ea9a..c453c14 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "html-webpack-plugin": "^5.5.0", "mini-css-extract-plugin": "^2.6.1", "pkg": "^5.8.0", + "showdown": "^2.1.0", "terser-webpack-plugin": "^5.3.3", "vue-eslint-parser": "^9.0.3", "vue-loader": "^17.0.0", From 13c3c98c63624531f583fc4eaa36b3c75e3d7c4f Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 17 Nov 2022 17:53:47 +0700 Subject: [PATCH 13/42] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D0=BB=20readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0ff9abd..9b0f869 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,21 @@ inpx-web ======== Веб-сервер для поиска по .inpx-коллекции. -Выглядит это так: https://lib.omnireader.ru + +Выглядит следующим образом: [https://lib.omnireader.ru](https://lib.omnireader.ru) .inpx - индексный файл для импорта\экспорта информации из базы данных сетевых библиотек в базу каталогизатора [MyHomeLib](https://alex80.github.io/mhl/) или [freeLib](http://sourceforge.net/projects/freelibdesign) или [LightLib](https://lightlib.azurewebsites.net) -Просто поместите приложение `inpx-web` в папку с .inpx-файлом и файлами библиотеки (zip-архивами) и запустите. -Сервер будет доступен по адресу http://127.0.0.1:12380 +[Установка](#usage): просто поместить приложение `inpx-web` в папку с .inpx-файлом и файлами библиотеки (zip-архивами) и запустить. +По умолчанию, сервер будет доступен по адресу [http://127.0.0.1:12380](http://127.0.0.1:12380) -После открытия веб-приложения в бразуере, для быстрого понимания того, как работает поиск, воспользуйтесь памяткой (кнопка со знаком вопроса). +Для указания местоположения .inpx-файла или папки с файлами библиотеки, воспользуйтесь [параметрами командной строки](#cli). +Дополнительные параметры сервера настраиваются в [конфигурационном файле](#config). -## +## * [Возможности программы](#capabilities) * [Использование](#usage) * [Параметры командной строки](#cli) @@ -45,7 +47,7 @@ inpx-web Там же, при первом запуске, будет создана рабочая директория `.inpx-web`, в которой хранится конфигурационный файл `config.json`, файлы базы данных, журналы и прочее. -По умолчанию сервер будет доступен по адресу http://127.0.0.1:12380 +По умолчанию сервер будет доступен по адресу [http://127.0.0.1:12380](http://127.0.0.1:12380) From 7e9f446079103e6e0a91803e4d421dc6d7cb047c Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 17 Nov 2022 17:56:46 +0700 Subject: [PATCH 14/42] =?UTF-8?q?=D0=92=20=D1=80=D0=B5=D0=BB=D0=B8=D0=B7?= =?UTF-8?q?=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=20readme.ht?= =?UTF-8?q?ml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/prepkg.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build/prepkg.js b/build/prepkg.js index 5e4ba08..60f9171 100644 --- a/build/prepkg.js +++ b/build/prepkg.js @@ -2,6 +2,8 @@ const fs = require('fs-extra'); const path = require('path'); const { execSync } = require('child_process'); +const showdown = require('showdown'); + const platform = process.argv[2]; const distDir = path.resolve(__dirname, '../dist'); @@ -15,6 +17,12 @@ async function build() { await fs.emptyDir(outDir); + //добавляем readme в релиз + let readme = await fs.readFile(path.resolve(__dirname, '../README.md'), 'utf-8'); + const converter = new showdown.Converter(); + readme = converter.makeHtml(readme); + await fs.writeFile(`${outDir}/readme.html`, readme); + // перемещаем public на место if (await fs.pathExists(publicDir)) { From b8b40e8cb037f69c8f61c7db0a130cdbc524b4c3 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 17 Nov 2022 18:00:05 +0700 Subject: [PATCH 15/42] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=20readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9b0f869..56cb118 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ Options: ### Фильтр по авторам и книгам -При создании поисковой БД во время загрузки и парсинга .inpx-файла, имеется возможность +При создании поисковой БД, во время загрузки и парсинга .inpx-файла, имеется возможность отфильтровать авторов и книги, задав определенные критерии. Для этого небходимо создать в рабочей директории (там же, где `config.json`) файл `filter.json` следующего вида: ```json @@ -186,7 +186,7 @@ Options: "excludeAuthors": ["Имя автора"] } ``` -При создании поисковой БД, авторы и книги из `includeAuthors` будут добавлены, а из `excludeAuthors` исключены. +При фильтрации, авторы и их книги из `includeAuthors` будут оставлены, а из `excludeAuthors` исключены. Использование совместно `includeAuthors` и `excludeAuthors` имеет мало смысла, поэтому для включения определенных авторов можно использовать только `includeAuthors`: ```json From e685f136e1e35961763a572379e31450b3af1dd0 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 18 Nov 2022 20:45:38 +0700 Subject: [PATCH 16/42] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BC=D0=B5=D0=BB=D0=BA=D0=BE=D0=B3=D0=BE=20?= =?UTF-8?q?=D0=B1=D0=B0=D0=B3=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/WebWorker.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index 547f55d..6f02b64 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -469,14 +469,14 @@ class WebWorker { const bookFile = `${this.config.filesDir}/${hash}`; const bookFileInfo = `${bookFile}.i.json`; + let rows = await db.select({table: 'book', where: `@@hash('_uid', ${db.esc(bookUid)})`}); + if (!rows.length) + throw new Error('404 Файл не найден'); + const book = rows[0]; + const restoreBookInfo = async(info) => { const result = {}; - let rows = await db.select({table: 'book', where: `@@hash('_uid', ${db.esc(bookUid)})`}); - if (!rows.length) - throw new Error('404 Файл не найден'); - const book = rows[0]; - result.book = book; result.cover = ''; result.fb2 = false; @@ -493,7 +493,8 @@ class WebWorker { } } - Object.assign(info ,result); + Object.assign(info, result); + await fs.writeFile(bookFileInfo, JSON.stringify(info)); if (this.config.branch === 'development') { @@ -513,7 +514,7 @@ class WebWorker { if (tmpInfo.cover) coverFile = `${this.config.publicFilesDir}${tmpInfo.cover}`; - if (coverFile && !await fs.pathExists(coverFile)) { + if (book.id != tmpInfo.book.id || (coverFile && !await fs.pathExists(coverFile))) { await restoreBookInfo(bookInfo); } else { bookInfo = tmpInfo; From 8a71c4040ce2c1b87402bde93d83a5e28e0fdacb Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 20 Nov 2022 17:47:15 +0700 Subject: [PATCH 17/42] =?UTF-8?q?=D0=9D=D0=B0=D1=87=D0=B0=D1=82=D0=B0=20?= =?UTF-8?q?=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20=D0=BD=D0=B0=D0=B4=20opd?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/BasePage.js | 30 ++++++++++++++++++++++++++++++ server/core/opds/RootPage.js | 17 +++++++++++++++++ server/core/opds/index.js | 34 ++++++++++++++++++++++++++++++++++ server/index.js | 2 ++ 4 files changed, 83 insertions(+) create mode 100644 server/core/opds/BasePage.js create mode 100644 server/core/opds/RootPage.js create mode 100644 server/core/opds/index.js diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js new file mode 100644 index 0000000..3527328 --- /dev/null +++ b/server/core/opds/BasePage.js @@ -0,0 +1,30 @@ +const XmlParser = require('../xml/XmlParser'); + +class BasePage { + constructor(config) { + this.config = config; + + this.rootTag = 'feed'; + } + + makeBody(content) { + if (!this.id) + throw new Error('makeBody: no id'); + + content.id = this.id; + + const xml = new XmlParser(); + const xmlObject = {}; + xmlObject[this.rootTag] = content; + + xml.fromObject(xmlObject); + + return xml.toString({format: true}); + } + + async body() { + throw new Error('Body not implemented'); + } +} + +module.exports = BasePage; \ No newline at end of file diff --git a/server/core/opds/RootPage.js b/server/core/opds/RootPage.js new file mode 100644 index 0000000..7b01694 --- /dev/null +++ b/server/core/opds/RootPage.js @@ -0,0 +1,17 @@ +const BasePage = require('./BasePage'); + +class RootPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'root'; + } + + async body() { + const result = {}; + + return this.makeBody(result); + } +} + +module.exports = RootPage; \ No newline at end of file diff --git a/server/core/opds/index.js b/server/core/opds/index.js new file mode 100644 index 0000000..d6ad0d3 --- /dev/null +++ b/server/core/opds/index.js @@ -0,0 +1,34 @@ +const path = require('path'); + +const RootPage = require('./RootPage'); + +module.exports = function(app, config) { + const root = new RootPage(config); + + const pages = new Map([ + ['opds', root] + ]); + + const opds = async(req, res, next) => { + try { + const pageName = path.basename(req.path); + const page = pages.get(pageName); + + if (page) { + res.set('Content-Type', 'application/atom+xml; charset=utf-8'); + + const result = await page.body(req, res); + + if (result !== false) + res.send(result); + } else { + next(); + } + } catch (e) { + res.status(500).send({error: e.message}); + } + }; + + app.get(['/opds', '/opds/*'], opds); +}; + diff --git a/server/index.js b/server/index.js index 9e2e8aa..d86e51c 100644 --- a/server/index.js +++ b/server/index.js @@ -5,6 +5,7 @@ const express = require('express'); const http = require('http'); const WebSocket = require ('ws'); +const opds = require('./core/opds'); const utils = require('./core/utils'); const ayncExit = new (require('./core/AsyncExit'))(); @@ -154,6 +155,7 @@ async function main() { if (devModule) devModule.logQueries(app); + opds(app, config); initStatic(app, config); const { WebSocketController } = require('./controllers'); From 037b42a5b43877837c4787ee926b87a8bf8da3f5 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 20 Nov 2022 19:22:54 +0700 Subject: [PATCH 18/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/BasePage.js | 42 ++++++++++++++++++++++++++++++++---- server/core/opds/RootPage.js | 25 ++++++++++++++++++++- server/index.js | 2 +- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 3527328..7556e18 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -1,21 +1,55 @@ +const WebWorker = require('../WebWorker');//singleton const XmlParser = require('../xml/XmlParser'); class BasePage { constructor(config) { this.config = config; + this.webWorker = new WebWorker(config); this.rootTag = 'feed'; + this.opdsRoot = '/opds'; + } + + makeEntry(entry = {}) { + if (!entry.id) + throw new Error('makeEntry: no id'); + if (!entry.title) + throw new Error('makeEntry: no title'); + + const result = { + updated: (new Date()).toISOString().substring(0, 19) + 'Z', + }; + + return Object.assign(result, entry); + } + + makeLink(attrs) { + return {'*ATTRS': attrs}; + } + + navLink(attrs) { + return this.makeLink({ + href: this.opdsRoot + (attrs.href || ''), + rel: attrs.rel || '', + type: 'application/atom+xml; profile=opds-catalog; kind=navigation', + }); } makeBody(content) { - if (!this.id) - throw new Error('makeBody: no id'); + const base = this.makeEntry({id: this.id, title: this.title}); + base['*ATTRS'] = { + 'xmlns': 'http://www.w3.org/2005/Atom', + 'xmlns:dc': 'http://purl.org/dc/terms/', + 'xmlns:opds': 'http://opds-spec.org/2010/catalog', + }; - content.id = this.id; + base.link = [ + this.navLink({rel: 'start'}), + ]; const xml = new XmlParser(); const xmlObject = {}; - xmlObject[this.rootTag] = content; + xmlObject[this.rootTag] = Object.assign(base, content); xml.fromObject(xmlObject); diff --git a/server/core/opds/RootPage.js b/server/core/opds/RootPage.js index 7b01694..53ca182 100644 --- a/server/core/opds/RootPage.js +++ b/server/core/opds/RootPage.js @@ -5,11 +5,34 @@ class RootPage extends BasePage { super(config); this.id = 'root'; + this.title = ''; } async body() { const result = {}; - + const ww = this.webWorker; + + if (!this.title) { + const dbConfig = await ww.dbConfig(); + const collection = dbConfig.inpxInfo.collection.split('\n'); + this.title = collection[0].trim(); + if (!this.title) + this.title = 'Неизвестная коллекция'; + } + + result.link = [ + this.navLink({rel: 'start'}), + this.navLink({rel: 'self'}), + ]; + + result.entry = [ + this.makeEntry({ + id: 'author', + title: 'Авторы', + link: this.navLink({rel: 'subsection', href: '/author'}), + }), + ]; + return this.makeBody(result); } } diff --git a/server/index.js b/server/index.js index d86e51c..0bc984b 100644 --- a/server/index.js +++ b/server/index.js @@ -5,7 +5,6 @@ const express = require('express'); const http = require('http'); const WebSocket = require ('ws'); -const opds = require('./core/opds'); const utils = require('./core/utils'); const ayncExit = new (require('./core/AsyncExit'))(); @@ -155,6 +154,7 @@ async function main() { if (devModule) devModule.logQueries(app); + const opds = require('./core/opds'); opds(app, config); initStatic(app, config); From aba0c206f8081a2416c031493b096727f2097141 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 20 Nov 2022 19:52:10 +0700 Subject: [PATCH 19/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 21 +++++++++++++++++++++ server/core/opds/BasePage.js | 22 ++++++++++++++++++---- server/core/opds/RootPage.js | 14 ++++---------- server/core/opds/index.js | 27 ++++++++++++++++++--------- 4 files changed, 61 insertions(+), 23 deletions(-) create mode 100644 server/core/opds/AuthorPage.js diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js new file mode 100644 index 0000000..491552b --- /dev/null +++ b/server/core/opds/AuthorPage.js @@ -0,0 +1,21 @@ +const BasePage = require('./BasePage'); + +class AuthorPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'author'; + this.title = 'Авторы'; + } + + async body() { + const result = {}; + + result.entry = [ + ]; + + return this.makeBody(result); + } +} + +module.exports = AuthorPage; \ No newline at end of file diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 7556e18..0b99bd6 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -7,7 +7,7 @@ class BasePage { this.webWorker = new WebWorker(config); this.rootTag = 'feed'; - this.opdsRoot = '/opds'; + this.opdsRoot = config.opdsRoot; } makeEntry(entry = {}) { @@ -23,6 +23,14 @@ class BasePage { return Object.assign(result, entry); } + myEntry() { + return this.makeEntry({ + id: this.id, + title: this.title, + link: this.navLink({rel: 'subsection', href: `/${this.id}`}), + }); + } + makeLink(attrs) { return {'*ATTRS': attrs}; } @@ -35,6 +43,13 @@ class BasePage { }); } + baseLinks() { + return [ + this.navLink({rel: 'start'}), + this.navLink({rel: 'self', href: (this.id ? `/${this.id}` : '')}), + ]; + } + makeBody(content) { const base = this.makeEntry({id: this.id, title: this.title}); base['*ATTRS'] = { @@ -43,9 +58,8 @@ class BasePage { 'xmlns:opds': 'http://opds-spec.org/2010/catalog', }; - base.link = [ - this.navLink({rel: 'start'}), - ]; + if (!content.link) + base.link = this.baseLinks(); const xml = new XmlParser(); const xmlObject = {}; diff --git a/server/core/opds/RootPage.js b/server/core/opds/RootPage.js index 53ca182..ad8e8c5 100644 --- a/server/core/opds/RootPage.js +++ b/server/core/opds/RootPage.js @@ -1,4 +1,5 @@ const BasePage = require('./BasePage'); +const AuthorPage = require('./AuthorPage'); class RootPage extends BasePage { constructor(config) { @@ -6,6 +7,8 @@ class RootPage extends BasePage { this.id = 'root'; this.title = ''; + + this.authorPage = new AuthorPage(config); } async body() { @@ -20,17 +23,8 @@ class RootPage extends BasePage { this.title = 'Неизвестная коллекция'; } - result.link = [ - this.navLink({rel: 'start'}), - this.navLink({rel: 'self'}), - ]; - result.entry = [ - this.makeEntry({ - id: 'author', - title: 'Авторы', - link: this.navLink({rel: 'subsection', href: '/author'}), - }), + this.authorPage.myEntry(), ]; return this.makeBody(result); diff --git a/server/core/opds/index.js b/server/core/opds/index.js index d6ad0d3..6f80b74 100644 --- a/server/core/opds/index.js +++ b/server/core/opds/index.js @@ -1,18 +1,27 @@ -const path = require('path'); - const RootPage = require('./RootPage'); +const AuthorPage = require('./AuthorPage'); module.exports = function(app, config) { - const root = new RootPage(config); + const opdsRoot = '/opds'; + config.opdsRoot = opdsRoot; - const pages = new Map([ - ['opds', root] - ]); + const root = new RootPage(config); + const author = new AuthorPage(config); + + const routes = [ + ['', root], + ['/root', root], + ['/author', author], + ]; + + const pages = new Map(); + for (const r of routes) { + pages.set(`${opdsRoot}${r[0]}`, r[1]); + } const opds = async(req, res, next) => { try { - const pageName = path.basename(req.path); - const page = pages.get(pageName); + const page = pages.get(req.path); if (page) { res.set('Content-Type', 'application/atom+xml; charset=utf-8'); @@ -29,6 +38,6 @@ module.exports = function(app, config) { } }; - app.get(['/opds', '/opds/*'], opds); + app.get([opdsRoot, `${opdsRoot}/*`], opds); }; From d0e79b0abb271f434d191261995a8d89584e063e Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 22 Nov 2022 19:55:54 +0700 Subject: [PATCH 20/42] + "he": "^1.2.0" --- package-lock.json | 5 ++--- package.json | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6deab87..3387606 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "dayjs": "^1.11.6", "express": "^4.18.1", "fs-extra": "^10.1.0", + "he": "^1.2.0", "iconv-lite": "^0.6.3", "jembadb": "^5.1.3", "localforage": "^1.10.0", @@ -4702,7 +4703,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true, "bin": { "he": "bin/he" } @@ -12309,8 +12309,7 @@ "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" }, "html-entities": { "version": "2.3.3", diff --git a/package.json b/package.json index c453c14..f9da849 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "dayjs": "^1.11.6", "express": "^4.18.1", "fs-extra": "^10.1.0", + "he": "^1.2.0", "iconv-lite": "^0.6.3", "jembadb": "^5.1.3", "localforage": "^1.10.0", From 35925dbc6e0254e80e5c2581142bd05ad4a89e06 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Tue, 22 Nov 2022 20:09:00 +0700 Subject: [PATCH 21/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 91 +++++++++++++++++++++++++++++++--- server/core/WebWorker.js | 10 +++- server/core/opds/AuthorPage.js | 60 ++++++++++++++++++++-- server/core/opds/BasePage.js | 66 ++++++++++++++++++++++++ server/core/opds/RootPage.js | 3 +- 5 files changed, 216 insertions(+), 14 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 04d0698..659d7b7 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -534,28 +534,105 @@ class DbSearcher { } } - async getAuthorBookList(authorId) { + async opdsQuery(from, query) { if (this.closed) throw new Error('DbSearcher closed'); - if (!authorId) + if (!['author', 'series', 'title'].includes(from)) + throw new Error(`Unknown value for param 'from'`); + + this.searchFlag++; + + try { + const db = this.db; + + const queryKey = this.queryKey(query); + const opdsKey = `${from}-opds-${queryKey}`; + let result = await this.getCached(opdsKey); + + if (result === null) { + const ids = await this.selectTableIds(from, query); + + const totalFound = ids.length; + const depth = query.depth || 1; + + //группировка по name длиной depth + const found = await db.select({ + table: from, + rawResult: true, + where: ` + const depth = ${db.esc(depth)}; + const group = new Map(); + + const ids = ${db.esc(Array.from(ids))}; + for (const id of ids) { + const row = @unsafeRow(id); + const s = row.name.substring(0, depth); + let g = group.get(s); + if (!g) { + g = {id: row.id, name: s, count: 0}; + group.set(s, g); + } + g.count++; + } + + const result = Array.from(group.values()); + result.sort((a, b) => a.name.localeCompare(b.name)); + + return result; + ` + }); + + result = {found: found[0].rawResult, totalFound}; + + await this.putCached(opdsKey, result); + } + + return result; + } finally { + this.searchFlag--; + } + } + + async getAuthorBookList(authorId, author) { + if (this.closed) + throw new Error('DbSearcher closed'); + + if (!authorId && !author) return {author: '', books: ''}; this.searchFlag++; try { - //выборка книг автора по authorId - const rows = await this.restoreBooks('author', [authorId]) + const db = this.db; - let author = ''; + if (!authorId) { + //восстановим authorId + authorId = 0; + author = author.toLowerCase(); + + const rows = await db.select({ + table: 'author', + rawResult: true, + where: `return Array.from(@dirtyIndexLR('value', ${db.esc(author)}, ${db.esc(author)}))` + }); + + if (rows.length && rows[0].rawResult.length) + authorId = rows[0].rawResult[0]; + } + + //выборка книг автора по authorId + const rows = await this.restoreBooks('author', [authorId]); + + let authorName = ''; let books = ''; if (rows.length) { - author = rows[0].name; + authorName = rows[0].name; books = rows[0].books; } - return {author, books: (books && books.length ? JSON.stringify(books) : '')}; + return {author: authorName, books: (books && books.length ? JSON.stringify(books) : '')}; } finally { this.searchFlag--; } diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index 6f02b64..8c4bac7 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -267,10 +267,16 @@ class WebWorker { return result; } - async getAuthorBookList(authorId) { + async opdsQuery(from, query) { this.checkMyState(); - return await this.dbSearcher.getAuthorBookList(authorId); + return await this.dbSearcher.opdsQuery(from, query); + } + + async getAuthorBookList(authorId, author) { + this.checkMyState(); + + return await this.dbSearcher.getAuthorBookList(authorId, author); } async getSeriesBookList(series) { diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index 491552b..2490ff2 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -8,12 +8,66 @@ class AuthorPage extends BasePage { this.title = 'Авторы'; } - async body() { + bookAuthor(author) { + if (author) { + let a = author.split(','); + return a.slice(0, 3).join(', ') + (a.length > 3 ? ' и др.' : ''); + } + + return ''; + } + + async body(req) { const result = {}; - result.entry = [ - ]; + const query = {author: '', depth: 1, del: 0, limit: 100}; + if (req.query.author) { + query.author = req.query.author; + query.depth = query.author.length + 1; + } + if (req.query.author == '___others') { + query.author = ''; + query.depth = 1; + query.others = true; + } + + const entry = []; + if (query.author && query.author[0] == '=') { + //книги по автору + const bookList = await this.webWorker.getAuthorBookList(0, query.author.substring(1)); + + if (bookList.books) { + const books = JSON.parse(bookList.books); + + for (const book of books) { + const title = book.title || 'Без названия'; + entry.push( + this.makeEntry({ + id: book._uid, + title, + link: this.navLink({rel: 'subsection', href: `/${this.id}?book=${book._uid}`}), + }) + ); + } + } + } else { + //поиск по каталогу + const queryRes = await this.opdsQuery('author', query); + + for (const rec of queryRes) { +console.log(rec); + entry.push( + this.makeEntry({ + id: rec.id, + title: this.bookAuthor(rec.title),//${(query.depth > 1 && rec.count ? ` (${rec.count})` : '')} + link: this.navLink({rel: 'subsection', href: `/${this.id}?author=${rec.q}`}), + }) + ); + } + } + + result.entry = entry; return this.makeBody(result); } } diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 0b99bd6..374ff01 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -1,6 +1,14 @@ +const he = require('he'); + const WebWorker = require('../WebWorker');//singleton const XmlParser = require('../xml/XmlParser'); +const spaceChar = String.fromCodePoint(0x00B7); +const ruAlphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'; +const enAlphabet = 'abcdefghijklmnopqrstuvwxyz'; +const enruArr = (ruAlphabet + enAlphabet).split(''); +const enru = new Set(enruArr); + class BasePage { constructor(config) { this.config = config; @@ -16,6 +24,8 @@ class BasePage { if (!entry.title) throw new Error('makeEntry: no title'); + entry.title = he.escape(entry.title); + const result = { updated: (new Date()).toISOString().substring(0, 19) + 'Z', }; @@ -73,6 +83,62 @@ class BasePage { async body() { throw new Error('Body not implemented'); } + + // -- stuff ------------------------------------------- + async search(from, query) { + const result = []; + const queryRes = await this.webWorker.search(from, query); + + for (const row of queryRes.found) { + const rec = { + id: row.id, + title: '=' + (row[from] || 'Без имени'), + q: `=${encodeURIComponent(row[from])}`, + }; + + result.push(rec); + } + + return result; + } + + async opdsQuery(from, query) { + const result = []; + + const queryRes = await this.webWorker.opdsQuery(from, query); + let count = 0; + for (const row of queryRes.found) + count += row.count; + + if (count <= query.limit) + return await this.search(from, query); + + const names = new Set(); + const others = []; + for (const row of queryRes.found) { + const name = row.name.toUpperCase(); + + if (!names.has(name)) { + const rec = { + id: row.id, + title: name.replace(/ /g, spaceChar), + q: encodeURIComponent(row.name.toLowerCase()), + count: row.count, + }; + if (query.depth > 1 || enru.has(row.name[0].toLowerCase())) { + result.push(rec); + } else { + others.push(rec); + } + names.add(name); + } + } + + if (!query.others && query.depth == 1) + result.push({id: 'other', title: 'Все остальные', q: '___others'}); + + return (!query.others ? result : others); + } } module.exports = BasePage; \ No newline at end of file diff --git a/server/core/opds/RootPage.js b/server/core/opds/RootPage.js index ad8e8c5..ffbe16c 100644 --- a/server/core/opds/RootPage.js +++ b/server/core/opds/RootPage.js @@ -13,10 +13,9 @@ class RootPage extends BasePage { async body() { const result = {}; - const ww = this.webWorker; if (!this.title) { - const dbConfig = await ww.dbConfig(); + const dbConfig = await this.webWorker.dbConfig(); const collection = dbConfig.inpxInfo.collection.split('\n'); this.title = collection[0].trim(); if (!this.title) From 8cf370c79d8a44756a65fc08d5b395860785206d Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 23 Nov 2022 01:21:29 +0700 Subject: [PATCH 22/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 110 +++++++++++++++++++---- server/core/opds/BasePage.js | 156 +++++++++++++++++++++++++++++++-- server/core/opds/BookPage.js | 37 ++++++++ server/core/opds/index.js | 6 ++ 4 files changed, 287 insertions(+), 22 deletions(-) create mode 100644 server/core/opds/BookPage.js diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index 2490ff2..42f732e 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -17,51 +17,129 @@ class AuthorPage extends BasePage { return ''; } + sortBooks(bookList) { + //схлопывание серий + const books = []; + const seriesSet = new Set(); + for (const book of bookList) { + if (book.series) { + if (!seriesSet.has(book.series)) { + books.push({ + type: 'series', + book + }); + + seriesSet.add(book.series); + } + } else { + books.push({ + type: 'book', + book + }); + } + } + + //сортировка + books.sort((a, b) => { + if (a.type == 'series') { + return (b.type == 'series' ? a.book.series.localeCompare(b.book.series) : -1); + } else { + return (b.type == 'book' ? a.book.title.localeCompare(b.book.title) : 1); + } + }); + + return books; + } + + sortSeriesBooks(seriesBooks) { + seriesBooks.sort((a, b) => { + const dserno = (a.serno || Number.MAX_VALUE) - (b.serno || Number.MAX_VALUE); + const dtitle = a.title.localeCompare(b.title); + const dext = a.ext.localeCompare(b.ext); + return (dserno ? dserno : (dtitle ? dtitle : dext)); + }); + + return seriesBooks; + } + async body(req) { const result = {}; - const query = {author: '', depth: 1, del: 0, limit: 100}; - if (req.query.author) { - query.author = req.query.author; - query.depth = query.author.length + 1; - } + const query = { + author: req.query.author || '', + series: req.query.series || '', + depth: 0, + del: 0, + limit: 100 + }; + query.depth = query.author.length + 1; - if (req.query.author == '___others') { + if (query.author == '___others') { query.author = ''; query.depth = 1; query.others = true; } const entry = []; - if (query.author && query.author[0] == '=') { - //книги по автору - const bookList = await this.webWorker.getAuthorBookList(0, query.author.substring(1)); - + if (query.series) { + //книги по серии + const bookList = await this.webWorker.getSeriesBookList(query.series); if (bookList.books) { - const books = JSON.parse(bookList.books); + let books = JSON.parse(bookList.books); + books = this.sortSeriesBooks(this.filterBooks(books, query)); for (const book of books) { - const title = book.title || 'Без названия'; + const title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'}`; entry.push( this.makeEntry({ id: book._uid, title, - link: this.navLink({rel: 'subsection', href: `/${this.id}?book=${book._uid}`}), + link: this.navLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), }) ); } } + } else if (query.author && query.author[0] == '=') { + //книги по автору + const bookList = await this.webWorker.getAuthorBookList(0, query.author.substring(1)); + + if (bookList.books) { + let books = JSON.parse(bookList.books); + books = this.sortBooks(this.filterBooks(books, query)); + + for (const b of books) { + if (b.type == 'series') { + entry.push( + this.makeEntry({ + id: b.book._uid, + title: `Серия: ${b.book.series}`, + link: this.navLink({ + href: `/${this.id}?author=${encodeURIComponent(query.author)}` + + `&series=${encodeURIComponent(b.book.series)}`}), + }) + ); + } else { + const title = b.book.title || 'Без названия'; + entry.push( + this.makeEntry({ + id: b.book._uid, + title, + link: this.navLink({href: `/book?uid=${encodeURIComponent(b.book._uid)}`}), + }) + ); + } + } + } } else { //поиск по каталогу const queryRes = await this.opdsQuery('author', query); - for (const rec of queryRes) { -console.log(rec); + for (const rec of queryRes) { entry.push( this.makeEntry({ id: rec.id, title: this.bookAuthor(rec.title),//${(query.depth > 1 && rec.count ? ` (${rec.count})` : '')} - link: this.navLink({rel: 'subsection', href: `/${this.id}?author=${rec.q}`}), + link: this.navLink({href: `/${this.id}?author=${rec.q}`}), }) ); } diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 374ff01..7f8d299 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -1,9 +1,12 @@ +const _ = require('lodash'); const he = require('he'); const WebWorker = require('../WebWorker');//singleton const XmlParser = require('../xml/XmlParser'); const spaceChar = String.fromCodePoint(0x00B7); +const emptyFieldValue = '?'; +const maxUtf8Char = String.fromCodePoint(0xFFFFF); const ruAlphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'; const enAlphabet = 'abcdefghijklmnopqrstuvwxyz'; const enruArr = (ruAlphabet + enAlphabet).split(''); @@ -37,7 +40,7 @@ class BasePage { return this.makeEntry({ id: this.id, title: this.title, - link: this.navLink({rel: 'subsection', href: `/${this.id}`}), + link: this.navLink({href: `/${this.id}`}), }); } @@ -48,11 +51,35 @@ class BasePage { navLink(attrs) { return this.makeLink({ href: this.opdsRoot + (attrs.href || ''), - rel: attrs.rel || '', + rel: attrs.rel || 'subsection', type: 'application/atom+xml; profile=opds-catalog; kind=navigation', }); } + acqLink(attrs) { + if (!attrs.href) + throw new Error('acqLink: no href'); + if (!attrs.type) + throw new Error('acqLink: no type'); + + return this.makeLink({ + href: attrs.href, + rel: 'http://opds-spec.org/acquisition/open-access', + type: attrs.type, + }); + } + + imgLink(attrs) { + if (!attrs.href) + throw new Error('acqLink: no href'); + + return this.makeLink({ + href: attrs.href, + rel: `http://opds-spec.org/image${attrs.thumb ? '/thumbnail' : ''}`, + type: attrs.type || 'image/jpeg', + }); + } + baseLinks() { return [ this.navLink({rel: 'start'}), @@ -92,7 +119,7 @@ class BasePage { for (const row of queryRes.found) { const rec = { id: row.id, - title: '=' + (row[from] || 'Без имени'), + title: (row[from] || 'Без автора'), q: `=${encodeURIComponent(row[from])}`, }; @@ -103,8 +130,6 @@ class BasePage { } async opdsQuery(from, query) { - const result = []; - const queryRes = await this.webWorker.opdsQuery(from, query); let count = 0; for (const row of queryRes.found) @@ -113,8 +138,9 @@ class BasePage { if (count <= query.limit) return await this.search(from, query); - const names = new Set(); + const result = []; const others = []; + const names = new Set(); for (const row of queryRes.found) { const name = row.name.toUpperCase(); @@ -134,11 +160,129 @@ class BasePage { } } + if (query.depth > 1 && result.length == 1 && query[from]) { + const newQuery = _.cloneDeep(query); + newQuery[from] = decodeURIComponent(result[0].q); + if (newQuery[from].length >= query.depth) { + newQuery.depth = newQuery[from].length + 1; + return await this.opdsQuery(from, newQuery); + } + } + if (!query.others && query.depth == 1) result.push({id: 'other', title: 'Все остальные', q: '___others'}); return (!query.others ? result : others); } + + //скопировано из BaseList.js, часть функционала не используется + filterBooks(books, query) { + const s = query; + + const splitAuthor = (author) => { + if (!author) { + author = emptyFieldValue; + } + + const result = author.split(','); + if (result.length > 1) + result.push(author); + + return result; + }; + + const filterBySearch = (bookValue, searchValue) => { + if (!searchValue) + return true; + + if (!bookValue) + bookValue = emptyFieldValue; + + bookValue = bookValue.toLowerCase(); + searchValue = searchValue.toLowerCase(); + + //особая обработка префиксов + if (searchValue[0] == '=') { + + searchValue = searchValue.substring(1); + return bookValue.localeCompare(searchValue) == 0; + } else if (searchValue[0] == '*') { + + searchValue = searchValue.substring(1); + return bookValue !== emptyFieldValue && bookValue.indexOf(searchValue) >= 0; + } else if (searchValue[0] == '#') { + + searchValue = searchValue.substring(1); + return !bookValue || (bookValue !== emptyFieldValue && !enru.has(bookValue[0]) && bookValue.indexOf(searchValue) >= 0); + } else { + //where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`; + return bookValue.localeCompare(searchValue) >= 0 && bookValue.localeCompare(searchValue + maxUtf8Char) <= 0; + } + }; + + return books.filter((book) => { + //author + let authorFound = false; + const authors = splitAuthor(book.author); + for (const a of authors) { + if (filterBySearch(a, s.author)) { + authorFound = true; + break; + } + } + + //genre + let genreFound = !s.genre; + if (!genreFound) { + const searchGenres = new Set(s.genre.split(',')); + const bookGenres = book.genre.split(','); + + for (let g of bookGenres) { + if (!g) + g = emptyFieldValue; + + if (searchGenres.has(g)) { + genreFound = true; + break; + } + } + } + + //lang + let langFound = !s.lang; + if (!langFound) { + const searchLang = new Set(s.lang.split(',')); + langFound = searchLang.has(book.lang || emptyFieldValue); + } + + //date + let dateFound = !s.date; + if (!dateFound) { + const date = this.queryDate(s.date).split(','); + let [from = '0000-00-00', to = '9999-99-99'] = date; + + dateFound = (book.date >= from && book.date <= to); + } + + //librate + let librateFound = !s.librate; + if (!librateFound) { + const searchLibrate = new Set(s.librate.split(',').map(n => parseInt(n, 10)).filter(n => !isNaN(n))); + librateFound = searchLibrate.has(book.librate); + } + + return (this.showDeleted || !book.del) + && authorFound + && filterBySearch(book.series, s.series) + && filterBySearch(book.title, s.title) + && genreFound + && langFound + && dateFound + && librateFound + ; + }); + } + } module.exports = BasePage; \ No newline at end of file diff --git a/server/core/opds/BookPage.js b/server/core/opds/BookPage.js new file mode 100644 index 0000000..469e5db --- /dev/null +++ b/server/core/opds/BookPage.js @@ -0,0 +1,37 @@ +const BasePage = require('./BasePage'); + +class BookPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'book'; + this.title = 'Книга'; + } + + async body(req) { + const result = {}; + + const bookUid = req.query.uid; + const entry = []; + if (bookUid) { + const {bookInfo} = await this.webWorker.getBookInfo(bookUid); + if (bookInfo) { + entry.push( + this.makeEntry({ + id: bookUid, + title: bookInfo.book.title || 'Без названия', + link: [ + //this.imgLink({href: bookInfo.cover, type: coverType}), + this.acqLink({href: bookInfo.link, type: `application/${bookInfo.book.ext}+gzip`}), + ], + }) + ); + } + } + + result.entry = entry; + return this.makeBody(result); + } +} + +module.exports = BookPage; \ No newline at end of file diff --git a/server/core/opds/index.js b/server/core/opds/index.js index 6f80b74..95c1ede 100644 --- a/server/core/opds/index.js +++ b/server/core/opds/index.js @@ -1,5 +1,6 @@ const RootPage = require('./RootPage'); const AuthorPage = require('./AuthorPage'); +const BookPage = require('./BookPage'); module.exports = function(app, config) { const opdsRoot = '/opds'; @@ -7,11 +8,13 @@ module.exports = function(app, config) { const root = new RootPage(config); const author = new AuthorPage(config); + const book = new BookPage(config); const routes = [ ['', root], ['/root', root], ['/author', author], + ['/book', book], ]; const pages = new Map(); @@ -35,6 +38,9 @@ module.exports = function(app, config) { } } catch (e) { res.status(500).send({error: e.message}); + if (config.branch == 'development') { + console.error({error: e.message, url: req.originalUrl}); + } } }; From a6d9df7dec78115785573bc62b46674a2b8d54f2 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 23 Nov 2022 14:38:23 +0700 Subject: [PATCH 23/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 6 +++--- server/core/opds/BasePage.js | 28 ++++++++++++++++++---------- server/core/opds/BookPage.js | 31 ++++++++++++++++++++----------- server/core/opds/RootPage.js | 4 ++-- 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index 42f732e..8e80742 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -94,7 +94,7 @@ class AuthorPage extends BasePage { this.makeEntry({ id: book._uid, title, - link: this.navLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), + link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), }) ); } @@ -124,7 +124,7 @@ class AuthorPage extends BasePage { this.makeEntry({ id: b.book._uid, title, - link: this.navLink({href: `/book?uid=${encodeURIComponent(b.book._uid)}`}), + link: this.acqLink({href: `/book?uid=${encodeURIComponent(b.book._uid)}`}), }) ); } @@ -146,7 +146,7 @@ class AuthorPage extends BasePage { } result.entry = entry; - return this.makeBody(result); + return this.makeBody(result, req); } } diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 7f8d299..08a1614 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -50,28 +50,36 @@ class BasePage { navLink(attrs) { return this.makeLink({ - href: this.opdsRoot + (attrs.href || ''), + href: (attrs.hrefAsIs ? attrs.href : `${this.opdsRoot}${attrs.href || ''}`), rel: attrs.rel || 'subsection', - type: 'application/atom+xml; profile=opds-catalog; kind=navigation', + type: 'application/atom+xml;profile=opds-catalog;kind=navigation', }); } acqLink(attrs) { + return this.makeLink({ + href: this.opdsRoot + (attrs.href || ''), + rel: attrs.rel || 'subsection', + type: 'application/atom+xml;profile=opds-catalog;kind=acquisition', + }); + } + + downLink(attrs) { if (!attrs.href) - throw new Error('acqLink: no href'); + throw new Error('downLink: no href'); if (!attrs.type) - throw new Error('acqLink: no type'); + throw new Error('downLink: no type'); return this.makeLink({ href: attrs.href, - rel: 'http://opds-spec.org/acquisition/open-access', + rel: 'http://opds-spec.org/acquisition', type: attrs.type, }); } imgLink(attrs) { if (!attrs.href) - throw new Error('acqLink: no href'); + throw new Error('imgLink: no href'); return this.makeLink({ href: attrs.href, @@ -80,14 +88,14 @@ class BasePage { }); } - baseLinks() { + baseLinks(req) { return [ this.navLink({rel: 'start'}), - this.navLink({rel: 'self', href: (this.id ? `/${this.id}` : '')}), + this.navLink({rel: 'self', href: req.originalUrl, hrefAsIs: true}), ]; } - makeBody(content) { + makeBody(content, req) { const base = this.makeEntry({id: this.id, title: this.title}); base['*ATTRS'] = { 'xmlns': 'http://www.w3.org/2005/Atom', @@ -96,7 +104,7 @@ class BasePage { }; if (!content.link) - base.link = this.baseLinks(); + base.link = this.baseLinks(req); const xml = new XmlParser(); const xmlObject = {}; diff --git a/server/core/opds/BookPage.js b/server/core/opds/BookPage.js index 469e5db..88db7d9 100644 --- a/server/core/opds/BookPage.js +++ b/server/core/opds/BookPage.js @@ -1,3 +1,4 @@ +const path = require('path'); const BasePage = require('./BasePage'); class BookPage extends BasePage { @@ -16,21 +17,29 @@ class BookPage extends BasePage { if (bookUid) { const {bookInfo} = await this.webWorker.getBookInfo(bookUid); if (bookInfo) { - entry.push( - this.makeEntry({ - id: bookUid, - title: bookInfo.book.title || 'Без названия', - link: [ - //this.imgLink({href: bookInfo.cover, type: coverType}), - this.acqLink({href: bookInfo.link, type: `application/${bookInfo.book.ext}+gzip`}), - ], - }) - ); + const e = this.makeEntry({ + id: bookUid, + title: bookInfo.book.title || 'Без названия', + link: [ + this.downLink({href: bookInfo.link, type: `application/${bookInfo.book.ext}+zip`}), + ], + }); + + if (bookInfo.cover) { + let coverType = 'image/jpeg'; + if (path.extname(bookInfo.cover) == '.png') + coverType = 'image/png'; + + e.link.push(this.imgLink({href: bookInfo.cover, type: coverType})); + e.link.push(this.imgLink({href: bookInfo.cover, type: coverType, thumb: true})); + } + + entry.push(e); } } result.entry = entry; - return this.makeBody(result); + return this.makeBody(result, req); } } diff --git a/server/core/opds/RootPage.js b/server/core/opds/RootPage.js index ffbe16c..3edb10a 100644 --- a/server/core/opds/RootPage.js +++ b/server/core/opds/RootPage.js @@ -11,7 +11,7 @@ class RootPage extends BasePage { this.authorPage = new AuthorPage(config); } - async body() { + async body(req) { const result = {}; if (!this.title) { @@ -26,7 +26,7 @@ class RootPage extends BasePage { this.authorPage.myEntry(), ]; - return this.makeBody(result); + return this.makeBody(result, req); } } From 5a04e4f0c7409d908cd02331d6281335c14aeb9f Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 23 Nov 2022 14:59:29 +0700 Subject: [PATCH 24/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index 8e80742..2ad07e9 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -68,6 +68,7 @@ class AuthorPage extends BasePage { const query = { author: req.query.author || '', series: req.query.series || '', + all: req.query.all || '', depth: 0, del: 0, limit: 100 @@ -84,12 +85,18 @@ class AuthorPage extends BasePage { if (query.series) { //книги по серии const bookList = await this.webWorker.getSeriesBookList(query.series); + if (bookList.books) { let books = JSON.parse(bookList.books); - books = this.sortSeriesBooks(this.filterBooks(books, query)); + const filtered = (query.all ? books : this.filterBooks(books, query)); + const sorted = this.sortSeriesBooks(filtered); + + for (const book of sorted) { + let title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'}`; + if (query.all) { + title = `${this.bookAuthor(book.author)} "${title}"`; + } - for (const book of books) { - const title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'}`; entry.push( this.makeEntry({ id: book._uid, @@ -98,6 +105,18 @@ class AuthorPage extends BasePage { }) ); } + + if (books.length > filtered.length) { + entry.push( + this.makeEntry({ + id: 'all_series_books', + title: 'Все книги серии', + link: this.navLink({ + href: `/${this.id}?author=${encodeURIComponent(query.author)}` + + `&series=${encodeURIComponent(query.series)}&all=1`}), + }) + ); + } } } else if (query.author && query.author[0] == '=') { //книги по автору From a8ed8b29e581ddeb329a05841cc99a0ec71eab1e Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 23 Nov 2022 17:03:33 +0700 Subject: [PATCH 25/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 3 +- server/core/opds/BasePage.js | 66 +++++++++---- server/core/opds/BookPage.js | 171 ++++++++++++++++++++++++++++++++- 3 files changed, 214 insertions(+), 26 deletions(-) diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index 2ad07e9..fb31546 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -85,7 +85,7 @@ class AuthorPage extends BasePage { if (query.series) { //книги по серии const bookList = await this.webWorker.getSeriesBookList(query.series); - + if (bookList.books) { let books = JSON.parse(bookList.books); const filtered = (query.all ? books : this.filterBooks(books, query)); @@ -96,6 +96,7 @@ class AuthorPage extends BasePage { if (query.all) { title = `${this.bookAuthor(book.author)} "${title}"`; } + title += ` (${book.ext})`; entry.push( this.makeEntry({ diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 08a1614..645e279 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -58,7 +58,7 @@ class BasePage { acqLink(attrs) { return this.makeLink({ - href: this.opdsRoot + (attrs.href || ''), + href: (attrs.hrefAsIs ? attrs.href : `${this.opdsRoot}${attrs.href || ''}`), rel: attrs.rel || 'subsection', type: 'application/atom+xml;profile=opds-catalog;kind=acquisition', }); @@ -143,41 +143,43 @@ class BasePage { for (const row of queryRes.found) count += row.count; - if (count <= query.limit) - return await this.search(from, query); - - const result = []; const others = []; - const names = new Set(); - for (const row of queryRes.found) { - const name = row.name.toUpperCase(); + let result = []; + if (count <= query.limit) { + result = await this.search(from, query); + } else { + const names = new Set(); + for (const row of queryRes.found) { + const name = row.name.toUpperCase(); - if (!names.has(name)) { - const rec = { - id: row.id, - title: name.replace(/ /g, spaceChar), - q: encodeURIComponent(row.name.toLowerCase()), - count: row.count, - }; - if (query.depth > 1 || enru.has(row.name[0].toLowerCase())) { - result.push(rec); - } else { - others.push(rec); + if (!names.has(name)) { + const rec = { + id: row.id, + title: name.replace(/ /g, spaceChar), + q: encodeURIComponent(row.name.toLowerCase()), + count: row.count, + }; + if (query.depth > 1 || enru.has(row.name[0].toLowerCase())) { + result.push(rec); + } else { + others.push(rec); + } + names.add(name); } - names.add(name); } } if (query.depth > 1 && result.length == 1 && query[from]) { const newQuery = _.cloneDeep(query); newQuery[from] = decodeURIComponent(result[0].q); + if (newQuery[from].length >= query.depth) { newQuery.depth = newQuery[from].length + 1; return await this.opdsQuery(from, newQuery); } } - if (!query.others && query.depth == 1) + if (!query.others && others.length) result.push({id: 'other', title: 'Все остальные', q: '___others'}); return (!query.others ? result : others); @@ -291,6 +293,28 @@ class BasePage { }); } + async getGenres() { + let result; + if (!this.genres) { + const res = await this.webWorker.getGenreTree(); + + result = { + genreTree: res.genreTree, + genreMap: new Map(), + }; + + for (const section of result.genreTree) { + for (const g of section.value) + result.genreMap.set(g.value, g.name); + } + + this.genres = result; + } else { + result = this.genres; + } + + return result; + } } module.exports = BasePage; \ No newline at end of file diff --git a/server/core/opds/BookPage.js b/server/core/opds/BookPage.js index 88db7d9..c765d1d 100644 --- a/server/core/opds/BookPage.js +++ b/server/core/opds/BookPage.js @@ -1,5 +1,10 @@ const path = require('path'); +const _ = require('lodash'); +const he = require('he'); +const dayjs = require('dayjs'); + const BasePage = require('./BasePage'); +const Fb2Parser = require('../fb2/Fb2Parser'); class BookPage extends BasePage { constructor(config) { @@ -7,24 +12,181 @@ class BookPage extends BasePage { this.id = 'book'; this.title = 'Книга'; + + } + + formatSize(size) { + size = size/1024; + let unit = 'KB'; + if (size > 1024) { + size = size/1024; + unit = 'MB'; + } + return `${size.toFixed(1)} ${unit}`; + } + + inpxInfo(bookRec) { + const mapping = [ + {name: 'fileInfo', label: 'Информация о файле', value: [ + {name: 'folder', label: 'Папка'}, + {name: 'file', label: 'Файл'}, + {name: 'size', label: 'Размер'}, + {name: 'date', label: 'Добавлен'}, + {name: 'del', label: 'Удален'}, + {name: 'libid', label: 'LibId'}, + {name: 'insno', label: 'InsideNo'}, + ]}, + + {name: 'titleInfo', label: 'Общая информация', value: [ + {name: 'author', label: 'Автор(ы)'}, + {name: 'title', label: 'Название'}, + {name: 'series', label: 'Серия'}, + {name: 'genre', label: 'Жанр'}, + {name: 'librate', label: 'Оценка'}, + {name: 'lang', label: 'Язык книги'}, + {name: 'keywords', label: 'Ключевые слова'}, + ]}, + ]; + + const valueToString = (value, nodePath, b) => {//eslint-disable-line no-unused-vars + if (nodePath == 'fileInfo/file') + return `${value}.${b.ext}`; + + if (nodePath == 'fileInfo/size') + return `${this.formatSize(value)} (${value.toString().replace(/(\d)(?=(\d{3})+(?!\d))/g, '$1 ')} Bytes)`; + + if (nodePath == 'fileInfo/date') + return dayjs(value, 'YYYY-MM-DD').format('DD.MM.YYYY'); + + if (nodePath == 'fileInfo/del') + return (value ? 'Да' : null); + + if (nodePath == 'fileInfo/insno') + return (value ? value : null); + + if (nodePath == 'titleInfo/author') + return value.split(',').join(', '); + + if (nodePath == 'titleInfo/librate' && !value) + return null; + + if (typeof(value) === 'string') { + return value; + } + + return (value.toString ? value.toString() : ''); + }; + + let result = []; + const book = _.cloneDeep(bookRec); + book.series = [book.series, book.serno].filter(v => v).join(' #'); + + for (const item of mapping) { + const itemOut = {name: item.name, label: item.label, value: []}; + + for (const subItem of item.value) { + const subItemOut = { + name: subItem.name, + label: subItem.label, + value: valueToString(book[subItem.name], `${item.name}/${subItem.name}`, book) + }; + if (subItemOut.value) + itemOut.value.push(subItemOut); + } + + if (itemOut.value.length) + result.push(itemOut); + } + + return result; + } + + htmlInfo(title, infoList) { + let info = ''; + for (const part of infoList) { + if (part.value.length) + info += `

${part.label}

`; + for (const rec of part.value) + info += `

${rec.label}: ${rec.value}

`; + } + + if (info) + info = `

${title}

${info}`; + + return info; } async body(req) { const result = {}; + result.link = [ + this.navLink({rel: 'start'}), + this.acqLink({rel: 'self', href: req.originalUrl, hrefAsIs: true}), + ]; + const bookUid = req.query.uid; const entry = []; - if (bookUid) { + if (bookUid) { const {bookInfo} = await this.webWorker.getBookInfo(bookUid); + if (bookInfo) { + const {genreMap} = await this.getGenres(); + const fileFormat = `${bookInfo.book.ext}+zip`; + + //entry const e = this.makeEntry({ id: bookUid, title: bookInfo.book.title || 'Без названия', - link: [ - this.downLink({href: bookInfo.link, type: `application/${bookInfo.book.ext}+zip`}), - ], }); + e['dc:language'] = bookInfo.book.lang; + e['dc:format'] = fileFormat; + + //genre + const genre = bookInfo.book.genre.split(','); + for (const g of genre) { + const genreName = genreMap.get(g); + if (genreName) { + if (!e.category) + e.category = []; + e.category.push({ + '*ATTRS': {term: genreName, label: genreName}, + }); + } + } + + let content = ''; + let ann = ''; + let info = ''; + //fb2 info + if (bookInfo.fb2) { + const parser = new Fb2Parser(bookInfo.fb2); + const infoObj = parser.bookInfo(); + + if (infoObj.titleInfo) { + if (infoObj.titleInfo.author.length) { + e.author = infoObj.titleInfo.author.map(a => ({name: a})); + } + + ann = infoObj.titleInfo.annotationHtml || ''; + const infoList = parser.bookInfoList(infoObj); + info += this.htmlInfo('Fb2 инфо', infoList); + } + } + + //content + info += this.htmlInfo('Inpx инфо', this.inpxInfo(bookInfo.book)); + + content = `${ann}${info}`; + if (content) { + e.content = { + '*ATTRS': {type: 'text/html'}, + '*TEXT': he.escape(content), + }; + } + + //links + e.link = [ this.downLink({href: bookInfo.link, type: `application/${fileFormat}`}) ]; if (bookInfo.cover) { let coverType = 'image/jpeg'; if (path.extname(bookInfo.cover) == '.png') @@ -39,6 +201,7 @@ class BookPage extends BasePage { } result.entry = entry; + return this.makeBody(result, req); } } From 410aa01ac9bc27dd4cc89ff0355dac7dde0043aa Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 23 Nov 2022 17:31:47 +0700 Subject: [PATCH 26/42] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=B1=D0=B0=D0=B3=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 2 +- server/core/opds/BasePage.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index fb31546..5733f59 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -139,7 +139,7 @@ class AuthorPage extends BasePage { }) ); } else { - const title = b.book.title || 'Без названия'; + const title = `${b.book.title || 'Без названия'} (${b.book.ext})`; entry.push( this.makeEntry({ id: b.book._uid, diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 645e279..85d316b 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -45,6 +45,7 @@ class BasePage { } makeLink(attrs) { + attrs.href = he.escape(attrs.href); return {'*ATTRS': attrs}; } From cac8e7c721895efa18b4ffa969e0bb4ba28afc87 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 23 Nov 2022 18:08:31 +0700 Subject: [PATCH 27/42] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B8=20=D0=B1=D0=B0=D0=B3=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 2 +- server/core/opds/BasePage.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index 5733f59..f2d81e5 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -152,7 +152,7 @@ class AuthorPage extends BasePage { } } else { //поиск по каталогу - const queryRes = await this.opdsQuery('author', query); + const queryRes = await this.opdsQuery('author', query, 'Остальные авторы'); for (const rec of queryRes) { entry.push( diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 85d316b..ceb9b5a 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -138,7 +138,7 @@ class BasePage { return result; } - async opdsQuery(from, query) { + async opdsQuery(from, query, otherTitle = 'Другие') { const queryRes = await this.webWorker.opdsQuery(from, query); let count = 0; for (const row of queryRes.found) @@ -181,7 +181,7 @@ class BasePage { } if (!query.others && others.length) - result.push({id: 'other', title: 'Все остальные', q: '___others'}); + result.unshift({id: 'other', title: otherTitle, q: '___others'}); return (!query.others ? result : others); } From 6a3b919f5f6f4eb06643513d6ef54d27728b67e7 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 23 Nov 2022 19:17:08 +0700 Subject: [PATCH 28/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 48 ++++++++++++++--------- server/core/opds/BasePage.js | 5 ++- server/core/opds/GenrePage.js | 72 ++++++++++++++++++++++++++++++++++ server/core/opds/index.js | 3 ++ 4 files changed, 109 insertions(+), 19 deletions(-) create mode 100644 server/core/opds/GenrePage.js diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index f2d81e5..00c6986 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -68,10 +68,12 @@ class AuthorPage extends BasePage { const query = { author: req.query.author || '', series: req.query.series || '', + genre: req.query.genre || '', + del: 0, + limit: 100, + all: req.query.all || '', depth: 0, - del: 0, - limit: 100 }; query.depth = query.author.length + 1; @@ -91,6 +93,18 @@ class AuthorPage extends BasePage { const filtered = (query.all ? books : this.filterBooks(books, query)); const sorted = this.sortSeriesBooks(filtered); + if (books.length > filtered.length) { + entry.push( + this.makeEntry({ + id: 'all_series_books', + title: '[Все книги серии]', + link: this.navLink({ + href: `/${this.id}?author=${encodeURIComponent(query.author)}` + + `&series=${encodeURIComponent(query.series)}&all=1`}), + }) + ); + } + for (const book of sorted) { let title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'}`; if (query.all) { @@ -106,18 +120,6 @@ class AuthorPage extends BasePage { }) ); } - - if (books.length > filtered.length) { - entry.push( - this.makeEntry({ - id: 'all_series_books', - title: 'Все книги серии', - link: this.navLink({ - href: `/${this.id}?author=${encodeURIComponent(query.author)}` + - `&series=${encodeURIComponent(query.series)}&all=1`}), - }) - ); - } } } else if (query.author && query.author[0] == '=') { //книги по автору @@ -135,7 +137,7 @@ class AuthorPage extends BasePage { title: `Серия: ${b.book.series}`, link: this.navLink({ href: `/${this.id}?author=${encodeURIComponent(query.author)}` + - `&series=${encodeURIComponent(b.book.series)}`}), + `&series=${encodeURIComponent(b.book.series)}&genre=${encodeURIComponent(query.genre)}`}), }) ); } else { @@ -151,15 +153,25 @@ class AuthorPage extends BasePage { } } } else { - //поиск по каталогу - const queryRes = await this.opdsQuery('author', query, 'Остальные авторы'); + if (query.depth == 1 && !query.genre && !query.others) { + entry.push( + this.makeEntry({ + id: 'select_genre', + title: '[Выбрать жанр]', + link: this.navLink({href: `/genre?from=${this.id}`}), + }) + ); + } + + //навигация по каталогу + const queryRes = await this.opdsQuery('author', query, '[Остальные авторы]'); for (const rec of queryRes) { entry.push( this.makeEntry({ id: rec.id, title: this.bookAuthor(rec.title),//${(query.depth > 1 && rec.count ? ` (${rec.count})` : '')} - link: this.navLink({href: `/${this.id}?author=${rec.q}`}), + link: this.navLink({href: `/${this.id}?author=${rec.q}&genre=${encodeURIComponent(query.genre)}`}), }) ); } diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index ceb9b5a..6dcec84 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -138,7 +138,7 @@ class BasePage { return result; } - async opdsQuery(from, query, otherTitle = 'Другие') { + async opdsQuery(from, query, otherTitle = '[Другие]') { const queryRes = await this.webWorker.opdsQuery(from, query); let count = 0; for (const row of queryRes.found) @@ -302,9 +302,12 @@ class BasePage { result = { genreTree: res.genreTree, genreMap: new Map(), + genreSection: new Map(), }; for (const section of result.genreTree) { + result.genreSection.set(section.name, section.value); + for (const g of section.value) result.genreMap.set(g.value, g.name); } diff --git a/server/core/opds/GenrePage.js b/server/core/opds/GenrePage.js new file mode 100644 index 0000000..f9df2b9 --- /dev/null +++ b/server/core/opds/GenrePage.js @@ -0,0 +1,72 @@ +const BasePage = require('./BasePage'); + +class GenrePage extends BasePage { + constructor(config) { + super(config); + + this.id = 'genre'; + this.title = 'Жанры'; + + } + + async body(req) { + const result = {}; + + const query = { + from: req.query.from || '', + section: req.query.section || '', + }; + + const entry = []; + if (query.from) { + + if (query.section) { + //выбираем подразделы + const {genreSection} = await this.getGenres(); + const section = genreSection.get(query.section); + + if (section) { + let id = 0; + const all = []; + for (const g of section) { + all.push(g.value); + entry.push( + this.makeEntry({ + id: ++id, + title: g.name, + link: this.navLink({href: `/${encodeURIComponent(query.from)}?genre=${encodeURIComponent(g.value)}`}), + }) + ); + } + + entry.unshift( + this.makeEntry({ + id: 'whole_section', + title: '[Весь раздел]', + link: this.navLink({href: `/${encodeURIComponent(query.from)}?genre=${encodeURIComponent(all.join(','))}`}), + }) + ); + } + } else { + //выбираем разделы + const {genreTree} = await this.getGenres(); + let id = 0; + for (const section of genreTree) { + entry.push( + this.makeEntry({ + id: ++id, + title: section.name, + link: this.navLink({href: `/genre?from=${encodeURIComponent(query.from)}§ion=${encodeURIComponent(section.name)}`}), + }) + ); + } + } + } + + result.entry = entry; + + return this.makeBody(result, req); + } +} + +module.exports = GenrePage; \ No newline at end of file diff --git a/server/core/opds/index.js b/server/core/opds/index.js index 95c1ede..136196f 100644 --- a/server/core/opds/index.js +++ b/server/core/opds/index.js @@ -1,5 +1,6 @@ const RootPage = require('./RootPage'); const AuthorPage = require('./AuthorPage'); +const GenrePage = require('./GenrePage'); const BookPage = require('./BookPage'); module.exports = function(app, config) { @@ -8,12 +9,14 @@ module.exports = function(app, config) { const root = new RootPage(config); const author = new AuthorPage(config); + const genre = new GenrePage(config); const book = new BookPage(config); const routes = [ ['', root], ['/root', root], ['/author', author], + ['/genre', genre], ['/book', book], ]; From 6dfa551b973575c5507d6d6376e20ecd8ee55253 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Wed, 23 Nov 2022 20:57:41 +0700 Subject: [PATCH 29/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 4 ++-- server/core/opds/AuthorPage.js | 1 - server/core/opds/BasePage.js | 32 +++++++++++++++++++------------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 659d7b7..2e8c0cb 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -546,15 +546,15 @@ class DbSearcher { try { const db = this.db; + const depth = query.depth || 1; const queryKey = this.queryKey(query); - const opdsKey = `${from}-opds-${queryKey}`; + const opdsKey = `${from}-opds-d${depth}-${queryKey}`; let result = await this.getCached(opdsKey); if (result === null) { const ids = await this.selectTableIds(from, query); const totalFound = ids.length; - const depth = query.depth || 1; //группировка по name длиной depth const found = await db.select({ diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index 00c6986..2b30ab2 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -70,7 +70,6 @@ class AuthorPage extends BasePage { series: req.query.series || '', genre: req.query.genre || '', del: 0, - limit: 100, all: req.query.all || '', depth: 0, diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 6dcec84..3d18c6c 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -138,7 +138,7 @@ class BasePage { return result; } - async opdsQuery(from, query, otherTitle = '[Другие]') { + async opdsQuery(from, query, otherTitle = '[Другие]', prevLen = 0) { const queryRes = await this.webWorker.opdsQuery(from, query); let count = 0; for (const row of queryRes.found) @@ -146,21 +146,30 @@ class BasePage { const others = []; let result = []; - if (count <= query.limit) { - result = await this.search(from, query); + if (count <= 50) { + //конец навигации + return await this.search(from, query); } else { const names = new Set(); + let len = 0; for (const row of queryRes.found) { const name = row.name.toUpperCase(); + const lowName = row.name.toLowerCase(); + len += name.length; + + if (lowName == query[from]) { + //конец навигации, результат содержит запрос + return await this.search(from, query); + } if (!names.has(name)) { const rec = { id: row.id, title: name.replace(/ /g, spaceChar), - q: encodeURIComponent(row.name.toLowerCase()), + q: encodeURIComponent(lowName), count: row.count, }; - if (query.depth > 1 || enru.has(row.name[0].toLowerCase())) { + if (query.depth > 1 || enru.has(lowName[0])) { result.push(rec); } else { others.push(rec); @@ -168,15 +177,12 @@ class BasePage { names.add(name); } } - } - if (query.depth > 1 && result.length == 1 && query[from]) { - const newQuery = _.cloneDeep(query); - newQuery[from] = decodeURIComponent(result[0].q); - - if (newQuery[from].length >= query.depth) { - newQuery.depth = newQuery[from].length + 1; - return await this.opdsQuery(from, newQuery); + if (query[from] && query.depth > 1 && result.length < 20 && len > prevLen) { + //рекурсия, с увеличением глубины, для облегчения навигации + const newQuery = _.cloneDeep(query); + newQuery.depth++; + return await this.opdsQuery(from, newQuery, otherTitle, len); } } From 95da605cb96148df19828d5c9d7ca1edf348a82a Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 16:04:27 +0700 Subject: [PATCH 30/42] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 2e8c0cb..4b7b8e1 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -779,34 +779,21 @@ class DbSearcher { return; //выберем всех кандидатов на удаление - //находим delCount минимальных по time rows = await db.select({ table: 'query_time', rawResult: true, where: ` - const res = Array(${db.esc(delCount)}).fill({time: Date.now()}); + const delCount = ${delCount}; + const rows = []; @unsafeIter(@all(), (r) => { - if (r.time >= res[${db.esc(delCount - 1)}].time) - return false; - - let ins = {id: r.id, time: r.time}; - - for (let i = 0; i < res.length; i++) { - if (!res[i].id || ins.time < res[i].time) { - const t = res[i]; - res[i] = ins; - ins = t; - } - - if (!ins.id) - break; - } - + rows.push(r); return false; }); - return res.filter(r => r.id).map(r => r.id); + rows.sort((a, b) => a.time - b.time); + + return rows.slice(0, delCount).map(r => r.id); ` }); From 4371e1a64165091ae12b6c194d86dad4db9a576a Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 16:36:40 +0700 Subject: [PATCH 31/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/DbSearcher.js | 8 ++++---- server/core/opds/BasePage.js | 40 +++++++++++++++++------------------- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index 4b7b8e1..4199208 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -567,17 +567,17 @@ class DbSearcher { const ids = ${db.esc(Array.from(ids))}; for (const id of ids) { const row = @unsafeRow(id); - const s = row.name.substring(0, depth); + const s = row.value.substring(0, depth); let g = group.get(s); if (!g) { - g = {id: row.id, name: s, count: 0}; + g = {id: row.id, name: row.name, value: s, count: 0}; group.set(s, g); } g.count++; } const result = Array.from(group.values()); - result.sort((a, b) => a.name.localeCompare(b.name)); + result.sort((a, b) => a.value.localeCompare(b.value)); return result; ` @@ -778,7 +778,7 @@ class DbSearcher { if (delCount < 1) return; - //выберем всех кандидатов на удаление + //выберем delCount кандидатов на удаление rows = await db.select({ table: 'query_time', rawResult: true, diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 3d18c6c..300b066 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -150,35 +150,33 @@ class BasePage { //конец навигации return await this.search(from, query); } else { - const names = new Set(); let len = 0; for (const row of queryRes.found) { - const name = row.name.toUpperCase(); - const lowName = row.name.toLowerCase(); - len += name.length; + const value = row.value; + len += value.length; - if (lowName == query[from]) { - //конец навигации, результат содержит запрос - return await this.search(from, query); - } - - if (!names.has(name)) { - const rec = { + let rec; + if (row.count == 1) { + rec = { id: row.id, - title: name.replace(/ /g, spaceChar), - q: encodeURIComponent(lowName), - count: row.count, + title: row.name, + q: `=${encodeURIComponent(row.name)}`, }; - if (query.depth > 1 || enru.has(lowName[0])) { - result.push(rec); - } else { - others.push(rec); - } - names.add(name); + } else { + rec = { + id: row.id, + title: `${value.toUpperCase().replace(/ /g, spaceChar)}~`, + q: encodeURIComponent(value), + }; + } + if (query.depth > 1 || enru.has(value[0])) { + result.push(rec); + } else { + others.push(rec); } } - if (query[from] && query.depth > 1 && result.length < 20 && len > prevLen) { + if (query[from] && query.depth > 1 && result.length < 10 && len > prevLen) { //рекурсия, с увеличением глубины, для облегчения навигации const newQuery = _.cloneDeep(query); newQuery.depth++; From e356b874943df44a8c1e0366f80eb3a7779faf84 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 16:54:56 +0700 Subject: [PATCH 32/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index 2b30ab2..e8c3309 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -89,10 +89,11 @@ class AuthorPage extends BasePage { if (bookList.books) { let books = JSON.parse(bookList.books); - const filtered = (query.all ? books : this.filterBooks(books, query)); + const booksAll = this.filterBooks(books, {del: 0}); + const filtered = (query.all ? booksAll : this.filterBooks(books, query)); const sorted = this.sortSeriesBooks(filtered); - if (books.length > filtered.length) { + if (booksAll.length > filtered.length) { entry.push( this.makeEntry({ id: 'all_series_books', @@ -105,18 +106,23 @@ class AuthorPage extends BasePage { } for (const book of sorted) { - let title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'}`; + const title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'} (${book.ext})`; + + const e = { + id: book._uid, + title, + link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), + }; + if (query.all) { - title = `${this.bookAuthor(book.author)} "${title}"`; + e.content = { + '*ATTRS': {type: 'text'}, + '*TEXT': this.bookAuthor(book.author), + } } - title += ` (${book.ext})`; entry.push( - this.makeEntry({ - id: book._uid, - title, - link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), - }) + this.makeEntry(e) ); } } From fd9bc45fb12fd0da508f8b8a247e274fff7efe8d Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 17:20:11 +0700 Subject: [PATCH 33/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/AuthorPage.js | 11 +--- server/core/opds/BasePage.js | 9 +++ server/core/opds/RootPage.js | 3 + server/core/opds/SeriesPage.js | 114 +++++++++++++++++++++++++++++++++ server/core/opds/index.js | 3 + 5 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 server/core/opds/SeriesPage.js diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js index e8c3309..f8cb8f1 100644 --- a/server/core/opds/AuthorPage.js +++ b/server/core/opds/AuthorPage.js @@ -8,15 +8,6 @@ class AuthorPage extends BasePage { this.title = 'Авторы'; } - bookAuthor(author) { - if (author) { - let a = author.split(','); - return a.slice(0, 3).join(', ') + (a.length > 3 ? ' и др.' : ''); - } - - return ''; - } - sortBooks(bookList) { //схлопывание серий const books = []; @@ -175,7 +166,7 @@ class AuthorPage extends BasePage { entry.push( this.makeEntry({ id: rec.id, - title: this.bookAuthor(rec.title),//${(query.depth > 1 && rec.count ? ` (${rec.count})` : '')} + title: this.bookAuthor(rec.title), link: this.navLink({href: `/${this.id}?author=${rec.q}&genre=${encodeURIComponent(query.genre)}`}), }) ); diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 300b066..461abc5 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -298,6 +298,15 @@ class BasePage { }); } + bookAuthor(author) { + if (author) { + let a = author.split(','); + return a.slice(0, 3).join(', ') + (a.length > 3 ? ' и др.' : ''); + } + + return ''; + } + async getGenres() { let result; if (!this.genres) { diff --git a/server/core/opds/RootPage.js b/server/core/opds/RootPage.js index 3edb10a..481884e 100644 --- a/server/core/opds/RootPage.js +++ b/server/core/opds/RootPage.js @@ -1,5 +1,6 @@ const BasePage = require('./BasePage'); const AuthorPage = require('./AuthorPage'); +const SeriesPage = require('./SeriesPage'); class RootPage extends BasePage { constructor(config) { @@ -9,6 +10,7 @@ class RootPage extends BasePage { this.title = ''; this.authorPage = new AuthorPage(config); + this.seriesPage = new SeriesPage(config); } async body(req) { @@ -24,6 +26,7 @@ class RootPage extends BasePage { result.entry = [ this.authorPage.myEntry(), + this.seriesPage.myEntry(), ]; return this.makeBody(result, req); diff --git a/server/core/opds/SeriesPage.js b/server/core/opds/SeriesPage.js new file mode 100644 index 0000000..260592a --- /dev/null +++ b/server/core/opds/SeriesPage.js @@ -0,0 +1,114 @@ +const BasePage = require('./BasePage'); + +class SeriesPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'series'; + this.title = 'Серии'; + } + + sortSeriesBooks(seriesBooks) { + seriesBooks.sort((a, b) => { + const dserno = (a.serno || Number.MAX_VALUE) - (b.serno || Number.MAX_VALUE); + const dtitle = a.title.localeCompare(b.title); + const dext = a.ext.localeCompare(b.ext); + return (dserno ? dserno : (dtitle ? dtitle : dext)); + }); + + return seriesBooks; + } + + async body(req) { + const result = {}; + + const query = { + series: req.query.series || '', + genre: req.query.genre || '', + del: 0, + + all: req.query.all || '', + depth: 0, + }; + query.depth = query.series.length + 1; + + if (query.series == '___others') { + query.series = ''; + query.depth = 1; + query.others = true; + } + + const entry = []; + if (query.series && query.series[0] == '=') { + //книги по серии + const bookList = await this.webWorker.getSeriesBookList(query.series.substring(1)); + + if (bookList.books) { + let books = JSON.parse(bookList.books); + const booksAll = this.filterBooks(books, {del: 0}); + const filtered = (query.all ? booksAll : this.filterBooks(books, query)); + const sorted = this.sortSeriesBooks(filtered); + + if (booksAll.length > filtered.length) { + entry.push( + this.makeEntry({ + id: 'all_series_books', + title: '[Все книги серии]', + link: this.navLink({ + href: `/${this.id}?series=${encodeURIComponent(query.series)}&all=1`}), + }) + ); + } + + for (const book of sorted) { + const title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'} (${book.ext})`; + + const e = { + id: book._uid, + title, + link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), + }; + + if (query.all) { + e.content = { + '*ATTRS': {type: 'text'}, + '*TEXT': this.bookAuthor(book.author), + } + } + + entry.push( + this.makeEntry(e) + ); + } + } + } else { + if (query.depth == 1 && !query.genre && !query.others) { + entry.push( + this.makeEntry({ + id: 'select_genre', + title: '[Выбрать жанр]', + link: this.navLink({href: `/genre?from=${this.id}`}), + }) + ); + } + + //навигация по каталогу + const queryRes = await this.opdsQuery('series', query, '[Остальные серии]'); + + for (const rec of queryRes) { + entry.push( + this.makeEntry({ + id: rec.id, + title: rec.title, + link: this.navLink({href: `/${this.id}?series=${rec.q}&genre=${encodeURIComponent(query.genre)}`}), + }) + ); + } + } + + result.entry = entry; + return this.makeBody(result, req); + } +} + +module.exports = SeriesPage; \ No newline at end of file diff --git a/server/core/opds/index.js b/server/core/opds/index.js index 136196f..60501b2 100644 --- a/server/core/opds/index.js +++ b/server/core/opds/index.js @@ -1,5 +1,6 @@ const RootPage = require('./RootPage'); const AuthorPage = require('./AuthorPage'); +const SeriesPage = require('./SeriesPage'); const GenrePage = require('./GenrePage'); const BookPage = require('./BookPage'); @@ -9,6 +10,7 @@ module.exports = function(app, config) { const root = new RootPage(config); const author = new AuthorPage(config); + const series = new SeriesPage(config); const genre = new GenrePage(config); const book = new BookPage(config); @@ -16,6 +18,7 @@ module.exports = function(app, config) { ['', root], ['/root', root], ['/author', author], + ['/series', series], ['/genre', genre], ['/book', book], ]; From 8de33fbd9a30aa58c8151826ae678fe5c168470e Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 17:50:35 +0700 Subject: [PATCH 34/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/RootPage.js | 3 ++ server/core/opds/TitlePage.js | 89 +++++++++++++++++++++++++++++++++++ server/core/opds/index.js | 3 ++ 3 files changed, 95 insertions(+) create mode 100644 server/core/opds/TitlePage.js diff --git a/server/core/opds/RootPage.js b/server/core/opds/RootPage.js index 481884e..4ee4fb6 100644 --- a/server/core/opds/RootPage.js +++ b/server/core/opds/RootPage.js @@ -1,6 +1,7 @@ const BasePage = require('./BasePage'); const AuthorPage = require('./AuthorPage'); const SeriesPage = require('./SeriesPage'); +const TitlePage = require('./TitlePage'); class RootPage extends BasePage { constructor(config) { @@ -11,6 +12,7 @@ class RootPage extends BasePage { this.authorPage = new AuthorPage(config); this.seriesPage = new SeriesPage(config); + this.titlePage = new TitlePage(config); } async body(req) { @@ -27,6 +29,7 @@ class RootPage extends BasePage { result.entry = [ this.authorPage.myEntry(), this.seriesPage.myEntry(), + this.titlePage.myEntry(), ]; return this.makeBody(result, req); diff --git a/server/core/opds/TitlePage.js b/server/core/opds/TitlePage.js new file mode 100644 index 0000000..beb92f3 --- /dev/null +++ b/server/core/opds/TitlePage.js @@ -0,0 +1,89 @@ +const BasePage = require('./BasePage'); + +class TitlePage extends BasePage { + constructor(config) { + super(config); + + this.id = 'title'; + this.title = 'Книги'; + } + + async body(req) { + const result = {}; + + const query = { + title: req.query.title || '', + genre: req.query.genre || '', + del: 0, + + depth: 0, + }; + query.depth = query.title.length + 1; + + if (query.title == '___others') { + query.title = ''; + query.depth = 1; + query.others = true; + } + + const entry = []; + if (query.title && query.title[0] == '=') { + //книги по названию + const res = await this.webWorker.search('title', query); + + if (res.found.length) { + const books = res.found[0].books || []; + const filtered = this.filterBooks(books, query); + + for (const book of filtered) { + const title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'} (${book.ext})`; + + const e = { + id: book._uid, + title, + link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), + }; + + if (query.all) { + e.content = { + '*ATTRS': {type: 'text'}, + '*TEXT': this.bookAuthor(book.author), + } + } + + entry.push( + this.makeEntry(e) + ); + } + } + } else { + if (query.depth == 1 && !query.genre && !query.others) { + entry.push( + this.makeEntry({ + id: 'select_genre', + title: '[Выбрать жанр]', + link: this.navLink({href: `/genre?from=${this.id}`}), + }) + ); + } + + //навигация по каталогу + const queryRes = await this.opdsQuery('title', query, '[Остальные названия]'); + + for (const rec of queryRes) { + entry.push( + this.makeEntry({ + id: rec.id, + title: rec.title, + link: this.navLink({href: `/${this.id}?title=${rec.q}&genre=${encodeURIComponent(query.genre)}`}), + }) + ); + } + } + + result.entry = entry; + return this.makeBody(result, req); + } +} + +module.exports = TitlePage; \ No newline at end of file diff --git a/server/core/opds/index.js b/server/core/opds/index.js index 60501b2..f9f28f4 100644 --- a/server/core/opds/index.js +++ b/server/core/opds/index.js @@ -1,6 +1,7 @@ const RootPage = require('./RootPage'); const AuthorPage = require('./AuthorPage'); const SeriesPage = require('./SeriesPage'); +const TitlePage = require('./TitlePage'); const GenrePage = require('./GenrePage'); const BookPage = require('./BookPage'); @@ -11,6 +12,7 @@ module.exports = function(app, config) { const root = new RootPage(config); const author = new AuthorPage(config); const series = new SeriesPage(config); + const title = new TitlePage(config); const genre = new GenrePage(config); const book = new BookPage(config); @@ -19,6 +21,7 @@ module.exports = function(app, config) { ['/root', root], ['/author', author], ['/series', series], + ['/title', title], ['/genre', genre], ['/book', book], ]; From 72ab94291cc45a4a802cf5fea970c1088f03eb09 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 18:29:14 +0700 Subject: [PATCH 35/42] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/TitlePage.js | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/server/core/opds/TitlePage.js b/server/core/opds/TitlePage.js index beb92f3..6509cfe 100644 --- a/server/core/opds/TitlePage.js +++ b/server/core/opds/TitlePage.js @@ -38,21 +38,16 @@ class TitlePage extends BasePage { for (const book of filtered) { const title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'} (${book.ext})`; - const e = { - id: book._uid, - title, - link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), - }; - - if (query.all) { - e.content = { - '*ATTRS': {type: 'text'}, - '*TEXT': this.bookAuthor(book.author), - } - } - entry.push( - this.makeEntry(e) + this.makeEntry({ + id: book._uid, + title, + link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), + content: { + '*ATTRS': {type: 'text'}, + '*TEXT': this.bookAuthor(book.author), + }, + }) ); } } From 870f95a51f155fc07848e88a2c3d6f72e82adeb7 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 19:08:34 +0700 Subject: [PATCH 36/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/BasePage.js | 16 +++++++++-- server/core/opds/BookPage.js | 5 +--- server/core/opds/OpensearchPage.js | 45 ++++++++++++++++++++++++++++++ server/core/opds/index.js | 6 ++++ 4 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 server/core/opds/OpensearchPage.js diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js index 461abc5..e2850a1 100644 --- a/server/core/opds/BasePage.js +++ b/server/core/opds/BasePage.js @@ -89,11 +89,21 @@ class BasePage { }); } - baseLinks(req) { - return [ + baseLinks(req, selfAcq = false) { + const result = [ + this.makeLink({href: `${this.opdsRoot}/opensearch`, rel: 'search', type: 'application/opensearchdescription+xml'}), + this.makeLink({href: `${this.opdsRoot}/search?term={searchTerms}`, rel: 'search', type: 'application/atom+xml'}), + this.navLink({rel: 'start'}), - this.navLink({rel: 'self', href: req.originalUrl, hrefAsIs: true}), ]; + + if (selfAcq) { + result.push(this.acqLink({rel: 'self', href: req.originalUrl, hrefAsIs: true})); + } else { + result.push(this.navLink({rel: 'self', href: req.originalUrl, hrefAsIs: true})); + } + + return result; } makeBody(content, req) { diff --git a/server/core/opds/BookPage.js b/server/core/opds/BookPage.js index c765d1d..b3ad1e8 100644 --- a/server/core/opds/BookPage.js +++ b/server/core/opds/BookPage.js @@ -119,10 +119,7 @@ class BookPage extends BasePage { async body(req) { const result = {}; - result.link = [ - this.navLink({rel: 'start'}), - this.acqLink({rel: 'self', href: req.originalUrl, hrefAsIs: true}), - ]; + result.link = this.baseLinks(req, true); const bookUid = req.query.uid; const entry = []; diff --git a/server/core/opds/OpensearchPage.js b/server/core/opds/OpensearchPage.js new file mode 100644 index 0000000..128868b --- /dev/null +++ b/server/core/opds/OpensearchPage.js @@ -0,0 +1,45 @@ +const BasePage = require('./BasePage'); +const XmlParser = require('../xml/XmlParser'); + +class OpensearchPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'opensearch'; + this.title = 'opensearch'; + } + + async body() { + const xml = new XmlParser(); + const xmlObject = {}; +/* + + + inpx-web + Поиск по каталогу + UTF-8 + UTF-8 + + +*/ + xmlObject['OpenSearchDescription'] = { + '*ATTRS': {xmlns: 'http://a9.com/-/spec/opensearch/1.1/'}, + ShortName: 'inpx-web', + Description: 'Поиск по каталогу', + InputEncoding: 'UTF-8', + OutputEncoding: 'UTF-8', + Url: { + '*ATTRS': { + type: 'application/atom+xml;profile=opds-catalog;kind=navigation', + template: `${this.opdsRoot}/search?term={searchTerms}`, + }, + }, + } + + xml.fromObject(xmlObject); + + return xml.toString({format: true}); + } +} + +module.exports = OpensearchPage; \ No newline at end of file diff --git a/server/core/opds/index.js b/server/core/opds/index.js index f9f28f4..4816ae6 100644 --- a/server/core/opds/index.js +++ b/server/core/opds/index.js @@ -5,6 +5,8 @@ const TitlePage = require('./TitlePage'); const GenrePage = require('./GenrePage'); const BookPage = require('./BookPage'); +const OpensearchPage = require('./OpensearchPage'); + module.exports = function(app, config) { const opdsRoot = '/opds'; config.opdsRoot = opdsRoot; @@ -16,6 +18,8 @@ module.exports = function(app, config) { const genre = new GenrePage(config); const book = new BookPage(config); + const opensearch = new OpensearchPage(config); + const routes = [ ['', root], ['/root', root], @@ -24,6 +28,8 @@ module.exports = function(app, config) { ['/title', title], ['/genre', genre], ['/book', book], + + ['/opensearch', opensearch], ]; const pages = new Map(); From 1dc169d14b9a397b8de526be779c6baea9e4d569 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 19:52:51 +0700 Subject: [PATCH 37/42] =?UTF-8?q?=D0=A0=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20?= =?UTF-8?q?=D0=BD=D0=B0=D0=B4=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/opds/SearchPage.js | 83 ++++++++++++++++++++++++++++++++++ server/core/opds/index.js | 3 ++ 2 files changed, 86 insertions(+) create mode 100644 server/core/opds/SearchPage.js diff --git a/server/core/opds/SearchPage.js b/server/core/opds/SearchPage.js new file mode 100644 index 0000000..239069d --- /dev/null +++ b/server/core/opds/SearchPage.js @@ -0,0 +1,83 @@ +const BasePage = require('./BasePage'); + +class SearchPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'search'; + this.title = 'Поиск'; + } + + async body(req) { + const result = {}; + + const query = { + type: req.query.type || '', + term: req.query.term || '', + page: parseInt(req.query.page, 10) || 1, + }; + + let entry = []; + if (query.type) { + if (['author', 'series', 'title'].includes(query.type)) { + const from = query.type; + const page = query.page; + + const limit = 100; + const offset = (page - 1)*limit; + const queryRes = await this.webWorker.search(from, {[from]: query.term, del: 0, offset, limit}); + + const found = queryRes.found; + + for (let i = 0; i < found.length; i++) { + if (i >= limit) + break; + + const row = found[i]; + + entry.push( + this.makeEntry({ + id: row.id, + title: row[from], + link: this.navLink({href: `/${from}?${from}==${encodeURIComponent(row[from])}`}), + }), + ); + } + + if (queryRes.totalFound > offset + found.length) { + entry.push( + this.makeEntry({ + id: 'next_page', + title: '[Следующая страница]', + link: this.navLink({href: `/${this.id}?type=${from}&term=${encodeURIComponent(query.term)}&page=${page + 1}`}), + }), + ); + } + } + } else { + //корневой раздел + entry = [ + this.makeEntry({ + id: 'search_author', + title: 'Поиск авторов', + link: this.navLink({href: `/${this.id}?type=author&term=${encodeURIComponent(query.term)}`}), + }), + this.makeEntry({ + id: 'search_series', + title: 'Поиск серий', + link: this.navLink({href: `/${this.id}?type=series&term=${encodeURIComponent(query.term)}`}), + }), + this.makeEntry({ + id: 'search_title', + title: 'Поиск книг', + link: this.navLink({href: `/${this.id}?type=title&term=${encodeURIComponent(query.term)}`}), + }), + ] + } + + result.entry = entry; + return this.makeBody(result, req); + } +} + +module.exports = SearchPage; \ No newline at end of file diff --git a/server/core/opds/index.js b/server/core/opds/index.js index 4816ae6..b5c85e1 100644 --- a/server/core/opds/index.js +++ b/server/core/opds/index.js @@ -6,6 +6,7 @@ const GenrePage = require('./GenrePage'); const BookPage = require('./BookPage'); const OpensearchPage = require('./OpensearchPage'); +const SearchPage = require('./SearchPage'); module.exports = function(app, config) { const opdsRoot = '/opds'; @@ -19,6 +20,7 @@ module.exports = function(app, config) { const book = new BookPage(config); const opensearch = new OpensearchPage(config); + const search = new SearchPage(config); const routes = [ ['', root], @@ -30,6 +32,7 @@ module.exports = function(app, config) { ['/book', book], ['/opensearch', opensearch], + ['/search', search], ]; const pages = new Map(); From fd29532cf1edaf4810a46b385a125097bb288202 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 20:14:07 +0700 Subject: [PATCH 38/42] + "express-basic-auth": "^1.2.1" --- package-lock.json | 48 +++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 49 insertions(+) diff --git a/package-lock.json b/package-lock.json index 3387606..dc3af4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "chardet": "^1.5.0", "dayjs": "^1.11.6", "express": "^4.18.1", + "express-basic-auth": "^1.2.1", "fs-extra": "^10.1.0", "he": "^1.2.0", "iconv-lite": "^0.6.3", @@ -2576,6 +2577,22 @@ } ] }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/basic-auth/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -4211,6 +4228,14 @@ "node": ">= 0.10.0" } }, + "node_modules/express-basic-auth": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/express-basic-auth/-/express-basic-auth-1.2.1.tgz", + "integrity": "sha512-L6YQ1wQ/mNjVLAmK3AG1RK6VkokA1BIY6wmiH304Xtt/cLTps40EusZsU1Uop+v9lTDPxdtzbFmdXfFO3KEnwA==", + "dependencies": { + "basic-auth": "^2.0.1" + } + }, "node_modules/express/node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -10708,6 +10733,21 @@ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", "dev": true }, + "basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "requires": { + "safe-buffer": "5.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -11932,6 +11972,14 @@ } } }, + "express-basic-auth": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/express-basic-auth/-/express-basic-auth-1.2.1.tgz", + "integrity": "sha512-L6YQ1wQ/mNjVLAmK3AG1RK6VkokA1BIY6wmiH304Xtt/cLTps40EusZsU1Uop+v9lTDPxdtzbFmdXfFO3KEnwA==", + "requires": { + "basic-auth": "^2.0.1" + } + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", diff --git a/package.json b/package.json index f9da849..6f01001 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "chardet": "^1.5.0", "dayjs": "^1.11.6", "express": "^4.18.1", + "express-basic-auth": "^1.2.1", "fs-extra": "^10.1.0", "he": "^1.2.0", "iconv-lite": "^0.6.3", From 74d8cd3f9485cf63990289661fec154f6d69f63a Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 20:37:35 +0700 Subject: [PATCH 39/42] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=20basic-auth=20=D0=B4=D0=BB=D1=8F=20opds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/config/base.js | 6 ++++++ server/config/index.js | 1 + server/core/opds/index.js | 17 ++++++++++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/server/config/base.js b/server/config/base.js index 40a0801..bd1bd53 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -45,5 +45,11 @@ module.exports = { host: '0.0.0.0', port: '22380', }, + //opds: false, + opds: { + enabled: true, + user: '', + password: '', + }, }; diff --git a/server/config/index.js b/server/config/index.js index 3552fce..049bcc0 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -20,6 +20,7 @@ const propsToSave = [ 'allowRemoteLib', 'remoteLib', 'server', + 'opds', ]; let instance = null; diff --git a/server/core/opds/index.js b/server/core/opds/index.js index b5c85e1..29aaab7 100644 --- a/server/core/opds/index.js +++ b/server/core/opds/index.js @@ -1,3 +1,5 @@ +const basicAuth = require('express-basic-auth'); + const RootPage = require('./RootPage'); const AuthorPage = require('./AuthorPage'); const SeriesPage = require('./SeriesPage'); @@ -9,6 +11,9 @@ const OpensearchPage = require('./OpensearchPage'); const SearchPage = require('./SearchPage'); module.exports = function(app, config) { + if (!config.opds || !config.opds.enabled) + return; + const opdsRoot = '/opds'; config.opdsRoot = opdsRoot; @@ -62,6 +67,16 @@ module.exports = function(app, config) { } }; - app.get([opdsRoot, `${opdsRoot}/*`], opds); + const opdsPaths = [opdsRoot, `${opdsRoot}/*`]; + if (config.opds.password) { + if (!config.opds.user) + throw new Error('User must not be empty if password set'); + + app.use(opdsPaths, basicAuth({ + users: {[config.opds.user]: config.opds.password}, + challenge: true, + })); + } + app.get(opdsPaths, opds); }; From 7fa203eaae24479f96fe1aed08dc508670643e43 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 20:50:04 +0700 Subject: [PATCH 40/42] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=20readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 56cb118..130f0e4 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,10 @@ inpx-web или [LightLib](https://lightlib.azurewebsites.net) [Установка](#usage): просто поместить приложение `inpx-web` в папку с .inpx-файлом и файлами библиотеки (zip-архивами) и запустить. -По умолчанию, сервер будет доступен по адресу [http://127.0.0.1:12380](http://127.0.0.1:12380) + +По умолчанию, веб-сервер будет доступен по адресу [http://127.0.0.1:12380](http://127.0.0.1:12380) + +OPDS-сервер доступен по адресу [http://127.0.0.1:12380/opds](http://127.0.0.1:12380/opds) Для указания местоположения .inpx-файла или папки с файлами библиотеки, воспользуйтесь [параметрами командной строки](#cli). Дополнительные параметры сервера настраиваются в [конфигурационном файле](#config). @@ -30,6 +33,7 @@ inpx-web ## Возможности программы +- веб-интерфейс и OPDS-сервер - поиск по автору, серии, названию и пр. - скачивание книги, копирование ссылки или открытие в читалке - возможность указать рабочий каталог при запуске, а также расположение .inpx и файлов библиотеки @@ -47,7 +51,9 @@ inpx-web Там же, при первом запуске, будет создана рабочая директория `.inpx-web`, в которой хранится конфигурационный файл `config.json`, файлы базы данных, журналы и прочее. -По умолчанию сервер будет доступен по адресу [http://127.0.0.1:12380](http://127.0.0.1:12380) +По умолчанию веб-интерфейс будет доступен по адресу [http://127.0.0.1:12380](http://127.0.0.1:12380) + +OPDS-сервер доступен по адресу [http://127.0.0.1:12380/opds](http://127.0.0.1:12380/opds) @@ -131,6 +137,14 @@ Options: "server": { "host": "0.0.0.0", "port": "12380" + }, + + // настройки opds-сервера + // user, password используются для Basic HTTP authentication + "opds": { + "enabled": true, + "user": "", + "password": "" } } ``` @@ -266,17 +280,12 @@ cd inpx-web npm i ``` -#### Для платформы Windows +#### Релизы ```sh -npm run build:win +npm run release ``` -#### Для платформы Linux -```sh -npm run build:linux -``` - -Результат сборки будет доступен в каталоге `dist/linux|win` в виде исполнимого (standalone) файла. +Результат сборки будет доступен в каталоге `dist/release` From ad1a6560fa80b8265d5807cf4dba7e51fa6056a9 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 20:58:33 +0700 Subject: [PATCH 41/42] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D1=8B=20=D1=86=D0=B5=D0=BB=D0=B8=20=D0=B4=D0=BB?= =?UTF-8?q?=D1=8F=20=D1=81=D0=B1=D0=BE=D1=80=D0=BA=D0=B8=20=D1=80=D0=B5?= =?UTF-8?q?=D0=BB=D0=B8=D0=B7=D0=B0=20macos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build/prepkg.js | 2 +- build/release.js | 1 + package.json | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build/prepkg.js b/build/prepkg.js index 60f9171..ce9f94a 100644 --- a/build/prepkg.js +++ b/build/prepkg.js @@ -12,7 +12,7 @@ const publicDir = `${tmpDir}/public`; const outDir = `${distDir}/${platform}`; async function build() { - if (platform != 'linux' && platform != 'win') + if (platform != 'linux' && platform != 'win' && platform != 'macos') throw new Error(`Unknown platform: ${platform}`); await fs.emptyDir(outDir); diff --git a/build/release.js b/build/release.js index 9610335..3799773 100644 --- a/build/release.js +++ b/build/release.js @@ -22,6 +22,7 @@ async function main() { await fs.emptyDir(outDir); await makeRelease('win'); await makeRelease('linux'); + await makeRelease('macos'); } catch(e) { console.error(e); process.exit(1); diff --git a/package.json b/package.json index 6f01001..48e40ff 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,9 @@ "build:client": "webpack --config build/webpack.prod.config.js", "build:linux": "npm run build:client && node build/prepkg.js linux && pkg -t node16-linux-x64 -C GZip --options max-old-space-size=4096,expose-gc -o dist/linux/inpx-web .", "build:win": "npm run build:client && node build/prepkg.js win && pkg -t node16-win-x64 -C GZip --options max-old-space-size=4096,expose-gc -o dist/win/inpx-web .", + "build:macos": "npm run build:client && node build/prepkg.js macos && pkg -t node16-macos-x64 -C GZip --options max-old-space-size=4096,expose-gc -o dist/macos/inpx-web .", "build:client-dev": "webpack --config build/webpack.dev.config.js", - "build:all": "npm run build:linux && npm run build:win", + "build:all": "npm run build:linux && npm run build:win && npm run build:macos", "release": "npm run build:all && node build/release.js", "postinstall": "npm run build:client-dev" }, From 15778eb3e47a76d38e6d216c092650767113d55e Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Thu, 24 Nov 2022 20:59:43 +0700 Subject: [PATCH 42/42] =?UTF-8?q?=D0=92=D0=B5=D1=80=D1=81=D0=B8=D1=8F=201.?= =?UTF-8?q?3.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index dc3af4f..6bc3dd2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "inpx-web", - "version": "1.2.4", + "version": "1.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "inpx-web", - "version": "1.2.4", + "version": "1.3.0", "hasInstallScript": true, "license": "CC0-1.0", "dependencies": { diff --git a/package.json b/package.json index 48e40ff..106257c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "inpx-web", - "version": "1.2.4", + "version": "1.3.0", "author": "Book Pauk ", "license": "CC0-1.0", "repository": "bookpauk/inpx-web",