diff --git a/README.md b/README.md index 8737970..130f0e4 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,24 @@ inpx-web ======== Веб-сервер для поиска по .inpx-коллекции. -Выглядит это так: https://lib.omnireader.ru + +Выглядит следующим образом: [https://lib.omnireader.ru](https://lib.omnireader.ru) .inpx - индексный файл для импорта\экспорта информации из базы данных сетевых библиотек в базу каталогизатора [MyHomeLib](https://alex80.github.io/mhl/) или [freeLib](http://sourceforge.net/projects/freelibdesign) или [LightLib](https://lightlib.azurewebsites.net) -Просто поместите приложение `inpx-web` в папку с .inpx-файлом и файлами библиотеки (zip-архивами) и запустите. -Сервер будет доступен по адресу http://127.0.0.1:12380 +[Установка](#usage): просто поместить приложение `inpx-web` в папку с .inpx-файлом и файлами библиотеки (zip-архивами) и запустить. -После открытия веб-приложения в бразуере, для быстрого понимания того, как работает поиск, воспользуйтесь памяткой (кнопка со знаком вопроса). +По умолчанию, веб-сервер будет доступен по адресу [http://127.0.0.1:12380](http://127.0.0.1:12380) -## +OPDS-сервер доступен по адресу [http://127.0.0.1:12380/opds](http://127.0.0.1:12380/opds) + +Для указания местоположения .inpx-файла или папки с файлами библиотеки, воспользуйтесь [параметрами командной строки](#cli). +Дополнительные параметры сервера настраиваются в [конфигурационном файле](#config). + +## * [Возможности программы](#capabilities) * [Использование](#usage) * [Параметры командной строки](#cli) @@ -28,6 +33,7 @@ inpx-web ## Возможности программы +- веб-интерфейс и OPDS-сервер - поиск по автору, серии, названию и пр. - скачивание книги, копирование ссылки или открытие в читалке - возможность указать рабочий каталог при запуске, а также расположение .inpx и файлов библиотеки @@ -45,7 +51,9 @@ inpx-web Там же, при первом запуске, будет создана рабочая директория `.inpx-web`, в которой хранится конфигурационный файл `config.json`, файлы базы данных, журналы и прочее. -По умолчанию сервер будет доступен по адресу http://127.0.0.1:12380 +По умолчанию веб-интерфейс будет доступен по адресу [http://127.0.0.1:12380](http://127.0.0.1:12380) + +OPDS-сервер доступен по адресу [http://127.0.0.1:12380/opds](http://127.0.0.1:12380/opds) @@ -89,9 +97,17 @@ Options: // чистка каждый час "maxFilesDirSize": 1073741824, - // включить(true)/выключить(false) кеширование запросов на сервере + // включить(true)/выключить(false) серверное кеширование запросов на диске и в памяти "queryCacheEnabled": true, + // размер кеша запросов в оперативной памяти (количество) + // 0 - отключить кеширование запросов в оперативной памяти + "queryCacheMemSize": 50, + + // размер кеша запросов на диске (количество) + // 0 - отключить кеширование запросов на диске + "queryCacheDiskSize": 500, + // периодичность чистки кеша запросов на сервере, в минутах // 0 - отключить чистку "cacheCleanInterval": 60, @@ -121,6 +137,14 @@ Options: "server": { "host": "0.0.0.0", "port": "12380" + }, + + // настройки opds-сервера + // user, password используются для Basic HTTP authentication + "opds": { + "enabled": true, + "user": "", + "password": "" } } ``` @@ -161,7 +185,7 @@ Options: ### Фильтр по авторам и книгам -При создании поисковой БД во время загрузки и парсинга .inpx-файла, имеется возможность +При создании поисковой БД, во время загрузки и парсинга .inpx-файла, имеется возможность отфильтровать авторов и книги, задав определенные критерии. Для этого небходимо создать в рабочей директории (там же, где `config.json`) файл `filter.json` следующего вида: ```json @@ -176,7 +200,7 @@ Options: "excludeAuthors": ["Имя автора"] } ``` -При создании поисковой БД, авторы и книги из `includeAuthors` будут добавлены, а из `excludeAuthors` исключены. +При фильтрации, авторы и их книги из `includeAuthors` будут оставлены, а из `excludeAuthors` исключены. Использование совместно `includeAuthors` и `excludeAuthors` имеет мало смысла, поэтому для включения определенных авторов можно использовать только `includeAuthors`: ```json @@ -256,17 +280,12 @@ cd inpx-web npm i ``` -#### Для платформы Windows +#### Релизы ```sh -npm run build:win +npm run release ``` -#### Для платформы Linux -```sh -npm run build:linux -``` - -Результат сборки будет доступен в каталоге `dist/linux|win` в виде исполнимого (standalone) файла. +Результат сборки будет доступен в каталоге `dist/release` diff --git a/build/prepkg.js b/build/prepkg.js index 5e4ba08..ce9f94a 100644 --- a/build/prepkg.js +++ b/build/prepkg.js @@ -2,6 +2,8 @@ const fs = require('fs-extra'); const path = require('path'); const { execSync } = require('child_process'); +const showdown = require('showdown'); + const platform = process.argv[2]; const distDir = path.resolve(__dirname, '../dist'); @@ -10,11 +12,17 @@ const publicDir = `${tmpDir}/public`; const outDir = `${distDir}/${platform}`; async function build() { - if (platform != 'linux' && platform != 'win') + if (platform != 'linux' && platform != 'win' && platform != 'macos') throw new Error(`Unknown platform: ${platform}`); await fs.emptyDir(outDir); + //добавляем readme в релиз + let readme = await fs.readFile(path.resolve(__dirname, '../README.md'), 'utf-8'); + const converter = new showdown.Converter(); + readme = converter.makeHtml(readme); + await fs.writeFile(`${outDir}/readme.html`, readme); + // перемещаем public на место if (await fs.pathExists(publicDir)) { diff --git a/build/release.js b/build/release.js index 9610335..3799773 100644 --- a/build/release.js +++ b/build/release.js @@ -22,6 +22,7 @@ async function main() { await fs.emptyDir(outDir); await makeRelease('win'); await makeRelease('linux'); + await makeRelease('macos'); } catch(e) { console.error(e); process.exit(1); diff --git a/client/components/Search/BookInfoDialog/BookInfoDialog.vue b/client/components/Search/BookInfoDialog/BookInfoDialog.vue index ee8b9bc..f20e3f6 100644 --- a/client/components/Search/BookInfoDialog/BookInfoDialog.vue +++ b/client/components/Search/BookInfoDialog/BookInfoDialog.vue @@ -174,7 +174,6 @@ class BookInfoDialog { {name: 'fileInfo', label: 'Информация о файле', value: [ {name: 'folder', label: 'Папка'}, {name: 'file', label: 'Файл'}, - {name: 'ext', label: 'Тип'}, {name: 'size', label: 'Размер'}, {name: 'date', label: 'Добавлен'}, {name: 'del', label: 'Удален'}, @@ -193,7 +192,10 @@ class BookInfoDialog { ]}, ]; - const valueToString = (value, nodePath) => {//eslint-disable-line no-unused-vars + const valueToString = (value, nodePath, b) => {//eslint-disable-line no-unused-vars + if (nodePath == 'fileInfo/file') + return `${value}.${b.ext}`; + if (nodePath == 'fileInfo/size') return `${this.formatSize(value)} (${value.toString().replace(/(\d)(?=(\d{3})+(?!\d))/g, '$1 ')} Bytes)`; @@ -230,7 +232,7 @@ class BookInfoDialog { const subItemOut = { name: subItem.name, label: subItem.label, - value: valueToString(book[subItem.name], `${item.name}/${subItem.name}`) + value: valueToString(book[subItem.name], `${item.name}/${subItem.name}`, book) }; if (subItemOut.value) itemOut.value.push(subItemOut); diff --git a/package-lock.json b/package-lock.json index e3f9747..6bc3dd2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "inpx-web", - "version": "1.2.4", + "version": "1.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "inpx-web", - "version": "1.2.4", + "version": "1.3.0", "hasInstallScript": true, "license": "CC0-1.0", "dependencies": { @@ -15,9 +15,11 @@ "chardet": "^1.5.0", "dayjs": "^1.11.6", "express": "^4.18.1", + "express-basic-auth": "^1.2.1", "fs-extra": "^10.1.0", + "he": "^1.2.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.0.2", + "jembadb": "^5.1.3", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", @@ -49,6 +51,7 @@ "html-webpack-plugin": "^5.5.0", "mini-css-extract-plugin": "^2.6.1", "pkg": "^5.8.0", + "showdown": "^2.1.0", "terser-webpack-plugin": "^5.3.3", "vue-eslint-parser": "^9.0.3", "vue-loader": "^17.0.0", @@ -2574,6 +2577,22 @@ } ] }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/basic-auth/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -4209,6 +4228,14 @@ "node": ">= 0.10.0" } }, + "node_modules/express-basic-auth": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/express-basic-auth/-/express-basic-auth-1.2.1.tgz", + "integrity": "sha512-L6YQ1wQ/mNjVLAmK3AG1RK6VkokA1BIY6wmiH304Xtt/cLTps40EusZsU1Uop+v9lTDPxdtzbFmdXfFO3KEnwA==", + "dependencies": { + "basic-auth": "^2.0.1" + } + }, "node_modules/express/node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -4701,7 +4728,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true, "bin": { "he": "bin/he" } @@ -5046,9 +5072,9 @@ } }, "node_modules/jembadb": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.2.tgz", - "integrity": "sha512-0309Qo4wSkyf154xTokxNl0DuBP5f2Q2MzWGUNX1JmMzlRypFsPY/9VDYV/htkxhT53f2prlQ2NUguQjG2lCRA==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.3.tgz", + "integrity": "sha512-HGl9d3/fcNNahOqEsb3ocpXRWEfmDwV2zgWvKXERwlsxOHqoEId2fHXPkjv97qRywEyE/n9U8WimIWsP2Evf4w==", "engines": { "node": ">=16.16.0" } @@ -7477,6 +7503,31 @@ "node": ">=8" } }, + "node_modules/showdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz", + "integrity": "sha512-/6NVYu4U819R2pUIk79n67SYgJHWCce0a5xTP979WbNp0FL9MN1I1QK662IDU1b6JzKTvmhgI7T7JYIxBi3kMQ==", + "dev": true, + "dependencies": { + "commander": "^9.0.0" + }, + "bin": { + "showdown": "bin/showdown.js" + }, + "funding": { + "type": "individual", + "url": "https://www.paypal.me/tiviesantos" + } + }, + "node_modules/showdown/node_modules/commander": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.4.1.tgz", + "integrity": "sha512-5EEkTNyHNGFPD2H+c/dXXfQZYa/scCKasxWcXJaWnNJ99pnQN9Vnmqow+p+PlFPE63Q6mThaZws1T+HxfpgtPw==", + "dev": true, + "engines": { + "node": "^12.20.0 || >=14" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -10682,6 +10733,21 @@ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", "dev": true }, + "basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "requires": { + "safe-buffer": "5.1.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -11906,6 +11972,14 @@ } } }, + "express-basic-auth": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/express-basic-auth/-/express-basic-auth-1.2.1.tgz", + "integrity": "sha512-L6YQ1wQ/mNjVLAmK3AG1RK6VkokA1BIY6wmiH304Xtt/cLTps40EusZsU1Uop+v9lTDPxdtzbFmdXfFO3KEnwA==", + "requires": { + "basic-auth": "^2.0.1" + } + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -12283,8 +12357,7 @@ "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", - "dev": true + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" }, "html-entities": { "version": "2.3.3", @@ -12521,9 +12594,9 @@ "dev": true }, "jembadb": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.0.2.tgz", - "integrity": "sha512-0309Qo4wSkyf154xTokxNl0DuBP5f2Q2MzWGUNX1JmMzlRypFsPY/9VDYV/htkxhT53f2prlQ2NUguQjG2lCRA==" + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/jembadb/-/jembadb-5.1.3.tgz", + "integrity": "sha512-HGl9d3/fcNNahOqEsb3ocpXRWEfmDwV2zgWvKXERwlsxOHqoEId2fHXPkjv97qRywEyE/n9U8WimIWsP2Evf4w==" }, "jest-worker": { "version": "27.5.1", @@ -14257,6 +14330,23 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, + "showdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/showdown/-/showdown-2.1.0.tgz", + "integrity": "sha512-/6NVYu4U819R2pUIk79n67SYgJHWCce0a5xTP979WbNp0FL9MN1I1QK662IDU1b6JzKTvmhgI7T7JYIxBi3kMQ==", + "dev": true, + "requires": { + "commander": "^9.0.0" + }, + "dependencies": { + "commander": { + "version": "9.4.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.4.1.tgz", + "integrity": "sha512-5EEkTNyHNGFPD2H+c/dXXfQZYa/scCKasxWcXJaWnNJ99pnQN9Vnmqow+p+PlFPE63Q6mThaZws1T+HxfpgtPw==", + "dev": true + } + } + }, "side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", diff --git a/package.json b/package.json index 727a9bc..106257c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "inpx-web", - "version": "1.2.4", + "version": "1.3.0", "author": "Book Pauk ", "license": "CC0-1.0", "repository": "bookpauk/inpx-web", @@ -12,8 +12,9 @@ "build:client": "webpack --config build/webpack.prod.config.js", "build:linux": "npm run build:client && node build/prepkg.js linux && pkg -t node16-linux-x64 -C GZip --options max-old-space-size=4096,expose-gc -o dist/linux/inpx-web .", "build:win": "npm run build:client && node build/prepkg.js win && pkg -t node16-win-x64 -C GZip --options max-old-space-size=4096,expose-gc -o dist/win/inpx-web .", + "build:macos": "npm run build:client && node build/prepkg.js macos && pkg -t node16-macos-x64 -C GZip --options max-old-space-size=4096,expose-gc -o dist/macos/inpx-web .", "build:client-dev": "webpack --config build/webpack.dev.config.js", - "build:all": "npm run build:linux && npm run build:win", + "build:all": "npm run build:linux && npm run build:win && npm run build:macos", "release": "npm run build:all && node build/release.js", "postinstall": "npm run build:client-dev" }, @@ -38,6 +39,7 @@ "html-webpack-plugin": "^5.5.0", "mini-css-extract-plugin": "^2.6.1", "pkg": "^5.8.0", + "showdown": "^2.1.0", "terser-webpack-plugin": "^5.3.3", "vue-eslint-parser": "^9.0.3", "vue-loader": "^17.0.0", @@ -54,9 +56,11 @@ "chardet": "^1.5.0", "dayjs": "^1.11.6", "express": "^4.18.1", + "express-basic-auth": "^1.2.1", "fs-extra": "^10.1.0", + "he": "^1.2.0", "iconv-lite": "^0.6.3", - "jembadb": "^5.0.2", + "jembadb": "^5.1.3", "localforage": "^1.10.0", "lodash": "^4.17.21", "minimist": "^1.2.6", diff --git a/server/config/base.js b/server/config/base.js index 923a53a..bd1bd53 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -16,12 +16,14 @@ module.exports = { //поправить в случае, если были критические изменения в DbCreator или InpxParser //иначе будет рассинхронизация между сервером и клиентом на уровне БД - dbVersion: '7', + dbVersion: '8', dbCacheSize: 5, maxPayloadSize: 500,//in MB maxFilesDirSize: 1024*1024*1024,//1Gb queryCacheEnabled: true, + queryCacheMemSize: 50, + queryCacheDiskSize: 500, cacheCleanInterval: 60,//minutes inpxCheckInterval: 60,//minutes lowMemoryMode: false, @@ -43,5 +45,11 @@ module.exports = { host: '0.0.0.0', port: '22380', }, + //opds: false, + opds: { + enabled: true, + user: '', + password: '', + }, }; diff --git a/server/config/index.js b/server/config/index.js index c30381c..049bcc0 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -11,6 +11,8 @@ const propsToSave = [ 'dbCacheSize', 'maxFilesDirSize', 'queryCacheEnabled', + 'queryCacheMemSize', + 'queryCacheDiskSize', 'cacheCleanInterval', 'inpxCheckInterval', 'lowMemoryMode', @@ -18,6 +20,7 @@ const propsToSave = [ 'allowRemoteLib', 'remoteLib', 'server', + 'opds', ]; let instance = null; diff --git a/server/core/DbCreator.js b/server/core/DbCreator.js index 3cfc00e..d1aba16 100644 --- a/server/core/DbCreator.js +++ b/server/core/DbCreator.js @@ -459,7 +459,6 @@ class DbCreator { const config = this.config; const to = `${from}_book`; - const toId = `${from}_id`; await db.open({table: from}); await db.create({table: to}); @@ -548,7 +547,7 @@ class DbCreator { await saveChunk(chunk); processed += chunk.length; - callback({progress: 0.5*processed/fromLength}); + callback({progress: 0.9*processed/fromLength}); } else break; @@ -562,24 +561,18 @@ class DbCreator { await db.close({table: to}); await db.close({table: from}); - await db.create({table: toId}); - - const chunkSize = 50000; - let idRows = []; - let proc = 0; + const idMap = {arr: [], map: []}; for (const [id, value] of bookId2RecId) { - idRows.push({id, value}); - if (idRows.length >= chunkSize) { - await db.insert({table: toId, rows: idRows}); - idRows = []; - - proc += chunkSize; - callback({progress: 0.5 + 0.5*proc/bookId2RecId.size}); + if (value.length > 1) { + idMap.map.push([id, value]); + idMap.arr[id] = 0; + } else { + idMap.arr[id] = value[0]; } } - if (idRows.length) - await db.insert({table: toId, rows: idRows}); - await db.close({table: toId}); + + callback({progress: 1}); + await fs.writeFile(`${this.config.dataDir}/db/${from}_id.map`, JSON.stringify(idMap)); bookId2RecId = null; utils.freeMemory(); diff --git a/server/core/DbSearcher.js b/server/core/DbSearcher.js index b198f59..4199208 100644 --- a/server/core/DbSearcher.js +++ b/server/core/DbSearcher.js @@ -1,8 +1,8 @@ +const fs = require('fs-extra'); //const _ = require('lodash'); const LockQueue = require('./LockQueue'); const utils = require('./utils'); -const maxMemCacheSize = 100; const maxLimit = 1000; const emptyFieldValue = '?'; @@ -14,6 +14,11 @@ const enruArr = (ruAlphabet + enAlphabet).split(''); class DbSearcher { constructor(config, db) { this.config = config; + this.queryCacheMemSize = this.config.queryCacheMemSize; + this.queryCacheDiskSize = this.config.queryCacheDiskSize; + this.queryCacheEnabled = this.config.queryCacheEnabled + && (this.queryCacheMemSize > 0 || this.queryCacheDiskSize > 0); + this.db = db; this.lock = new LockQueue(); @@ -77,7 +82,7 @@ class DbSearcher { result.add(bookId); } - return Array.from(result); + return new Uint32Array(result); ` }); @@ -151,7 +156,7 @@ class DbSearcher { result.add(bookId); } - return Array.from(result); + return new Uint32Array(result); ` }); @@ -187,7 +192,7 @@ class DbSearcher { result.add(bookId); } - return Array.from(result); + return new Uint32Array(result); ` }); @@ -252,7 +257,7 @@ class DbSearcher { result.add(bookId); } - return Array.from(result); + return new Uint32Array(result); ` }); @@ -285,7 +290,7 @@ class DbSearcher { inter = newInter; } - return Array.from(inter); + return new Uint32Array(inter); } else if (idsArr.length == 1) { return idsArr[0]; } else { @@ -299,29 +304,13 @@ class DbSearcher { await this.lock.get(); try { - const db = this.db; - const map = new Map(); - const table = `${from}_id`; + const data = await fs.readFile(`${this.config.dataDir}/db/${from}_id.map`, 'utf-8'); - await db.open({table}); - let rows = await db.select({table}); - await db.close({table}); + const idMap = JSON.parse(data); + idMap.arr = new Uint32Array(idMap.arr); + idMap.map = new Map(idMap.map); - for (const row of rows) { - if (!row.value.length) - continue; - - if (row.value.length > 1) - map.set(row.id, row.value); - else - map.set(row.id, row.value[0]); - } - - this.bookIdMap[from] = map; - - rows = null; - await db.freeMemory(); - utils.freeMemory(); + this.bookIdMap[from] = idMap; return this.bookIdMap[from]; } finally { @@ -330,15 +319,20 @@ class DbSearcher { } async fillBookIdMapAll() { - await this.fillBookIdMap('author'); - await this.fillBookIdMap('series'); - await this.fillBookIdMap('title'); + try { + await this.fillBookIdMap('author'); + await this.fillBookIdMap('series'); + await this.fillBookIdMap('title'); + } catch (e) { + throw new Error(`DbSearcher.fillBookIdMapAll error: ${e.message}`) + } } - async filterTableIds(tableIds, from, query) { - let result = tableIds; - - //т.к. авторы у книги идут списком, то дополнительно фильтруем + async tableIdsFilter(from, query) { + //т.к. авторы у книги идут списком (т.е. одна книга относиться сразу к нескольким авторам), + //то в выборку по bookId могут попасть авторы, которые отсутствуют в критерии query.author, + //поэтому дополнительно фильтруем + let result = null; if (from == 'author' && query.author && query.author !== '*') { const key = `filter-ids-author-${query.author}`; let authorIds = await this.getCached(key); @@ -347,7 +341,7 @@ class DbSearcher { const rows = await this.db.select({ table: 'author', rawResult: true, - where: `return Array.from(${this.getWhere(query.author)})` + where: `return new Uint32Array(${this.getWhere(query.author)})` }); authorIds = rows[0].rawResult; @@ -355,12 +349,7 @@ class DbSearcher { await this.putCached(key, authorIds); } - //пересечение tableIds и authorIds - result = []; - const authorIdsSet = new Set(authorIds); - for (const id of tableIds) - if (authorIdsSet.has(id)) - result.push(id); + result = new Set(authorIds); } return result; @@ -381,24 +370,30 @@ class DbSearcher { await this.putCached(bookKey, bookIds); } + //id книг (bookIds) нашли, теперь надо их смаппировать в id таблицы from (авторов, серий, названий) if (bookIds) { + //т.к. авторы у книги идут списком, то дополнительно фильтруем + const filter = await this.tableIdsFilter(from, query); + const tableIdsSet = new Set(); - const bookIdMap = await this.fillBookIdMap(from); + const idMap = await this.fillBookIdMap(from); let proc = 0; let nextProc = 0; for (const bookId of bookIds) { - const tableIdValue = bookIdMap.get(bookId); - if (!tableIdValue) - continue; - - if (Array.isArray(tableIdValue)) { - for (const tableId of tableIdValue) { + const tableId = idMap.arr[bookId]; + if (tableId) { + if (!filter || filter.has(tableId)) tableIdsSet.add(tableId); - proc++; - } - } else { - tableIdsSet.add(tableIdValue); proc++; + } else { + const tableIdArr = idMap.map.get(bookId); + if (tableIdArr) { + for (const tableId of tableIdArr) { + if (!filter || filter.has(tableId)) + tableIdsSet.add(tableId); + proc++; + } + } } //прерываемся иногда, чтобы не блокировать Event Loop @@ -408,19 +403,19 @@ class DbSearcher { } } - tableIds = Array.from(tableIdsSet); - } else { + tableIds = new Uint32Array(tableIdsSet); + } else {//bookIds пустой - критерии не заданы, значит берем все id из from const rows = await db.select({ table: from, rawResult: true, - where: `return Array.from(@all())` + where: `return new Uint32Array(@all())` }); tableIds = rows[0].rawResult; } - tableIds = await this.filterTableIds(tableIds, from, query); - + //сортируем по id + //порядок id соответствует ASC-сортировке по строковому значению из from (имя автора, назание серии, название книги) tableIds.sort((a, b) => a - b); await this.putCached(tableKey, tableIds); @@ -509,11 +504,13 @@ class DbSearcher { limit = (limit > maxLimit ? maxLimit : limit); const offset = (query.offset ? query.offset : 0); + const slice = ids.slice(offset, offset + limit); + //выборка найденных значений const found = await db.select({ table: from, map: `(r) => ({id: r.id, ${from}: r.name, bookCount: r.bookCount, bookDelCount: r.bookDelCount})`, - where: `@@id(${db.esc(ids.slice(offset, offset + limit))})` + where: `@@id(${db.esc(Array.from(slice))})` }); //для title восстановим books @@ -537,28 +534,105 @@ class DbSearcher { } } - async getAuthorBookList(authorId) { + async opdsQuery(from, query) { if (this.closed) throw new Error('DbSearcher closed'); - if (!authorId) + if (!['author', 'series', 'title'].includes(from)) + throw new Error(`Unknown value for param 'from'`); + + this.searchFlag++; + + try { + const db = this.db; + + const depth = query.depth || 1; + const queryKey = this.queryKey(query); + const opdsKey = `${from}-opds-d${depth}-${queryKey}`; + let result = await this.getCached(opdsKey); + + if (result === null) { + const ids = await this.selectTableIds(from, query); + + const totalFound = ids.length; + + //группировка по name длиной depth + const found = await db.select({ + table: from, + rawResult: true, + where: ` + const depth = ${db.esc(depth)}; + const group = new Map(); + + const ids = ${db.esc(Array.from(ids))}; + for (const id of ids) { + const row = @unsafeRow(id); + const s = row.value.substring(0, depth); + let g = group.get(s); + if (!g) { + g = {id: row.id, name: row.name, value: s, count: 0}; + group.set(s, g); + } + g.count++; + } + + const result = Array.from(group.values()); + result.sort((a, b) => a.value.localeCompare(b.value)); + + return result; + ` + }); + + result = {found: found[0].rawResult, totalFound}; + + await this.putCached(opdsKey, result); + } + + return result; + } finally { + this.searchFlag--; + } + } + + async getAuthorBookList(authorId, author) { + if (this.closed) + throw new Error('DbSearcher closed'); + + if (!authorId && !author) return {author: '', books: ''}; this.searchFlag++; try { - //выборка книг автора по authorId - const rows = await this.restoreBooks('author', [authorId]) + const db = this.db; - let author = ''; + if (!authorId) { + //восстановим authorId + authorId = 0; + author = author.toLowerCase(); + + const rows = await db.select({ + table: 'author', + rawResult: true, + where: `return Array.from(@dirtyIndexLR('value', ${db.esc(author)}, ${db.esc(author)}))` + }); + + if (rows.length && rows[0].rawResult.length) + authorId = rows[0].rawResult[0]; + } + + //выборка книг автора по authorId + const rows = await this.restoreBooks('author', [authorId]); + + let authorName = ''; let books = ''; if (rows.length) { - author = rows[0].name; + authorName = rows[0].name; books = rows[0].books; } - return {author, books: (books && books.length ? JSON.stringify(books) : '')}; + return {author: authorName, books: (books && books.length ? JSON.stringify(books) : '')}; } finally { this.searchFlag--; } @@ -601,7 +675,7 @@ class DbSearcher { } async getCached(key) { - if (!this.config.queryCacheEnabled) + if (!this.queryCacheEnabled) return null; let result = null; @@ -609,13 +683,13 @@ class DbSearcher { const db = this.db; const memCache = this.memCache; - if (memCache.has(key)) {//есть в недавних + if (this.queryCacheMemSize > 0 && memCache.has(key)) {//есть в недавних result = memCache.get(key); //изменим порядок ключей, для последующей правильной чистки старых memCache.delete(key); memCache.set(key, result); - } else {//смотрим в таблице + } else if (this.queryCacheDiskSize > 0) {//смотрим в таблице const rows = await db.select({table: 'query_cache', where: `@@id(${db.esc(key)})`}); if (rows.length) {//нашли в кеше @@ -626,13 +700,17 @@ class DbSearcher { }); result = rows[0].value; - memCache.set(key, result); - if (memCache.size > maxMemCacheSize) { - //удаляем самый старый ключ-значение - for (const k of memCache.keys()) { - memCache.delete(k); - break; + //заполняем кеш в памяти + if (this.queryCacheMemSize > 0) { + memCache.set(key, result); + + if (memCache.size > this.queryCacheMemSize) { + //удаляем самый старый ключ-значение + for (const k of memCache.keys()) { + memCache.delete(k); + break; + } } } } @@ -642,40 +720,44 @@ class DbSearcher { } async putCached(key, value) { - if (!this.config.queryCacheEnabled) + if (!this.queryCacheEnabled) return; const db = this.db; - const memCache = this.memCache; - memCache.set(key, value); + if (this.queryCacheMemSize > 0) { + const memCache = this.memCache; + memCache.set(key, value); - if (memCache.size > maxMemCacheSize) { - //удаляем самый старый ключ-значение - for (const k of memCache.keys()) { - memCache.delete(k); - break; + if (memCache.size > this.queryCacheMemSize) { + //удаляем самый старый ключ-значение + for (const k of memCache.keys()) { + memCache.delete(k); + break; + } } } - //кладем в таблицу асинхронно - (async() => { - try { - await db.insert({ - table: 'query_cache', - replace: true, - rows: [{id: key, value}], - }); + if (this.queryCacheDiskSize > 0) { + //кладем в таблицу асинхронно + (async() => { + try { + await db.insert({ + table: 'query_cache', + replace: true, + rows: [{id: key, value}], + }); - await db.insert({ - table: 'query_time', - replace: true, - rows: [{id: key, time: Date.now()}], - }); - } catch(e) { - console.error(`putCached: ${e.message}`); - } - })(); + await db.insert({ + table: 'query_time', + replace: true, + rows: [{id: key, time: Date.now()}], + }); + } catch(e) { + console.error(`putCached: ${e.message}`); + } + })(); + } } async periodicCleanCache() { @@ -685,21 +767,37 @@ class DbSearcher { return; try { + if (!this.queryCacheEnabled || this.queryCacheDiskSize <= 0) + return; + const db = this.db; - const oldThres = Date.now() - cleanInterval; + let rows = await db.select({table: 'query_time', count: true}); + const delCount = rows[0].count - this.queryCacheDiskSize; - //выберем всех кандидатов на удаление - const rows = await db.select({ + if (delCount < 1) + return; + + //выберем delCount кандидатов на удаление + rows = await db.select({ table: 'query_time', + rawResult: true, where: ` - @@iter(@all(), (r) => (r.time < ${db.esc(oldThres)})); + const delCount = ${delCount}; + const rows = []; + + @unsafeIter(@all(), (r) => { + rows.push(r); + return false; + }); + + rows.sort((a, b) => a.time - b.time); + + return rows.slice(0, delCount).map(r => r.id); ` }); - const ids = []; - for (const row of rows) - ids.push(row.id); + const ids = rows[0].rawResult; //удаляем await db.delete({table: 'query_cache', where: `@@id(${db.esc(ids)})`}); diff --git a/server/core/WebWorker.js b/server/core/WebWorker.js index 547f55d..8c4bac7 100644 --- a/server/core/WebWorker.js +++ b/server/core/WebWorker.js @@ -267,10 +267,16 @@ class WebWorker { return result; } - async getAuthorBookList(authorId) { + async opdsQuery(from, query) { this.checkMyState(); - return await this.dbSearcher.getAuthorBookList(authorId); + return await this.dbSearcher.opdsQuery(from, query); + } + + async getAuthorBookList(authorId, author) { + this.checkMyState(); + + return await this.dbSearcher.getAuthorBookList(authorId, author); } async getSeriesBookList(series) { @@ -469,14 +475,14 @@ class WebWorker { const bookFile = `${this.config.filesDir}/${hash}`; const bookFileInfo = `${bookFile}.i.json`; + let rows = await db.select({table: 'book', where: `@@hash('_uid', ${db.esc(bookUid)})`}); + if (!rows.length) + throw new Error('404 Файл не найден'); + const book = rows[0]; + const restoreBookInfo = async(info) => { const result = {}; - let rows = await db.select({table: 'book', where: `@@hash('_uid', ${db.esc(bookUid)})`}); - if (!rows.length) - throw new Error('404 Файл не найден'); - const book = rows[0]; - result.book = book; result.cover = ''; result.fb2 = false; @@ -493,7 +499,8 @@ class WebWorker { } } - Object.assign(info ,result); + Object.assign(info, result); + await fs.writeFile(bookFileInfo, JSON.stringify(info)); if (this.config.branch === 'development') { @@ -513,7 +520,7 @@ class WebWorker { if (tmpInfo.cover) coverFile = `${this.config.publicFilesDir}${tmpInfo.cover}`; - if (coverFile && !await fs.pathExists(coverFile)) { + if (book.id != tmpInfo.book.id || (coverFile && !await fs.pathExists(coverFile))) { await restoreBookInfo(bookInfo); } else { bookInfo = tmpInfo; diff --git a/server/core/opds/AuthorPage.js b/server/core/opds/AuthorPage.js new file mode 100644 index 0000000..f8cb8f1 --- /dev/null +++ b/server/core/opds/AuthorPage.js @@ -0,0 +1,181 @@ +const BasePage = require('./BasePage'); + +class AuthorPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'author'; + this.title = 'Авторы'; + } + + sortBooks(bookList) { + //схлопывание серий + const books = []; + const seriesSet = new Set(); + for (const book of bookList) { + if (book.series) { + if (!seriesSet.has(book.series)) { + books.push({ + type: 'series', + book + }); + + seriesSet.add(book.series); + } + } else { + books.push({ + type: 'book', + book + }); + } + } + + //сортировка + books.sort((a, b) => { + if (a.type == 'series') { + return (b.type == 'series' ? a.book.series.localeCompare(b.book.series) : -1); + } else { + return (b.type == 'book' ? a.book.title.localeCompare(b.book.title) : 1); + } + }); + + return books; + } + + sortSeriesBooks(seriesBooks) { + seriesBooks.sort((a, b) => { + const dserno = (a.serno || Number.MAX_VALUE) - (b.serno || Number.MAX_VALUE); + const dtitle = a.title.localeCompare(b.title); + const dext = a.ext.localeCompare(b.ext); + return (dserno ? dserno : (dtitle ? dtitle : dext)); + }); + + return seriesBooks; + } + + async body(req) { + const result = {}; + + const query = { + author: req.query.author || '', + series: req.query.series || '', + genre: req.query.genre || '', + del: 0, + + all: req.query.all || '', + depth: 0, + }; + query.depth = query.author.length + 1; + + if (query.author == '___others') { + query.author = ''; + query.depth = 1; + query.others = true; + } + + const entry = []; + if (query.series) { + //книги по серии + const bookList = await this.webWorker.getSeriesBookList(query.series); + + if (bookList.books) { + let books = JSON.parse(bookList.books); + const booksAll = this.filterBooks(books, {del: 0}); + const filtered = (query.all ? booksAll : this.filterBooks(books, query)); + const sorted = this.sortSeriesBooks(filtered); + + if (booksAll.length > filtered.length) { + entry.push( + this.makeEntry({ + id: 'all_series_books', + title: '[Все книги серии]', + link: this.navLink({ + href: `/${this.id}?author=${encodeURIComponent(query.author)}` + + `&series=${encodeURIComponent(query.series)}&all=1`}), + }) + ); + } + + for (const book of sorted) { + const title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'} (${book.ext})`; + + const e = { + id: book._uid, + title, + link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), + }; + + if (query.all) { + e.content = { + '*ATTRS': {type: 'text'}, + '*TEXT': this.bookAuthor(book.author), + } + } + + entry.push( + this.makeEntry(e) + ); + } + } + } else if (query.author && query.author[0] == '=') { + //книги по автору + const bookList = await this.webWorker.getAuthorBookList(0, query.author.substring(1)); + + if (bookList.books) { + let books = JSON.parse(bookList.books); + books = this.sortBooks(this.filterBooks(books, query)); + + for (const b of books) { + if (b.type == 'series') { + entry.push( + this.makeEntry({ + id: b.book._uid, + title: `Серия: ${b.book.series}`, + link: this.navLink({ + href: `/${this.id}?author=${encodeURIComponent(query.author)}` + + `&series=${encodeURIComponent(b.book.series)}&genre=${encodeURIComponent(query.genre)}`}), + }) + ); + } else { + const title = `${b.book.title || 'Без названия'} (${b.book.ext})`; + entry.push( + this.makeEntry({ + id: b.book._uid, + title, + link: this.acqLink({href: `/book?uid=${encodeURIComponent(b.book._uid)}`}), + }) + ); + } + } + } + } else { + if (query.depth == 1 && !query.genre && !query.others) { + entry.push( + this.makeEntry({ + id: 'select_genre', + title: '[Выбрать жанр]', + link: this.navLink({href: `/genre?from=${this.id}`}), + }) + ); + } + + //навигация по каталогу + const queryRes = await this.opdsQuery('author', query, '[Остальные авторы]'); + + for (const rec of queryRes) { + entry.push( + this.makeEntry({ + id: rec.id, + title: this.bookAuthor(rec.title), + link: this.navLink({href: `/${this.id}?author=${rec.q}&genre=${encodeURIComponent(query.genre)}`}), + }) + ); + } + } + + result.entry = entry; + return this.makeBody(result, req); + } +} + +module.exports = AuthorPage; \ No newline at end of file diff --git a/server/core/opds/BasePage.js b/server/core/opds/BasePage.js new file mode 100644 index 0000000..e2850a1 --- /dev/null +++ b/server/core/opds/BasePage.js @@ -0,0 +1,347 @@ +const _ = require('lodash'); +const he = require('he'); + +const WebWorker = require('../WebWorker');//singleton +const XmlParser = require('../xml/XmlParser'); + +const spaceChar = String.fromCodePoint(0x00B7); +const emptyFieldValue = '?'; +const maxUtf8Char = String.fromCodePoint(0xFFFFF); +const ruAlphabet = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'; +const enAlphabet = 'abcdefghijklmnopqrstuvwxyz'; +const enruArr = (ruAlphabet + enAlphabet).split(''); +const enru = new Set(enruArr); + +class BasePage { + constructor(config) { + this.config = config; + + this.webWorker = new WebWorker(config); + this.rootTag = 'feed'; + this.opdsRoot = config.opdsRoot; + } + + makeEntry(entry = {}) { + if (!entry.id) + throw new Error('makeEntry: no id'); + if (!entry.title) + throw new Error('makeEntry: no title'); + + entry.title = he.escape(entry.title); + + const result = { + updated: (new Date()).toISOString().substring(0, 19) + 'Z', + }; + + return Object.assign(result, entry); + } + + myEntry() { + return this.makeEntry({ + id: this.id, + title: this.title, + link: this.navLink({href: `/${this.id}`}), + }); + } + + makeLink(attrs) { + attrs.href = he.escape(attrs.href); + return {'*ATTRS': attrs}; + } + + navLink(attrs) { + return this.makeLink({ + href: (attrs.hrefAsIs ? attrs.href : `${this.opdsRoot}${attrs.href || ''}`), + rel: attrs.rel || 'subsection', + type: 'application/atom+xml;profile=opds-catalog;kind=navigation', + }); + } + + acqLink(attrs) { + return this.makeLink({ + href: (attrs.hrefAsIs ? attrs.href : `${this.opdsRoot}${attrs.href || ''}`), + rel: attrs.rel || 'subsection', + type: 'application/atom+xml;profile=opds-catalog;kind=acquisition', + }); + } + + downLink(attrs) { + if (!attrs.href) + throw new Error('downLink: no href'); + if (!attrs.type) + throw new Error('downLink: no type'); + + return this.makeLink({ + href: attrs.href, + rel: 'http://opds-spec.org/acquisition', + type: attrs.type, + }); + } + + imgLink(attrs) { + if (!attrs.href) + throw new Error('imgLink: no href'); + + return this.makeLink({ + href: attrs.href, + rel: `http://opds-spec.org/image${attrs.thumb ? '/thumbnail' : ''}`, + type: attrs.type || 'image/jpeg', + }); + } + + baseLinks(req, selfAcq = false) { + const result = [ + this.makeLink({href: `${this.opdsRoot}/opensearch`, rel: 'search', type: 'application/opensearchdescription+xml'}), + this.makeLink({href: `${this.opdsRoot}/search?term={searchTerms}`, rel: 'search', type: 'application/atom+xml'}), + + this.navLink({rel: 'start'}), + ]; + + if (selfAcq) { + result.push(this.acqLink({rel: 'self', href: req.originalUrl, hrefAsIs: true})); + } else { + result.push(this.navLink({rel: 'self', href: req.originalUrl, hrefAsIs: true})); + } + + return result; + } + + makeBody(content, req) { + const base = this.makeEntry({id: this.id, title: this.title}); + base['*ATTRS'] = { + 'xmlns': 'http://www.w3.org/2005/Atom', + 'xmlns:dc': 'http://purl.org/dc/terms/', + 'xmlns:opds': 'http://opds-spec.org/2010/catalog', + }; + + if (!content.link) + base.link = this.baseLinks(req); + + const xml = new XmlParser(); + const xmlObject = {}; + xmlObject[this.rootTag] = Object.assign(base, content); + + xml.fromObject(xmlObject); + + return xml.toString({format: true}); + } + + async body() { + throw new Error('Body not implemented'); + } + + // -- stuff ------------------------------------------- + async search(from, query) { + const result = []; + const queryRes = await this.webWorker.search(from, query); + + for (const row of queryRes.found) { + const rec = { + id: row.id, + title: (row[from] || 'Без автора'), + q: `=${encodeURIComponent(row[from])}`, + }; + + result.push(rec); + } + + return result; + } + + async opdsQuery(from, query, otherTitle = '[Другие]', prevLen = 0) { + const queryRes = await this.webWorker.opdsQuery(from, query); + let count = 0; + for (const row of queryRes.found) + count += row.count; + + const others = []; + let result = []; + if (count <= 50) { + //конец навигации + return await this.search(from, query); + } else { + let len = 0; + for (const row of queryRes.found) { + const value = row.value; + len += value.length; + + let rec; + if (row.count == 1) { + rec = { + id: row.id, + title: row.name, + q: `=${encodeURIComponent(row.name)}`, + }; + } else { + rec = { + id: row.id, + title: `${value.toUpperCase().replace(/ /g, spaceChar)}~`, + q: encodeURIComponent(value), + }; + } + if (query.depth > 1 || enru.has(value[0])) { + result.push(rec); + } else { + others.push(rec); + } + } + + if (query[from] && query.depth > 1 && result.length < 10 && len > prevLen) { + //рекурсия, с увеличением глубины, для облегчения навигации + const newQuery = _.cloneDeep(query); + newQuery.depth++; + return await this.opdsQuery(from, newQuery, otherTitle, len); + } + } + + if (!query.others && others.length) + result.unshift({id: 'other', title: otherTitle, q: '___others'}); + + return (!query.others ? result : others); + } + + //скопировано из BaseList.js, часть функционала не используется + filterBooks(books, query) { + const s = query; + + const splitAuthor = (author) => { + if (!author) { + author = emptyFieldValue; + } + + const result = author.split(','); + if (result.length > 1) + result.push(author); + + return result; + }; + + const filterBySearch = (bookValue, searchValue) => { + if (!searchValue) + return true; + + if (!bookValue) + bookValue = emptyFieldValue; + + bookValue = bookValue.toLowerCase(); + searchValue = searchValue.toLowerCase(); + + //особая обработка префиксов + if (searchValue[0] == '=') { + + searchValue = searchValue.substring(1); + return bookValue.localeCompare(searchValue) == 0; + } else if (searchValue[0] == '*') { + + searchValue = searchValue.substring(1); + return bookValue !== emptyFieldValue && bookValue.indexOf(searchValue) >= 0; + } else if (searchValue[0] == '#') { + + searchValue = searchValue.substring(1); + return !bookValue || (bookValue !== emptyFieldValue && !enru.has(bookValue[0]) && bookValue.indexOf(searchValue) >= 0); + } else { + //where = `@dirtyIndexLR('value', ${db.esc(a)}, ${db.esc(a + maxUtf8Char)})`; + return bookValue.localeCompare(searchValue) >= 0 && bookValue.localeCompare(searchValue + maxUtf8Char) <= 0; + } + }; + + return books.filter((book) => { + //author + let authorFound = false; + const authors = splitAuthor(book.author); + for (const a of authors) { + if (filterBySearch(a, s.author)) { + authorFound = true; + break; + } + } + + //genre + let genreFound = !s.genre; + if (!genreFound) { + const searchGenres = new Set(s.genre.split(',')); + const bookGenres = book.genre.split(','); + + for (let g of bookGenres) { + if (!g) + g = emptyFieldValue; + + if (searchGenres.has(g)) { + genreFound = true; + break; + } + } + } + + //lang + let langFound = !s.lang; + if (!langFound) { + const searchLang = new Set(s.lang.split(',')); + langFound = searchLang.has(book.lang || emptyFieldValue); + } + + //date + let dateFound = !s.date; + if (!dateFound) { + const date = this.queryDate(s.date).split(','); + let [from = '0000-00-00', to = '9999-99-99'] = date; + + dateFound = (book.date >= from && book.date <= to); + } + + //librate + let librateFound = !s.librate; + if (!librateFound) { + const searchLibrate = new Set(s.librate.split(',').map(n => parseInt(n, 10)).filter(n => !isNaN(n))); + librateFound = searchLibrate.has(book.librate); + } + + return (this.showDeleted || !book.del) + && authorFound + && filterBySearch(book.series, s.series) + && filterBySearch(book.title, s.title) + && genreFound + && langFound + && dateFound + && librateFound + ; + }); + } + + bookAuthor(author) { + if (author) { + let a = author.split(','); + return a.slice(0, 3).join(', ') + (a.length > 3 ? ' и др.' : ''); + } + + return ''; + } + + async getGenres() { + let result; + if (!this.genres) { + const res = await this.webWorker.getGenreTree(); + + result = { + genreTree: res.genreTree, + genreMap: new Map(), + genreSection: new Map(), + }; + + for (const section of result.genreTree) { + result.genreSection.set(section.name, section.value); + + for (const g of section.value) + result.genreMap.set(g.value, g.name); + } + + this.genres = result; + } else { + result = this.genres; + } + + return result; + } +} + +module.exports = BasePage; \ No newline at end of file diff --git a/server/core/opds/BookPage.js b/server/core/opds/BookPage.js new file mode 100644 index 0000000..b3ad1e8 --- /dev/null +++ b/server/core/opds/BookPage.js @@ -0,0 +1,206 @@ +const path = require('path'); +const _ = require('lodash'); +const he = require('he'); +const dayjs = require('dayjs'); + +const BasePage = require('./BasePage'); +const Fb2Parser = require('../fb2/Fb2Parser'); + +class BookPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'book'; + this.title = 'Книга'; + + } + + formatSize(size) { + size = size/1024; + let unit = 'KB'; + if (size > 1024) { + size = size/1024; + unit = 'MB'; + } + return `${size.toFixed(1)} ${unit}`; + } + + inpxInfo(bookRec) { + const mapping = [ + {name: 'fileInfo', label: 'Информация о файле', value: [ + {name: 'folder', label: 'Папка'}, + {name: 'file', label: 'Файл'}, + {name: 'size', label: 'Размер'}, + {name: 'date', label: 'Добавлен'}, + {name: 'del', label: 'Удален'}, + {name: 'libid', label: 'LibId'}, + {name: 'insno', label: 'InsideNo'}, + ]}, + + {name: 'titleInfo', label: 'Общая информация', value: [ + {name: 'author', label: 'Автор(ы)'}, + {name: 'title', label: 'Название'}, + {name: 'series', label: 'Серия'}, + {name: 'genre', label: 'Жанр'}, + {name: 'librate', label: 'Оценка'}, + {name: 'lang', label: 'Язык книги'}, + {name: 'keywords', label: 'Ключевые слова'}, + ]}, + ]; + + const valueToString = (value, nodePath, b) => {//eslint-disable-line no-unused-vars + if (nodePath == 'fileInfo/file') + return `${value}.${b.ext}`; + + if (nodePath == 'fileInfo/size') + return `${this.formatSize(value)} (${value.toString().replace(/(\d)(?=(\d{3})+(?!\d))/g, '$1 ')} Bytes)`; + + if (nodePath == 'fileInfo/date') + return dayjs(value, 'YYYY-MM-DD').format('DD.MM.YYYY'); + + if (nodePath == 'fileInfo/del') + return (value ? 'Да' : null); + + if (nodePath == 'fileInfo/insno') + return (value ? value : null); + + if (nodePath == 'titleInfo/author') + return value.split(',').join(', '); + + if (nodePath == 'titleInfo/librate' && !value) + return null; + + if (typeof(value) === 'string') { + return value; + } + + return (value.toString ? value.toString() : ''); + }; + + let result = []; + const book = _.cloneDeep(bookRec); + book.series = [book.series, book.serno].filter(v => v).join(' #'); + + for (const item of mapping) { + const itemOut = {name: item.name, label: item.label, value: []}; + + for (const subItem of item.value) { + const subItemOut = { + name: subItem.name, + label: subItem.label, + value: valueToString(book[subItem.name], `${item.name}/${subItem.name}`, book) + }; + if (subItemOut.value) + itemOut.value.push(subItemOut); + } + + if (itemOut.value.length) + result.push(itemOut); + } + + return result; + } + + htmlInfo(title, infoList) { + let info = ''; + for (const part of infoList) { + if (part.value.length) + info += `

${part.label}

`; + for (const rec of part.value) + info += `

${rec.label}: ${rec.value}

`; + } + + if (info) + info = `

${title}

${info}`; + + return info; + } + + async body(req) { + const result = {}; + + result.link = this.baseLinks(req, true); + + const bookUid = req.query.uid; + const entry = []; + if (bookUid) { + const {bookInfo} = await this.webWorker.getBookInfo(bookUid); + + if (bookInfo) { + const {genreMap} = await this.getGenres(); + const fileFormat = `${bookInfo.book.ext}+zip`; + + //entry + const e = this.makeEntry({ + id: bookUid, + title: bookInfo.book.title || 'Без названия', + }); + + e['dc:language'] = bookInfo.book.lang; + e['dc:format'] = fileFormat; + + //genre + const genre = bookInfo.book.genre.split(','); + for (const g of genre) { + const genreName = genreMap.get(g); + if (genreName) { + if (!e.category) + e.category = []; + e.category.push({ + '*ATTRS': {term: genreName, label: genreName}, + }); + } + } + + let content = ''; + let ann = ''; + let info = ''; + //fb2 info + if (bookInfo.fb2) { + const parser = new Fb2Parser(bookInfo.fb2); + const infoObj = parser.bookInfo(); + + if (infoObj.titleInfo) { + if (infoObj.titleInfo.author.length) { + e.author = infoObj.titleInfo.author.map(a => ({name: a})); + } + + ann = infoObj.titleInfo.annotationHtml || ''; + const infoList = parser.bookInfoList(infoObj); + info += this.htmlInfo('Fb2 инфо', infoList); + } + } + + //content + info += this.htmlInfo('Inpx инфо', this.inpxInfo(bookInfo.book)); + + content = `${ann}${info}`; + if (content) { + e.content = { + '*ATTRS': {type: 'text/html'}, + '*TEXT': he.escape(content), + }; + } + + //links + e.link = [ this.downLink({href: bookInfo.link, type: `application/${fileFormat}`}) ]; + if (bookInfo.cover) { + let coverType = 'image/jpeg'; + if (path.extname(bookInfo.cover) == '.png') + coverType = 'image/png'; + + e.link.push(this.imgLink({href: bookInfo.cover, type: coverType})); + e.link.push(this.imgLink({href: bookInfo.cover, type: coverType, thumb: true})); + } + + entry.push(e); + } + } + + result.entry = entry; + + return this.makeBody(result, req); + } +} + +module.exports = BookPage; \ No newline at end of file diff --git a/server/core/opds/GenrePage.js b/server/core/opds/GenrePage.js new file mode 100644 index 0000000..f9df2b9 --- /dev/null +++ b/server/core/opds/GenrePage.js @@ -0,0 +1,72 @@ +const BasePage = require('./BasePage'); + +class GenrePage extends BasePage { + constructor(config) { + super(config); + + this.id = 'genre'; + this.title = 'Жанры'; + + } + + async body(req) { + const result = {}; + + const query = { + from: req.query.from || '', + section: req.query.section || '', + }; + + const entry = []; + if (query.from) { + + if (query.section) { + //выбираем подразделы + const {genreSection} = await this.getGenres(); + const section = genreSection.get(query.section); + + if (section) { + let id = 0; + const all = []; + for (const g of section) { + all.push(g.value); + entry.push( + this.makeEntry({ + id: ++id, + title: g.name, + link: this.navLink({href: `/${encodeURIComponent(query.from)}?genre=${encodeURIComponent(g.value)}`}), + }) + ); + } + + entry.unshift( + this.makeEntry({ + id: 'whole_section', + title: '[Весь раздел]', + link: this.navLink({href: `/${encodeURIComponent(query.from)}?genre=${encodeURIComponent(all.join(','))}`}), + }) + ); + } + } else { + //выбираем разделы + const {genreTree} = await this.getGenres(); + let id = 0; + for (const section of genreTree) { + entry.push( + this.makeEntry({ + id: ++id, + title: section.name, + link: this.navLink({href: `/genre?from=${encodeURIComponent(query.from)}§ion=${encodeURIComponent(section.name)}`}), + }) + ); + } + } + } + + result.entry = entry; + + return this.makeBody(result, req); + } +} + +module.exports = GenrePage; \ No newline at end of file diff --git a/server/core/opds/OpensearchPage.js b/server/core/opds/OpensearchPage.js new file mode 100644 index 0000000..128868b --- /dev/null +++ b/server/core/opds/OpensearchPage.js @@ -0,0 +1,45 @@ +const BasePage = require('./BasePage'); +const XmlParser = require('../xml/XmlParser'); + +class OpensearchPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'opensearch'; + this.title = 'opensearch'; + } + + async body() { + const xml = new XmlParser(); + const xmlObject = {}; +/* + + + inpx-web + Поиск по каталогу + UTF-8 + UTF-8 + + +*/ + xmlObject['OpenSearchDescription'] = { + '*ATTRS': {xmlns: 'http://a9.com/-/spec/opensearch/1.1/'}, + ShortName: 'inpx-web', + Description: 'Поиск по каталогу', + InputEncoding: 'UTF-8', + OutputEncoding: 'UTF-8', + Url: { + '*ATTRS': { + type: 'application/atom+xml;profile=opds-catalog;kind=navigation', + template: `${this.opdsRoot}/search?term={searchTerms}`, + }, + }, + } + + xml.fromObject(xmlObject); + + return xml.toString({format: true}); + } +} + +module.exports = OpensearchPage; \ No newline at end of file diff --git a/server/core/opds/RootPage.js b/server/core/opds/RootPage.js new file mode 100644 index 0000000..4ee4fb6 --- /dev/null +++ b/server/core/opds/RootPage.js @@ -0,0 +1,39 @@ +const BasePage = require('./BasePage'); +const AuthorPage = require('./AuthorPage'); +const SeriesPage = require('./SeriesPage'); +const TitlePage = require('./TitlePage'); + +class RootPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'root'; + this.title = ''; + + this.authorPage = new AuthorPage(config); + this.seriesPage = new SeriesPage(config); + this.titlePage = new TitlePage(config); + } + + async body(req) { + const result = {}; + + if (!this.title) { + const dbConfig = await this.webWorker.dbConfig(); + const collection = dbConfig.inpxInfo.collection.split('\n'); + this.title = collection[0].trim(); + if (!this.title) + this.title = 'Неизвестная коллекция'; + } + + result.entry = [ + this.authorPage.myEntry(), + this.seriesPage.myEntry(), + this.titlePage.myEntry(), + ]; + + return this.makeBody(result, req); + } +} + +module.exports = RootPage; \ No newline at end of file diff --git a/server/core/opds/SearchPage.js b/server/core/opds/SearchPage.js new file mode 100644 index 0000000..239069d --- /dev/null +++ b/server/core/opds/SearchPage.js @@ -0,0 +1,83 @@ +const BasePage = require('./BasePage'); + +class SearchPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'search'; + this.title = 'Поиск'; + } + + async body(req) { + const result = {}; + + const query = { + type: req.query.type || '', + term: req.query.term || '', + page: parseInt(req.query.page, 10) || 1, + }; + + let entry = []; + if (query.type) { + if (['author', 'series', 'title'].includes(query.type)) { + const from = query.type; + const page = query.page; + + const limit = 100; + const offset = (page - 1)*limit; + const queryRes = await this.webWorker.search(from, {[from]: query.term, del: 0, offset, limit}); + + const found = queryRes.found; + + for (let i = 0; i < found.length; i++) { + if (i >= limit) + break; + + const row = found[i]; + + entry.push( + this.makeEntry({ + id: row.id, + title: row[from], + link: this.navLink({href: `/${from}?${from}==${encodeURIComponent(row[from])}`}), + }), + ); + } + + if (queryRes.totalFound > offset + found.length) { + entry.push( + this.makeEntry({ + id: 'next_page', + title: '[Следующая страница]', + link: this.navLink({href: `/${this.id}?type=${from}&term=${encodeURIComponent(query.term)}&page=${page + 1}`}), + }), + ); + } + } + } else { + //корневой раздел + entry = [ + this.makeEntry({ + id: 'search_author', + title: 'Поиск авторов', + link: this.navLink({href: `/${this.id}?type=author&term=${encodeURIComponent(query.term)}`}), + }), + this.makeEntry({ + id: 'search_series', + title: 'Поиск серий', + link: this.navLink({href: `/${this.id}?type=series&term=${encodeURIComponent(query.term)}`}), + }), + this.makeEntry({ + id: 'search_title', + title: 'Поиск книг', + link: this.navLink({href: `/${this.id}?type=title&term=${encodeURIComponent(query.term)}`}), + }), + ] + } + + result.entry = entry; + return this.makeBody(result, req); + } +} + +module.exports = SearchPage; \ No newline at end of file diff --git a/server/core/opds/SeriesPage.js b/server/core/opds/SeriesPage.js new file mode 100644 index 0000000..260592a --- /dev/null +++ b/server/core/opds/SeriesPage.js @@ -0,0 +1,114 @@ +const BasePage = require('./BasePage'); + +class SeriesPage extends BasePage { + constructor(config) { + super(config); + + this.id = 'series'; + this.title = 'Серии'; + } + + sortSeriesBooks(seriesBooks) { + seriesBooks.sort((a, b) => { + const dserno = (a.serno || Number.MAX_VALUE) - (b.serno || Number.MAX_VALUE); + const dtitle = a.title.localeCompare(b.title); + const dext = a.ext.localeCompare(b.ext); + return (dserno ? dserno : (dtitle ? dtitle : dext)); + }); + + return seriesBooks; + } + + async body(req) { + const result = {}; + + const query = { + series: req.query.series || '', + genre: req.query.genre || '', + del: 0, + + all: req.query.all || '', + depth: 0, + }; + query.depth = query.series.length + 1; + + if (query.series == '___others') { + query.series = ''; + query.depth = 1; + query.others = true; + } + + const entry = []; + if (query.series && query.series[0] == '=') { + //книги по серии + const bookList = await this.webWorker.getSeriesBookList(query.series.substring(1)); + + if (bookList.books) { + let books = JSON.parse(bookList.books); + const booksAll = this.filterBooks(books, {del: 0}); + const filtered = (query.all ? booksAll : this.filterBooks(books, query)); + const sorted = this.sortSeriesBooks(filtered); + + if (booksAll.length > filtered.length) { + entry.push( + this.makeEntry({ + id: 'all_series_books', + title: '[Все книги серии]', + link: this.navLink({ + href: `/${this.id}?series=${encodeURIComponent(query.series)}&all=1`}), + }) + ); + } + + for (const book of sorted) { + const title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'} (${book.ext})`; + + const e = { + id: book._uid, + title, + link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), + }; + + if (query.all) { + e.content = { + '*ATTRS': {type: 'text'}, + '*TEXT': this.bookAuthor(book.author), + } + } + + entry.push( + this.makeEntry(e) + ); + } + } + } else { + if (query.depth == 1 && !query.genre && !query.others) { + entry.push( + this.makeEntry({ + id: 'select_genre', + title: '[Выбрать жанр]', + link: this.navLink({href: `/genre?from=${this.id}`}), + }) + ); + } + + //навигация по каталогу + const queryRes = await this.opdsQuery('series', query, '[Остальные серии]'); + + for (const rec of queryRes) { + entry.push( + this.makeEntry({ + id: rec.id, + title: rec.title, + link: this.navLink({href: `/${this.id}?series=${rec.q}&genre=${encodeURIComponent(query.genre)}`}), + }) + ); + } + } + + result.entry = entry; + return this.makeBody(result, req); + } +} + +module.exports = SeriesPage; \ No newline at end of file diff --git a/server/core/opds/TitlePage.js b/server/core/opds/TitlePage.js new file mode 100644 index 0000000..6509cfe --- /dev/null +++ b/server/core/opds/TitlePage.js @@ -0,0 +1,84 @@ +const BasePage = require('./BasePage'); + +class TitlePage extends BasePage { + constructor(config) { + super(config); + + this.id = 'title'; + this.title = 'Книги'; + } + + async body(req) { + const result = {}; + + const query = { + title: req.query.title || '', + genre: req.query.genre || '', + del: 0, + + depth: 0, + }; + query.depth = query.title.length + 1; + + if (query.title == '___others') { + query.title = ''; + query.depth = 1; + query.others = true; + } + + const entry = []; + if (query.title && query.title[0] == '=') { + //книги по названию + const res = await this.webWorker.search('title', query); + + if (res.found.length) { + const books = res.found[0].books || []; + const filtered = this.filterBooks(books, query); + + for (const book of filtered) { + const title = `${book.serno ? `${book.serno}. `: ''}${book.title || 'Без названия'} (${book.ext})`; + + entry.push( + this.makeEntry({ + id: book._uid, + title, + link: this.acqLink({href: `/book?uid=${encodeURIComponent(book._uid)}`}), + content: { + '*ATTRS': {type: 'text'}, + '*TEXT': this.bookAuthor(book.author), + }, + }) + ); + } + } + } else { + if (query.depth == 1 && !query.genre && !query.others) { + entry.push( + this.makeEntry({ + id: 'select_genre', + title: '[Выбрать жанр]', + link: this.navLink({href: `/genre?from=${this.id}`}), + }) + ); + } + + //навигация по каталогу + const queryRes = await this.opdsQuery('title', query, '[Остальные названия]'); + + for (const rec of queryRes) { + entry.push( + this.makeEntry({ + id: rec.id, + title: rec.title, + link: this.navLink({href: `/${this.id}?title=${rec.q}&genre=${encodeURIComponent(query.genre)}`}), + }) + ); + } + } + + result.entry = entry; + return this.makeBody(result, req); + } +} + +module.exports = TitlePage; \ No newline at end of file diff --git a/server/core/opds/index.js b/server/core/opds/index.js new file mode 100644 index 0000000..29aaab7 --- /dev/null +++ b/server/core/opds/index.js @@ -0,0 +1,82 @@ +const basicAuth = require('express-basic-auth'); + +const RootPage = require('./RootPage'); +const AuthorPage = require('./AuthorPage'); +const SeriesPage = require('./SeriesPage'); +const TitlePage = require('./TitlePage'); +const GenrePage = require('./GenrePage'); +const BookPage = require('./BookPage'); + +const OpensearchPage = require('./OpensearchPage'); +const SearchPage = require('./SearchPage'); + +module.exports = function(app, config) { + if (!config.opds || !config.opds.enabled) + return; + + const opdsRoot = '/opds'; + config.opdsRoot = opdsRoot; + + const root = new RootPage(config); + const author = new AuthorPage(config); + const series = new SeriesPage(config); + const title = new TitlePage(config); + const genre = new GenrePage(config); + const book = new BookPage(config); + + const opensearch = new OpensearchPage(config); + const search = new SearchPage(config); + + const routes = [ + ['', root], + ['/root', root], + ['/author', author], + ['/series', series], + ['/title', title], + ['/genre', genre], + ['/book', book], + + ['/opensearch', opensearch], + ['/search', search], + ]; + + const pages = new Map(); + for (const r of routes) { + pages.set(`${opdsRoot}${r[0]}`, r[1]); + } + + const opds = async(req, res, next) => { + try { + const page = pages.get(req.path); + + if (page) { + res.set('Content-Type', 'application/atom+xml; charset=utf-8'); + + const result = await page.body(req, res); + + if (result !== false) + res.send(result); + } else { + next(); + } + } catch (e) { + res.status(500).send({error: e.message}); + if (config.branch == 'development') { + console.error({error: e.message, url: req.originalUrl}); + } + } + }; + + const opdsPaths = [opdsRoot, `${opdsRoot}/*`]; + if (config.opds.password) { + if (!config.opds.user) + throw new Error('User must not be empty if password set'); + + app.use(opdsPaths, basicAuth({ + users: {[config.opds.user]: config.opds.password}, + challenge: true, + })); + } + app.get(opdsPaths, opds); +}; + diff --git a/server/index.js b/server/index.js index 9e2e8aa..0bc984b 100644 --- a/server/index.js +++ b/server/index.js @@ -154,6 +154,8 @@ async function main() { if (devModule) devModule.logQueries(app); + const opds = require('./core/opds'); + opds(app, config); initStatic(app, config); const { WebSocketController } = require('./controllers');