Merge branch 'release/1.5.6'

1.5.5

CHANGELOG.md

@@ -1,3 +1,15 @@
+1.5.6 / 2024-03-25
+
+- Добавлен вывод version.info в статистику по коллекции (#27)
+- Убрано (по умолчанию) ежеминутное журналирование статистики сервера. В конфиг добавлен параметр logServerStats (#26)
+- В конфиг добавлен параметр logQueries для журналирования запросов и времени их выполнения
+- Добавлена расшифровка имен жанров в информации о книге(#24)
+
+1.5.5 / 2023-04-25
+
+- Улучшение работы с inpx: теперь понимает zip-архивы, вложенные в каталоги (библиотека Траума)
+- Улучшение работы с zip-файлами: теперь понимает кодировку cp866 в именах файлов
+
 1.5.4 / 2023-04-12
 
 - Добавлена возможность поиска по типу файла

README.md

@@ -111,6 +111,12 @@ Options:
     // включить(true)/выключить(false) журналирование
     "loggingEnabled": true,
 
+    // включить/выключить ежеминутный вывод в лог memUsage и loadAvg
+    "logServerStats": false,
+
+    // включить/выключить вывод в лог запросов и времени их выполнения
+    "logQueries": false,
+
     // максимальный размер кеша каждой таблицы в БД, в блоках (требуется примерно 1-10Мб памяти на один блок)
     // если надо кешировать всю БД, можно поставить значение от 1000 и больше
     "dbCacheSize": 5,

client/components/Search/BookInfoDialog/BookInfoDialog.vue

@@ -118,6 +118,7 @@ class BookInfoDialog {
     _props = {
         modelValue: Boolean,
         bookInfo: Object,
+        genreMap: Object,
     };
 
     dialogVisible = false;
@@ -169,6 +170,19 @@ class BookInfoDialog {
         return `${size.toFixed(1)} ${unit}`;
     }
 
+    convertGenres(genreArr) {
+        let result = [];
+        if (genreArr) {
+            for (const genre of genreArr) {
+                const g = genre.trim();
+                const name = this.genreMap.get(g);
+                result.push(name ? name : g);
+            }
+        }
+
+        return result.join(', ');
+    }
+
     get inpx() {
         const mapping = [
             {name: 'fileInfo', label: 'Информация о файле', value: [
@@ -211,6 +225,9 @@ class BookInfoDialog {
             if (nodePath == 'titleInfo/author')
                 return value.split(',').join(', ');
 
+            if (nodePath == 'titleInfo/genre')
+                return this.convertGenres(value.split(','));
+
             if (nodePath == 'titleInfo/librate' && !value)
                 return null;
 
@@ -279,11 +296,16 @@ class BookInfoDialog {
                 }
             }
 
+            const self = this;
             this.fb2 = parser.bookInfoList(infoObj, {
                 valueToString(value, nodePath, origVTS) {//eslint-disable-line no-unused-vars
                     if (nodePath == 'documentInfo/historyHtml' && value)
                         return value.replace(/<p>/g, `<p class="p-history">`);
 
+                    if ((nodePath == 'titleInfo/genre' || nodePath == 'srcTitleInfo/genre') && value) {
+                        return self.convertGenres(value);
+                    }
+
                     return origVTS(value, nodePath);
                 },
             });

client/components/Search/Search.vue

@@ -347,7 +347,7 @@
         <SelectLibRateDialog v-model="selectLibRateDialogVisible" v-model:librate="search.librate" />
         <SelectDateDialog v-model="selectDateDialogVisible" v-model:date="search.date" />
         <SelectExtDialog v-model="selectExtDialogVisible" v-model:ext="search.ext" :ext-list="extList" />        
-        <BookInfoDialog v-model="bookInfoDialogVisible" :book-info="bookInfo" />
+        <BookInfoDialog v-model="bookInfoDialogVisible" :book-info="bookInfo" :genre-map="genreMap" />
         <SelectExtSearchDialog v-model="selectExtSearchDialogVisible" v-model:ext-search="extSearch" />        
     </div>
 </template>
@@ -931,6 +931,10 @@ class Search {
     <b>collection.info:</b>
     <pre>${inpxInfo.collection}</pre>
 </div>
+<div><hr/>
+    <b>version.info:</b>
+    <pre>${inpxInfo.version}</pre>
+</div>
 `;        
 
         this.$root.stdDialog.alert(info, 'Статистика по коллекции', {iconName: 'la la-info-circle'});

package.json

@@ -1,6 +1,6 @@
 {
   "name": "inpx-web",
-  "version": "1.5.4",
+  "version": "1.5.6",
   "author": "Book Pauk <bookpauk@gmail.com>",
   "license": "CC0-1.0",
   "repository": "bookpauk/inpx-web",
@@ -25,38 +25,38 @@
     "assets": "dist/public.json"
   },
   "devDependencies": {
-    "@babel/core": "^7.20.5",
+    "@babel/core": "^7.24.3",
-    "@babel/eslint-parser": "^7.19.1",
+    "@babel/eslint-parser": "^7.24.1",
-    "@babel/eslint-plugin": "^7.19.1",
+    "@babel/eslint-plugin": "^7.23.5",
-    "@babel/plugin-proposal-decorators": "^7.20.5",
+    "@babel/plugin-proposal-decorators": "^7.24.1",
-    "@babel/preset-env": "^7.20.2",
+    "@babel/preset-env": "^7.24.3",
     "@vue/compiler-sfc": "^3.2.22",
-    "babel-loader": "^9.1.0",
+    "babel-loader": "^9.1.3",
     "copy-webpack-plugin": "^11.0.0",
-    "css-loader": "^6.7.2",
+    "css-loader": "^6.10.0",
     "css-minimizer-webpack-plugin": "^4.2.2",
-    "eslint": "^8.28.0",
+    "eslint": "^8.57.0",
-    "eslint-plugin-vue": "^9.8.0",
+    "eslint-plugin-vue": "^9.23.0",
-    "html-webpack-plugin": "^5.5.0",
+    "html-webpack-plugin": "^5.6.0",
-    "mini-css-extract-plugin": "^2.7.1",
+    "mini-css-extract-plugin": "^2.8.1",
-    "pkg": "^5.8.0",
+    "pkg": "^5.8.1",
     "showdown": "^2.1.0",
-    "terser-webpack-plugin": "^5.3.6",
+    "terser-webpack-plugin": "^5.3.10",
-    "vue-eslint-parser": "^9.1.0",
+    "vue-eslint-parser": "^9.4.2",
-    "vue-loader": "^17.0.1",
+    "vue-loader": "^17.4.2",
     "vue-style-loader": "^4.1.3",
-    "webpack": "^5.75.0",
+    "webpack": "^5.91.0",
-    "webpack-cli": "^5.0.0",
+    "webpack-cli": "^5.1.4",
-    "webpack-dev-middleware": "^6.0.1",
+    "webpack-dev-middleware": "^6.1.2",
-    "webpack-hot-middleware": "^2.25.3",
+    "webpack-hot-middleware": "^2.26.1",
-    "webpack-merge": "^5.8.0"
+    "webpack-merge": "^5.10.0"
   },
   "dependencies": {
-    "@quasar/extras": "^1.15.6",
+    "@quasar/extras": "^1.16.9",
     "axios": "^0.27.2",
-    "chardet": "^1.5.0",
+    "chardet": "^1.6.0",
-    "dayjs": "^1.11.6",
+    "dayjs": "^1.11.10",
-    "express": "^4.18.2",
+    "express": "^4.19.1",
     "express-basic-auth": "^1.2.1",
     "fs-extra": "^10.1.0",
     "he": "^1.2.0",
@@ -64,15 +64,15 @@
     "jembadb": "^5.1.7",
     "localforage": "^1.10.0",
     "lodash": "^4.17.21",
-    "minimist": "^1.2.7",
+    "minimist": "^1.2.8",
     "node-stream-zip": "^1.15.0",
-    "quasar": "^2.10.2",
+    "quasar": "^2.15.1",
     "safe-buffer": "^5.2.1",
     "vue": "^3.2.37",
-    "vue-router": "^4.1.6",
+    "vue-router": "^4.3.0",
     "vuex": "^4.1.0",
     "vuex-persist": "^3.1.3",
-    "ws": "^8.11.0",
+    "ws": "^8.16.0",
     "yazl": "^2.5.1"
   }
 }

server/config/base.js

@@ -18,6 +18,8 @@ module.exports = {
     extendedSearch: true,
     bookReadLink: '',
     loggingEnabled: true,
+    logServerStats: false,
+    logQueries: false,
 
     //поправить в случае, если были критические изменения в DbCreator или InpxParser
     //иначе будет рассинхронизация по кешу между сервером и клиентом на уровне БД

server/config/index.js

@@ -12,6 +12,8 @@ const propsToSave = [
     'extendedSearch',
     'bookReadLink',
     'loggingEnabled',
+    'logServerStats',
+    'logQueries',
     'dbCacheSize',
     'maxFilesDirSize',
     'queryCacheEnabled',

server/controllers/WebSocketController.js

@@ -56,11 +56,12 @@ class WebSocketController {
     async onMessage(ws, message) {
         let req = {};
         try {
-            if (this.isDevelopment) {
+            if (this.isDevelopment || this.config.logQueries) {
-                log(`WebSocket-IN:  ${message.substr(0, 4000)}`);
+                log(`WebSocket-IN:  ${utils.cutString(message)}`);
             }
 
             req = JSON.parse(message);
+            req.__startTime = Date.now();
 
             ws.lastActivity = Date.now();
             
@@ -123,8 +124,9 @@ class WebSocketController {
             const message = JSON.stringify(r);
             ws.send(message);
 
-            if (this.isDevelopment) {
+            if (this.isDevelopment || this.config.logQueries) {
-                log(`WebSocket-OUT: ${message.substr(0, 200)}`);
+                log(`WebSocket-OUT: ${utils.cutString(message)}`);
+                log(`${Date.now() - req.__startTime}ms`);
             }
 
         }

server/core/WebWorker.js

@@ -2,6 +2,7 @@ const os = require('os');
 const path = require('path');
 const fs = require('fs-extra');
 const _ = require('lodash');
+const iconv = require('iconv-lite');
 
 const ZipReader = require('./ZipReader');
 const WorkerState = require('./WorkerState');//singleton
@@ -313,7 +314,7 @@ class WebWorker {
 
         let result;
         const db = this.db;
-        if (!db.wwCache.genres) {
+        if (!db.wwCache.genreTree) {
             const genres = _.cloneDeep(genreTree);
             const last = genres[genres.length - 1];
 
@@ -361,22 +362,43 @@ class WebWorker {
                 inpxHash: (config.inpxHash ? config.inpxHash : ''),
             };
 
-            db.wwCache.genres = result;
+            db.wwCache.genreTree = result;
         } else {
-            result = db.wwCache.genres;
+            result = db.wwCache.genreTree;
         }
 
         return result;
     }
 
-    async extractBook(bookPath) {
+    async getGenreMap() {
+        this.checkMyState();
+
+        let result;
+        const db = this.db;
+        if (!db.wwCache.genreMap) {
+            const genreTree = await this.getGenreTree();
+
+            result = new Map();
+            for (const section of genreTree.genreTree) {
+                for (const g of section.value)
+                    result.set(g.value, g.name);
+            }
+
+            db.wwCache.genreMap = result;
+        } else {
+            result = db.wwCache.genreMap;
+        }
+
+        return result;
+    }
+
+    async extractBook(libFolder, libFile) {
         const outFile = `${this.config.tempDir}/${utils.randomHexString(30)}`;
 
-        bookPath = bookPath.replace(/\\/g, '/').replace(/\/\//g, '/');
+        libFolder = libFolder.replace(/\\/g, '/').replace(/\/\//g, '/');
 
-        const i = bookPath.indexOf('/');
+        const folder = `${this.config.libDir}/${libFolder}`;
-        const folder = `${this.config.libDir}/${(i >= 0 ? bookPath.substring(0, i) : bookPath )}`;
+        const file = libFile;
-        const file = (i >= 0 ? bookPath.substring(i + 1) : '' );
         
         const fullPath = `${folder}/${file}`;
 
@@ -390,6 +412,11 @@ class WebWorker {
 
             try {
                 await zipReader.extractToFile(file, outFile);
+
+                if (!await fs.pathExists(outFile)) {//не удалось найти в архиве, попробуем имя файла в кодировке cp866
+                    await zipReader.extractToFile(iconv.encode(file, 'cp866').toString(), outFile);                    
+                }
+
                 return outFile;
             } finally {
                 await zipReader.close();
@@ -397,14 +424,14 @@ class WebWorker {
         }
     }
 
-    async restoreBook(bookUid, bookPath, downFileName) {
+    async restoreBook(bookUid, libFolder, libFile, downFileName) {
         const db = this.db;
 
         let extractedFile = '';
         let hash = '';
 
         if (!this.remoteLib) {
-            extractedFile = await this.extractBook(bookPath);
+            extractedFile = await this.extractBook(libFolder, libFile);
             hash = await utils.getFileHash(extractedFile, 'sha256', 'hex');
         } else {
             hash = await this.remoteLib.downloadBook(bookUid);
@@ -424,7 +451,7 @@ class WebWorker {
                 await utils.touchFile(bookFile);
             }
 
-            await fs.writeFile(bookFileDesc, JSON.stringify({bookPath, downFileName}));
+            await fs.writeFile(bookFileDesc, JSON.stringify({libFolder, libFile, downFileName}));
         } else {
             if (extractedFile)
                 await fs.remove(extractedFile);
@@ -437,8 +464,7 @@ class WebWorker {
             table: 'file_hash',
             replace: true,
             rows: [
-                {id: bookPath, hash},
+                {id: bookUid, hash},
-                {id: hash, bookPath, downFileName}
             ]
         });
 
@@ -452,7 +478,7 @@ class WebWorker {
             const db = this.db;
             let link = '';
 
-            //найдем downFileName и bookPath
+            //найдем downFileName, libFolder, libFile
             let rows = await db.select({table: 'book', where: `@@hash('_uid', ${db.esc(bookUid)})`});
             if (!rows.length)
                 throw new Error('404 Файл не найден');
@@ -479,11 +505,12 @@ class WebWorker {
             if (downFileName.substring(downFileName.length - ext.length) != ext)
                 downFileName += ext;
 
-            const bookPath = `${book.folder}/${book.file}${ext}`;
+            const libFolder = book.folder;
+            const libFile = `${book.file}${ext}`;
 
             //найдем хеш
-            rows = await db.select({table: 'file_hash', where: `@@id(${db.esc(bookPath)})`});
+            rows = await db.select({table: 'file_hash', where: `@@id(${db.esc(bookUid)})`});
-            if (rows.length) {//хеш найден по bookPath
+            if (rows.length) {//хеш найден по bookUid
                 const hash = rows[0].hash;
                 const bookFile = `${this.config.bookDir}/${hash}`;
                 const bookFileDesc = `${bookFile}.d.json`;
@@ -494,13 +521,13 @@ class WebWorker {
             }
 
             if (!link) {
-                link = await this.restoreBook(bookUid, bookPath, downFileName)
+                link = await this.restoreBook(bookUid, libFolder, libFile, downFileName);
             }
 
             if (!link)
                 throw new Error('404 Файл не найден');
 
-            return {link, bookPath, downFileName};
+            return {link, libFolder, libFile, downFileName};
         } catch(e) {
             log(LM_ERR, `getBookLink error: ${e.message}`);
             if (e.message.indexOf('ENOENT') >= 0)
@@ -599,16 +626,16 @@ class WebWorker {
             let loadAvg = os.loadavg();
             loadAvg = loadAvg.map(v => v.toFixed(2));
 
-            log(`Server info [ memUsage: ${memUsage.toFixed(2)}MB, loadAvg: (${loadAvg.join(', ')}) ]`);
+            log(`Server stats [ memUsage: ${memUsage.toFixed(2)}MB, loadAvg: (${loadAvg.join(', ')}) ]`);
-
-            if (this.config.server.ready)
-                log(`Server accessible at http://127.0.0.1:${this.config.server.port} (listening on ${this.config.server.host}:${this.config.server.port})`);
         } catch (e) {
             log(LM_ERR, e.message);
         }
     }
     
     async periodicLogServerStats() {
+        if (!this.config.logServerStats)
+            return;
+
         while (1) {// eslint-disable-line
             this.logServerStats();
             await utils.sleep(60*1000);

server/core/opds/BookPage.js

@@ -24,6 +24,19 @@ class BookPage extends BasePage {
         return `${size.toFixed(1)} ${unit}`;
     }
 
+    convertGenres(genreArr) {
+        let result = [];
+        if (genreArr) {
+            for (const genre of genreArr) {
+                const g = genre.trim();
+                const name = this.genreMap.get(g);
+                result.push(name ? name : g);
+            }
+        }
+
+        return result.join(', ');
+    }
+
     inpxInfo(bookRec) {
         const mapping = [
             {name: 'fileInfo', label: 'Информация о файле', value: [
@@ -66,6 +79,9 @@ class BookPage extends BasePage {
             if (nodePath == 'titleInfo/author')
                 return value.split(',').join(', ');
 
+            if (nodePath == 'titleInfo/genre')
+                return this.convertGenres(value.split(','));
+
             if (nodePath == 'titleInfo/librate' && !value)
                 return null;
 
@@ -118,6 +134,7 @@ class BookPage extends BasePage {
     async body(req) {
         const result = {};
 
+        this.genreMap = await this.webWorker.getGenreMap();
         result.link = this.baseLinks(req, true);
 
         const bookUid = req.query.uid;
@@ -183,7 +200,17 @@ class BookPage extends BasePage {
                         }
 
                         ann = infoObj.titleInfo.annotationHtml || '';
-                        const infoList = parser.bookInfoList(infoObj);
+                        const self = this;
+                        const infoList = parser.bookInfoList(infoObj, {
+                            valueToString(value, nodePath, origVTS) {//eslint-disable-line no-unused-vars
+                                if ((nodePath == 'titleInfo/genre' || nodePath == 'srcTitleInfo/genre') && value) {
+                                    return self.convertGenres(value);
+                                }
+
+                                return origVTS(value, nodePath);
+                            },
+                        });
+
                         info += this.htmlInfo('Fb2 инфо', infoList);
                     }
                 }

server/core/opds/index.js

@@ -74,7 +74,12 @@ module.exports = function(app, config) {
     if (config.opds.password) {
         if (!config.opds.user)
             throw new Error('User must not be empty if password set');
-
+/*
+        app.use((req, res, next) => {
+            console.log(req.headers);
+            next();
+        });
+*/
         app.use(opdsPaths, basicAuth({
             users: {[config.opds.user]: config.opds.password},
             challenge: true,

server/core/utils.js

@@ -196,6 +196,20 @@ function wordEnding(num, type = 0) {
     }
 }
 
+function cutString(data, len = 500) {
+    try {
+        if (!data)
+            return '';
+
+        if (typeof(data) !== 'string')
+            data = JSON.stringify(data);
+
+        return `${data.substring(0, len)}${data.length > len ? ' ...' : ''}`;
+    } catch (e) {
+        return '';
+    }
+}
+
 module.exports = {
     sleep,
     processLoop,
@@ -216,4 +230,5 @@ module.exports = {
     makeValidFileName,
     makeValidFileNameOrEmpty,
     wordEnding,
+    cutString,
 };

server/index.js

@@ -137,6 +137,18 @@ async function init() {
     }
 }
 
+function logQueries(app) {
+    app.use(function(req, res, next) {
+        const start = Date.now();
+        log(`${req.method} ${req.originalUrl} ${utils.cutString(req.body)}`);
+        //log(`${JSON.stringify(req.headers, null, 2)}`)
+        res.once('finish', () => {
+            log(`${Date.now() - start}ms`);
+        });
+        next();
+    });
+}
+
 async function main() {
     const log = new (require('./core/AppLogger'))().log;//singleton
 
@@ -168,6 +180,10 @@ async function main() {
     const { WebSocketController } = require('./controllers');
     new WebSocketController(wss, webAccess, config);
 
+    if (config.logQueries) {
+        logQueries(app);
+    }
+
     if (devModule) {
         devModule.logErrors(app);
     } else {
@@ -179,7 +195,7 @@ async function main() {
 
     server.listen(config.server.port, config.server.host, () => {
         config.server.ready = true;
-        log(`Server ready`);
+        log(`Server accessible at http://127.0.0.1:${config.server.port} (listening on ${config.server.host}:${config.server.port})`);
     });
 }