diff --git a/client/components/Reader/ServerStorage/ServerStorage.vue b/client/components/Reader/ServerStorage/ServerStorage.vue index 93c1ee48..2a338139 100644 --- a/client/components/Reader/ServerStorage/ServerStorage.vue +++ b/client/components/Reader/ServerStorage/ServerStorage.vue @@ -403,7 +403,7 @@ class ServerStorage extends Vue { const md = newRecentMod.data; if (md.key && result[md.key]) - result[md.key] = utils.applyObjDiff(result[md.key], md.mod); + result[md.key] = utils.applyObjDiff(result[md.key], md.mod, true); if (!bookManager.loaded) { this.warning('Ожидание загрузки списка книг перед синхронизацией'); @@ -463,11 +463,11 @@ class ServerStorage extends Vue { if (itemKey && !needSaveRecentMod) { newRecentPatch = _.cloneDeep(this.cachedRecentPatch); newRecentPatch.rev++; - newRecentPatch.data[itemKey] = bm.recent[itemKey]; + newRecentPatch.data[itemKey] = _.cloneDeep(bm.recent[itemKey]); let applyMod = this.cachedRecentMod.data; if (applyMod && applyMod.key && newRecentPatch.data[applyMod.key]) - newRecentPatch.data[applyMod.key] = utils.applyObjDiff(newRecentPatch.data[applyMod.key], applyMod.mod); + newRecentPatch.data[applyMod.key] = utils.applyObjDiff(newRecentPatch.data[applyMod.key], applyMod.mod, true); newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}}; needSaveRecentPatch = true; @@ -481,7 +481,7 @@ class ServerStorage extends Vue { while (!bookManager.loaded) await utils.sleep(100); - newRecent = {rev: this.cachedRecent.rev + 1, data: bm.recent}; + newRecent = {rev: this.cachedRecent.rev + 1, data: _.cloneDeep(bm.recent)}; newRecentPatch = {rev: this.cachedRecentPatch.rev + 1, data: {}}; newRecentMod = {rev: this.cachedRecentMod.rev + 1, data: {}}; needSaveRecent = true; @@ -510,9 +510,10 @@ class ServerStorage extends Vue { if (result.state == 'reject') { - await this.loadRecent(false, false); + const res = await this.loadRecent(false, false); - this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`); + if (res) + this.warning(`Последние изменения отменены. Данные синхронизированы с сервером.`); if (!recurse && itemKey) { this.savingRecent = false; this.saveRecent(itemKey, true); diff --git a/client/components/Reader/TextPage/TextPage.vue b/client/components/Reader/TextPage/TextPage.vue index 79b66777..77c51b4f 100644 --- a/client/components/Reader/TextPage/TextPage.vue +++ b/client/components/Reader/TextPage/TextPage.vue @@ -477,7 +477,7 @@ class TextPage extends Vue { generateWaitingFunc(waitingHandlerName, stopPropertyName) { const func = (timeout) => { - return new Promise(async(resolve) => { + return new Promise((resolve, reject) => { (async() => { this[waitingHandlerName] = resolve; let wait = (timeout + 201)/100; while (wait > 0 && !this[stopPropertyName]) { @@ -485,7 +485,7 @@ class TextPage extends Vue { await sleep(100); } resolve(); - }); + })().catch(reject); }); }; return func; } diff --git a/client/components/Reader/share/BookParser.js b/client/components/Reader/share/BookParser.js index a5faba72..8afe9875 100644 --- a/client/components/Reader/share/BookParser.js +++ b/client/components/Reader/share/BookParser.js @@ -1,5 +1,5 @@ import he from 'he'; -import sax from '../../../../server/core/BookConverter/sax'; +import sax from '../../../../server/core/sax'; import {sleep} from '../../../share/utils'; const maxImageLineCount = 100; @@ -67,7 +67,7 @@ export default class BookParser { } */ const getImageDimensions = (binaryId, binaryType, data) => { - return new Promise (async(resolve, reject) => { + return new Promise ((resolve, reject) => { (async() => { const i = new Image(); let resolved = false; i.onload = () => { @@ -81,19 +81,17 @@ export default class BookParser { resolve(); }; - i.onerror = (e) => { - reject(e); - }; + i.onerror = reject; i.src = `data:${binaryType};base64,${data}`; await sleep(30*1000); if (!resolved) reject('Не удалось получить размер изображения'); - }); + })().catch(reject); }); }; const getExternalImageDimensions = (src) => { - return new Promise (async(resolve, reject) => { + return new Promise ((resolve, reject) => { (async() => { const i = new Image(); let resolved = false; i.onload = () => { @@ -105,15 +103,13 @@ export default class BookParser { resolve(); }; - i.onerror = (e) => { - reject(e); - }; + i.onerror = reject; i.src = src; await sleep(30*1000); if (!resolved) reject('Не удалось получить размер изображения'); - }); + })().catch(reject); }); }; const newParagraph = (text, len, addIndex) => { diff --git a/package-lock.json b/package-lock.json index 736cb578..0bf5bd7e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.7.3", + "version": "0.7.5", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -541,11 +541,6 @@ "integrity": "sha512-tiNTrP1MP0QrChmD2DdupCr6HWSFeKVw5d/dHTu4Y7rkAkRhU/Dt7dphAfIUyxtHpl/eBVip5uTNSpQJHylpAw==", "dev": true }, - "adm-zip": { - "version": "0.4.13", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.4.13.tgz", - "integrity": "sha512-fERNJX8sOXfel6qCBCMPvZLzENBEhZTzKqg6vrOW5pvoEaQuJhRU4ndTAh6lHOxn1I6jnz2NHra56ZODM751uw==" - }, "ajv": { "version": "6.10.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz", @@ -640,6 +635,57 @@ "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" }, + "archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "requires": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "are-we-there-yet": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", @@ -1868,6 +1914,11 @@ "ieee754": "^1.1.4" } }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -2359,6 +2410,51 @@ "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", "dev": true }, + "compress-commons": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-2.1.1.tgz", + "integrity": "sha512-eVw6n7CnEMFzc3duyFVrQEuY1BlHR3rYsSztyG32ibGMW722i3C6IizEGMFmfMU+A+fALvBIwxN3czffTcdA+Q==", + "requires": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^3.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^2.3.6" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "compressible": { "version": "2.0.17", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.17.tgz", @@ -2598,6 +2694,43 @@ } } }, + "crc": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz", + "integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==", + "requires": { + "buffer": "^5.1.0" + } + }, + "crc32-stream": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-3.0.1.tgz", + "integrity": "sha512-mctvpXlbzsvK+6z8kJwSJ5crm7yBwrQMTybJzMw1O4lLGJqjlDCXY2Zw7KheiA6XBEcBmfLx1D88mjRGVJtY9w==", + "requires": { + "crc": "^3.4.4", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", + "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + } + } + }, "create-ecdh": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz", @@ -6252,6 +6385,48 @@ "webpack-sources": "^1.1.0" } }, + "lazystream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.0.tgz", + "integrity": "sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=", + "requires": { + "readable-stream": "^2.0.5" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "lcid": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz", @@ -6330,12 +6505,37 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, "lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", "integrity": "sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4=", "dev": true }, + "lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=" + }, "lodash.uniq": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", @@ -7011,11 +7211,15 @@ "semver": "^5.3.0" } }, + "node-stream-zip": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.8.2.tgz", + "integrity": "sha512-zwP2F/R28Oqtl0gOLItk5QjJ6jEU8XO4kaUMgeqvCyXPgdCZlm8T/5qLMiNy+moJCBCiMQAaX7aVMRhT0t2vkQ==" + }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, "normalize-url": { "version": "4.4.0", @@ -12647,6 +12851,36 @@ "camelcase": "^5.0.0", "decamelize": "^1.2.0" } + }, + "zip-stream": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-2.1.2.tgz", + "integrity": "sha512-ykebHGa2+uzth/R4HZLkZh3XFJzivhVsjJt8bN3GvBzLaqqrUdRacu+c4QtnUgjkkQfsOuNE1JgLKMCPNmkKgg==", + "requires": { + "archiver-utils": "^2.1.0", + "compress-commons": "^2.1.1", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", + "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + } + } } } } diff --git a/package.json b/package.json index 8a4d7311..defa4793 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,6 @@ "webpack-merge": "^4.2.2" }, "dependencies": { - "adm-zip": "^0.4.13", "appcache-webpack-plugin": "^1.4.0", "axios": "^0.18.1", "base-x": "^3.0.6", @@ -72,6 +71,7 @@ "lodash": "^4.17.15", "minimist": "^1.2.0", "multer": "^1.4.2", + "node-stream-zip": "^1.8.2", "pako": "^1.0.10", "path-browserify": "^1.0.0", "safe-buffer": "^5.2.0", @@ -83,6 +83,7 @@ "vue": "github:paulkamer/vue#fix_palemoon_clickhandlers_dist", "vue-router": "^3.1.3", "vuex": "^3.1.1", - "vuex-persistedstate": "^2.5.4" + "vuex-persistedstate": "^2.5.4", + "zip-stream": "^2.1.2" } } diff --git a/server/config/base.js b/server/config/base.js index 668131dd..33007a57 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -14,6 +14,7 @@ module.exports = { logDir: `${dataDir}/log`, publicDir: `${execDir}/public`, uploadDir: `${execDir}/public/upload`, + sharedDir: `${execDir}/public/shared`, loggingEnabled: true, maxUploadFileSize: 50*1024*1024,//50Мб diff --git a/server/config/configSaver.js b/server/config/configSaver.js deleted file mode 100644 index f4844a5c..00000000 --- a/server/config/configSaver.js +++ /dev/null @@ -1,37 +0,0 @@ -const fs = require('fs-extra'); -const _ = require('lodash'); - -const propsToSave = [ - 'maxUploadFileSize', - 'maxTempPublicDirSize', - 'maxUploadPublicDirSize', - 'useExternalBookConverter', - - 'servers', -]; - -async function load(config, configFilename) { - if (!configFilename) { - configFilename = `${config.dataDir}/config.json`; - - if (!await fs.pathExists(configFilename)) { - save(config); - return; - } - } - - const data = await fs.readFile(configFilename, 'utf8'); - Object.assign(config, JSON.parse(data)); -} - -async function save(config) { - const configFilename = `${config.dataDir}/config.json`; - const dataToSave = _.pick(config, propsToSave); - - await fs.writeFile(configFilename, JSON.stringify(dataToSave, null, 4)); -} - -module.exports = { - load, - save -}; \ No newline at end of file diff --git a/server/config/development.js b/server/config/development.js index a7e5f125..89c647ca 100644 --- a/server/config/development.js +++ b/server/config/development.js @@ -1,6 +1,5 @@ const base = require('./base'); module.exports = Object.assign({}, base, { - branch: 'development', - } -); + branch: 'development', +}); diff --git a/server/config/index.js b/server/config/index.js index c8b29ea8..fd8ae045 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -1,23 +1,91 @@ +const _ = require('lodash'); const fs = require('fs-extra'); -const utils = require('../core/utils'); const branchFilename = __dirname + '/application_env'; -let branch = 'production'; -try { - fs.accessSync(branchFilename); - branch = fs.readFileSync(branchFilename, 'utf8').trim(); -} catch (err) { +const propsToSave = [ + 'maxUploadFileSize', + 'maxTempPublicDirSize', + 'maxUploadPublicDirSize', + 'useExternalBookConverter', + + 'servers', +]; + +let instance = null; + +//singleton +class ConfigManager { + constructor() { + if (!instance) { + this.inited = false; + + instance = this; + } + + return instance; + } + + async init() { + if (this.inited) + throw new Error('already inited'); + + this.branch = 'production'; + try { + await fs.access(branchFilename); + this.branch = (await fs.readFile(branchFilename, 'utf8')).trim(); + } catch (err) { + // + } + + process.env.NODE_ENV = this.branch; + + this.branchConfigFile = __dirname + `/${this.branch}.js`; + await fs.access(this.branchConfigFile); + this._config = require(this.branchConfigFile); + + this._userConfigFile = `${this._config.dataDir}/config.json`; + + this.inited = true; + } + + get config() { + if (!this.inited) + throw new Error('not inited'); + return _.cloneDeep(this._config); + } + + set config(value) { + Object.assign(this._config, value); + } + + get userConfigFile() { + return this._userConfigFile; + } + + set userConfigFile(value) { + if (value) + this._userConfigFile = value; + } + + async load() { + if (!this.inited) + throw new Error('not inited'); + if (!await fs.pathExists(this.userConfigFile)) { + await this.save(); + return; + } + + const data = await fs.readFile(this.userConfigFile, 'utf8'); + this.config = JSON.parse(data); + } + + async save() { + if (!this.inited) + throw new Error('not inited'); + const dataToSave = _.pick(this._config, propsToSave); + await fs.writeFile(this.userConfigFile, JSON.stringify(dataToSave, null, 4)); + } } -process.env.NODE_ENV = branch; - -const confFilename = __dirname + `/${branch}.js`; - -fs.accessSync(confFilename); - -const config = require(confFilename); - -//fs.ensureDirSync(config.dataDir); - -module.exports = config; \ No newline at end of file +module.exports = ConfigManager; \ No newline at end of file diff --git a/server/config/production.js b/server/config/production.js index d98af428..79a155dc 100644 --- a/server/config/production.js +++ b/server/config/production.js @@ -5,21 +5,21 @@ const execDir = path.dirname(process.execPath); const dataDir = `${execDir}/data`; module.exports = Object.assign({}, base, { - branch: 'production', - dataDir: dataDir, - tempDir: `${dataDir}/tmp`, - logDir: `${dataDir}/log`, - publicDir: `${execDir}/public`, - uploadDir: `${execDir}/public/upload`, + branch: 'production', + dataDir: dataDir, + tempDir: `${dataDir}/tmp`, + logDir: `${dataDir}/log`, + publicDir: `${execDir}/public`, + uploadDir: `${execDir}/public/upload`, + sharedDir: `${execDir}/public/shared`, - servers: [ - { - serverName: '1', - mode: 'normal', //'none', 'normal', 'site', 'reader', 'omnireader' - ip: '0.0.0.0', - port: '44080', - }, - ], + servers: [ + { + serverName: '1', + mode: 'normal', //'none', 'normal', 'site', 'reader', 'omnireader' + ip: '0.0.0.0', + port: '44080', + }, + ], - } -); +}); diff --git a/server/controllers/MiscController.js b/server/controllers/MiscController.js index 2e10f188..84385462 100644 --- a/server/controllers/MiscController.js +++ b/server/controllers/MiscController.js @@ -1,5 +1,4 @@ const BaseController = require('./BaseController'); -const log = require('../core/getLogger').getLog(); const _ = require('lodash'); class MiscController extends BaseController { diff --git a/server/controllers/ReaderController.js b/server/controllers/ReaderController.js index 1fd19366..c2288ec7 100644 --- a/server/controllers/ReaderController.js +++ b/server/controllers/ReaderController.js @@ -1,12 +1,14 @@ const BaseController = require('./BaseController'); -const ReaderWorker = require('../core/ReaderWorker'); -const readerStorage = require('../core/readerStorage'); -const workerState = require('../core/workerState'); +const ReaderWorker = require('../core/Reader/ReaderWorker');//singleton +const ReaderStorage = require('../core/Reader/ReaderStorage');//singleton +const WorkerState = require('../core/WorkerState');//singleton class ReaderController extends BaseController { constructor(config) { super(config); + this.readerStorage = new ReaderStorage(); this.readerWorker = new ReaderWorker(config); + this.workerState = new WorkerState(); } async loadBook(req, res) { @@ -19,7 +21,7 @@ class ReaderController extends BaseController { url: request.url, enableSitesFilter: (request.hasOwnProperty('enableSitesFilter') ? request.enableSitesFilter : true) }); - const state = workerState.getState(workerId); + const state = this.workerState.getState(workerId); return (state ? state : {}); } catch (e) { error = e.message; @@ -38,7 +40,7 @@ class ReaderController extends BaseController { if (!request.items || Array.isArray(request.data)) throw new Error(`key 'items' is empty`); - return await readerStorage.doAction(request); + return await this.readerStorage.doAction(request); } catch (e) { error = e.message; } diff --git a/server/controllers/WorkerController.js b/server/controllers/WorkerController.js index 3437077f..69bd4982 100644 --- a/server/controllers/WorkerController.js +++ b/server/controllers/WorkerController.js @@ -1,7 +1,12 @@ const BaseController = require('./BaseController'); -const workerState = require('../core/workerState'); +const WorkerState = require('../core/WorkerState');//singleton class WorkerController extends BaseController { + constructor(config) { + super(config); + this.workerState = new WorkerState(); + } + async getState(req, res) { const request = req.body; let error = ''; @@ -9,7 +14,7 @@ class WorkerController extends BaseController { if (!request.workerId) throw new Error(`key 'workerId' is wrong`); - const state = workerState.getState(request.workerId); + const state = this.workerState.getState(request.workerId); return (state ? state : {}); } catch (e) { error = e.message; diff --git a/server/core/AppLogger.js b/server/core/AppLogger.js new file mode 100644 index 00000000..0d3bb7fa --- /dev/null +++ b/server/core/AppLogger.js @@ -0,0 +1,49 @@ +const fs = require('fs-extra'); +const Logger = require('./Logger'); + +let instance = null; + +//singleton +class AppLogger { + constructor() { + if (!instance) { + instance = this; + } + + this.inited = false; + return instance; + } + + async init(config) { + if (this.inited) + throw new Error('already inited'); + + let loggerParams = null; + + if (config.loggingEnabled) { + await fs.ensureDir(config.logDir); + loggerParams = [ + {log: 'ConsoleLog'}, + {log: 'FileLog', fileName: `${config.logDir}/${config.name}.log`}, + ]; + } + + this._logger = new Logger(loggerParams); + + this.inited = true; + return this.logger; + } + + get logger() { + if (!this.inited) + throw new Error('not inited'); + return this._logger; + } + + get log() { + const l = this.logger; + return l.log.bind(l); + } +} + +module.exports = AppLogger; diff --git a/server/core/FileDecompressor.js b/server/core/FileDecompressor.js index 95a4a46b..ad4cf7b8 100644 --- a/server/core/FileDecompressor.js +++ b/server/core/FileDecompressor.js @@ -1,10 +1,9 @@ const fs = require('fs-extra'); const zlib = require('zlib'); -const crypto = require('crypto'); const path = require('path'); const unbzip2Stream = require('unbzip2-stream'); const tar = require('tar-fs'); -const AdmZip = require('adm-zip'); +const ZipStreamer = require('./ZipStreamer'); const utils = require('./utils'); const FileDetector = require('./FileDetector'); @@ -112,18 +111,8 @@ class FileDecompressor { } async unZip(filename, outputDir) { - return new Promise((resolve) => { - const files = []; - const zip = new AdmZip(filename); - - zip.getEntries().forEach(function(zipEntry) { - files.push({path: zipEntry.entryName, size: zipEntry.header.size}); - }); - - zip.extractAllTo(outputDir, true); - - resolve(files); - }); + const zip = new ZipStreamer(); + return await await zip.unpack(filename, outputDir); } unBz2(filename, outputDir) { @@ -163,7 +152,7 @@ class FileDecompressor { } decompressByStream(stream, filename, outputDir) { - return new Promise(async(resolve, reject) => { + return new Promise((resolve, reject) => { (async() => { const file = {path: path.parse(filename).name}; let outFilename = `${outputDir}/${file.path}`; if (await fs.pathExists(outFilename)) { @@ -183,20 +172,12 @@ class FileDecompressor { resolve([file]); }); - stream.on('error', (err) => { - reject(err); - }); - - inputStream.on('error', (err) => { - reject(err); - }); - - outputStream.on('error', (err) => { - reject(err); - }); + stream.on('error', reject); + inputStream.on('error', reject); + outputStream.on('error', reject); inputStream.pipe(stream).pipe(outputStream); - }); + })().catch(reject); }); } async gzipBuffer(buf) { @@ -208,15 +189,26 @@ class FileDecompressor { }); } - async gzipFileIfNotExists(filename, outDir) { - const buf = await fs.readFile(filename); + async gzipFile(inputFile, outputFile) { + return new Promise((resolve, reject) => { + const gzip = zlib.createGzip({level: 1}); + const input = fs.createReadStream(inputFile); + const output = fs.createWriteStream(outputFile); - const hash = crypto.createHash('sha256').update(buf).digest('hex'); + input.pipe(gzip).pipe(output).on('finish', (err) => { + if (err) reject(err); + else resolve(); + }); + }); + } + + async gzipFileIfNotExists(filename, outDir) { + const hash = await utils.getFileHash(filename, 'sha256', 'hex'); const outFilename = `${outDir}/${hash}`; if (!await fs.pathExists(outFilename)) { - await fs.writeFile(outFilename, await this.gzipBuffer(buf)) + await this.gzipFile(filename, outFilename); } else { await utils.touchFile(outFilename); } diff --git a/server/core/LibSharedStorage/MegaStorage.js b/server/core/LibSharedStorage/MegaStorage.js new file mode 100644 index 00000000..c59b13d6 --- /dev/null +++ b/server/core/LibSharedStorage/MegaStorage.js @@ -0,0 +1,209 @@ +const _ = require('lodash'); +const fs = require('fs-extra'); +const path = require('path'); + +const log = new (require('../AppLogger'))().log;//singleton +const ZipStreamer = require('../ZipStreamer'); + +const utils = require('../utils'); + +const zeroStats = { + zipFilesCount: 0, + descFilesCount: 0, + zipFilesSize: 0, + descFilesSize: 0, +}; + +let instance = null; + +//singleton +class MegaStorage { + constructor() { + if (!instance) { + this.inited = false; + + this.debouncedSaveStats = _.debounce(() => { + this.saveStats().catch((e) => { + log(LM_ERR, `MegaStorage::saveStats ${e.message}`); + //process.exit(1); + }); + }, 5000, {'maxWait':6000}); + + process.on('exit', () => { + this.saveStatsSync(); + }); + + + instance = this; + } + + return instance; + } + + async init(config) { + this.config = config; + this.megaStorageDir = config.megaStorageDir; + this.statsPath = `${this.megaStorageDir}/stats.json`; + this.compressLevel = (config.compressLevel ? config.compressLevel : 4); + await fs.ensureDir(this.megaStorageDir); + + this.readingFiles = false; + this.stats = _.cloneDeep(zeroStats); + + if (await fs.pathExists(this.statsPath)) { + this.stats = Object.assign({}, + this.stats, + JSON.parse(await fs.readFile(this.statsPath, 'utf8')) + ); + } + + this.inited = true; + } + + async nameHash(filename) { + if (!this.inited) + throw new Error('not inited'); + const hash = utils.toBase36(await utils.getFileHash(filename, 'sha1')); + const hashPath = `${hash.substr(0, 2)}/${hash.substr(2, 2)}/${hash}`; + const fullHashPath = `${this.megaStorageDir}/${hashPath}`; + return { + filename, + hash, + hashPath, + fullHashPath, + zipPath: `${fullHashPath}.zip`, + descPath: `${fullHashPath}.desc`, + }; + } + + async checkFileExists(nameHash) { + return await fs.pathExists(nameHash.zipPath); + } + + async addFile(nameHash, desc = null, force = false) { + if (!this.inited) + throw new Error('not inited'); + if (await this.checkFileExists(nameHash) && !force) + return false; + + await fs.ensureDir(path.dirname(nameHash.zipPath)); + let oldZipSize = 0; + let newZipCount = 1; + if (await fs.pathExists(nameHash.zipPath)) { + oldZipSize = (await fs.stat(nameHash.zipPath)).size; + newZipCount = 0; + } + + const zip = new ZipStreamer(); + let entry = {}; + let resultFile = await zip.pack(nameHash.zipPath, [nameHash.filename], {zlib: {level: this.compressLevel}}, (ent) => { + entry = ent; + }); + + if (desc) { + desc = Object.assign({}, desc, {fileSize: entry.size, zipFileSize: resultFile.size}); + await this.updateDesc(nameHash, desc); + } + + this.stats.zipFilesSize += -oldZipSize + resultFile.size; + this.stats.zipFilesCount += newZipCount; + this.needSaveStats = true; + + this.debouncedSaveStats(); + return desc; + } + + async updateDesc(nameHash, desc) { + let oldDescSize = 0; + let newDescCount = 1; + if (await fs.pathExists(nameHash.descPath)) { + oldDescSize = (await fs.stat(nameHash.descPath)).size; + newDescCount = 0; + } + + const data = JSON.stringify(desc, null, 2); + await fs.writeFile(nameHash.descPath, data); + + this.stats.descFilesSize += -oldDescSize + data.length; + this.stats.descFilesCount += newDescCount; + this.needSaveStats = true; + + this.debouncedSaveStats(); + } + + async _findFiles(callback, dir) { + if (!callback || !this.readingFiles) + return; + + let result = true; + const files = await fs.readdir(dir, { withFileTypes: true }); + for (const file of files) { + if (!this.readingFiles) + return; + const found = path.resolve(dir, file.name); + if (file.isDirectory()) + result = await this._findFiles(callback, found); + else + await callback(found); + } + return result; + } + + async startFindFiles(callback) { + if (!this.inited) + throw new Error('not inited'); + this.readingFiles = true; + try { + return await this._findFiles(callback, this.megaStorageDir); + } finally { + this.readingFiles = false; + } + } + + async stopFindFiles() { + this.readingFiles = false; + } + + async saveStats() { + if (this.needSaveStats) { + await fs.writeFile(this.statsPath, JSON.stringify(this.stats, null, 2)); + this.needSaveStats = false; + } + } + + saveStatsSync() { + if (this.needSaveStats) { + fs.writeFileSync(this.statsPath, JSON.stringify(this.stats, null, 2)); + this.needSaveStats = false; + } + } + + async getStats(gather = false) { + if (!this.inited) + throw new Error('MegaStorage::not inited'); + if (!gather || this.readingFiles) + return this.stats; + + let stats = _.cloneDeep(zeroStats); + const result = await this.startFindFiles(async(entry) => { + if (path.extname(entry) == '.zip') { + stats.zipFilesSize += (await fs.stat(entry)).size; + stats.zipFilesCount++; + } + + if (path.extname(entry) == '.desc') { + stats.descFilesSize += (await fs.stat(entry)).size; + stats.descFilesCount++; + } + }); + + if (result) { + this.stats = stats; + this.needSaveStats = true; + this.debouncedSaveStats(); + } + return this.stats; + } +} + +module.exports = MegaStorage; \ No newline at end of file diff --git a/server/core/LibSharedStorage/index.js b/server/core/LibSharedStorage/index.js new file mode 100644 index 00000000..6cf772b6 --- /dev/null +++ b/server/core/LibSharedStorage/index.js @@ -0,0 +1,4 @@ +class LibSharedStorage { +} + +module.exports = LibSharedStorage; \ No newline at end of file diff --git a/server/core/BookConverter/ConvertBase.js b/server/core/Reader/BookConverter/ConvertBase.js similarity index 99% rename from server/core/BookConverter/ConvertBase.js rename to server/core/Reader/BookConverter/ConvertBase.js index e5abd389..3a0e6a0e 100644 --- a/server/core/BookConverter/ConvertBase.js +++ b/server/core/Reader/BookConverter/ConvertBase.js @@ -4,7 +4,7 @@ const chardet = require('chardet'); const he = require('he'); const textUtils = require('./textUtils'); -const utils = require('../utils'); +const utils = require('../../utils'); let execConverterCounter = 0; diff --git a/server/core/BookConverter/ConvertDoc.js b/server/core/Reader/BookConverter/ConvertDoc.js similarity index 100% rename from server/core/BookConverter/ConvertDoc.js rename to server/core/Reader/BookConverter/ConvertDoc.js diff --git a/server/core/BookConverter/ConvertDocX.js b/server/core/Reader/BookConverter/ConvertDocX.js similarity index 100% rename from server/core/BookConverter/ConvertDocX.js rename to server/core/Reader/BookConverter/ConvertDocX.js diff --git a/server/core/BookConverter/ConvertEpub.js b/server/core/Reader/BookConverter/ConvertEpub.js similarity index 100% rename from server/core/BookConverter/ConvertEpub.js rename to server/core/Reader/BookConverter/ConvertEpub.js diff --git a/server/core/BookConverter/ConvertFb2.js b/server/core/Reader/BookConverter/ConvertFb2.js similarity index 100% rename from server/core/BookConverter/ConvertFb2.js rename to server/core/Reader/BookConverter/ConvertFb2.js diff --git a/server/core/BookConverter/ConvertHtml.js b/server/core/Reader/BookConverter/ConvertHtml.js similarity index 99% rename from server/core/BookConverter/ConvertHtml.js rename to server/core/Reader/BookConverter/ConvertHtml.js index e47e23ea..60d5433f 100644 --- a/server/core/BookConverter/ConvertHtml.js +++ b/server/core/Reader/BookConverter/ConvertHtml.js @@ -1,5 +1,5 @@ const ConvertBase = require('./ConvertBase'); -const sax = require('./sax'); +const sax = require('../../sax'); const textUtils = require('./textUtils'); class ConvertHtml extends ConvertBase { diff --git a/server/core/BookConverter/ConvertMobi.js b/server/core/Reader/BookConverter/ConvertMobi.js similarity index 100% rename from server/core/BookConverter/ConvertMobi.js rename to server/core/Reader/BookConverter/ConvertMobi.js diff --git a/server/core/BookConverter/ConvertPdf.js b/server/core/Reader/BookConverter/ConvertPdf.js similarity index 99% rename from server/core/BookConverter/ConvertPdf.js rename to server/core/Reader/BookConverter/ConvertPdf.js index 77f1142c..be885369 100644 --- a/server/core/BookConverter/ConvertPdf.js +++ b/server/core/Reader/BookConverter/ConvertPdf.js @@ -1,8 +1,8 @@ const fs = require('fs-extra'); const path = require('path'); -const sax = require('./sax'); -const utils = require('../utils'); +const sax = require('../../sax'); +const utils = require('../../utils'); const ConvertHtml = require('./ConvertHtml'); class ConvertPdf extends ConvertHtml { diff --git a/server/core/BookConverter/ConvertRtf.js b/server/core/Reader/BookConverter/ConvertRtf.js similarity index 100% rename from server/core/BookConverter/ConvertRtf.js rename to server/core/Reader/BookConverter/ConvertRtf.js diff --git a/server/core/BookConverter/ConvertSamlib.js b/server/core/Reader/BookConverter/ConvertSamlib.js similarity index 99% rename from server/core/BookConverter/ConvertSamlib.js rename to server/core/Reader/BookConverter/ConvertSamlib.js index a690ade7..5b095c07 100644 --- a/server/core/BookConverter/ConvertSamlib.js +++ b/server/core/Reader/BookConverter/ConvertSamlib.js @@ -1,7 +1,7 @@ const _ = require('lodash'); const URL = require('url').URL; -const sax = require('./sax'); +const sax = require('../../sax'); const ConvertBase = require('./ConvertBase'); class ConvertSamlib extends ConvertBase { diff --git a/server/core/BookConverter/ConvertSites.js b/server/core/Reader/BookConverter/ConvertSites.js similarity index 100% rename from server/core/BookConverter/ConvertSites.js rename to server/core/Reader/BookConverter/ConvertSites.js diff --git a/server/core/BookConverter/index.js b/server/core/Reader/BookConverter/index.js similarity index 97% rename from server/core/BookConverter/index.js rename to server/core/Reader/BookConverter/index.js index 66657a25..ebb4c1da 100644 --- a/server/core/BookConverter/index.js +++ b/server/core/Reader/BookConverter/index.js @@ -1,5 +1,5 @@ const fs = require('fs-extra'); -const FileDetector = require('../FileDetector'); +const FileDetector = require('../../FileDetector'); //порядок важен const convertClassFactory = [ diff --git a/server/core/BookConverter/textUtils.js b/server/core/Reader/BookConverter/textUtils.js similarity index 100% rename from server/core/BookConverter/textUtils.js rename to server/core/Reader/BookConverter/textUtils.js diff --git a/server/core/readerStorage.js b/server/core/Reader/ReaderStorage.js similarity index 89% rename from server/core/readerStorage.js rename to server/core/Reader/ReaderStorage.js index adb88870..d3828102 100644 --- a/server/core/readerStorage.js +++ b/server/core/Reader/ReaderStorage.js @@ -1,12 +1,22 @@ const SQL = require('sql-template-strings'); const _ = require('lodash'); -const connManager = require('../db/connManager'); +const ConnManager = require('../../db/ConnManager');//singleton +let instance = null; + +//singleton class ReaderStorage { constructor() { - this.storagePool = connManager.pool.readerStorage; - this.periodicCleanCache(3*3600*1000);//1 раз в 3 часа + if (!instance) { + this.connManager = new ConnManager(); + this.storagePool = this.connManager.pool.readerStorage; + this.periodicCleanCache(3*3600*1000);//1 раз в 3 часа + + instance = this; + } + + return instance; } async doAction(act) { @@ -113,6 +123,4 @@ class ReaderStorage { } } -const readerStorage = new ReaderStorage(); - -module.exports = readerStorage; \ No newline at end of file +module.exports = ReaderStorage; \ No newline at end of file diff --git a/server/core/ReaderWorker.js b/server/core/Reader/ReaderWorker.js similarity index 82% rename from server/core/ReaderWorker.js rename to server/core/Reader/ReaderWorker.js index 24038693..ac2ea681 100644 --- a/server/core/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -1,35 +1,40 @@ const fs = require('fs-extra'); const path = require('path'); -const crypto = require('crypto'); -const workerState = require('./workerState'); -const FileDownloader = require('./FileDownloader'); -const FileDecompressor = require('./FileDecompressor'); +const WorkerState = require('../WorkerState');//singleton +const FileDownloader = require('../FileDownloader'); +const FileDecompressor = require('../FileDecompressor'); const BookConverter = require('./BookConverter'); -const utils = require('./utils'); -const log = require('./getLogger').getLog(); -let singleCleanExecute = false; +const utils = require('../utils'); +const log = new (require('../AppLogger'))().log;//singleton +let instance = null; + +//singleton class ReaderWorker { constructor(config) { - this.config = Object.assign({}, config); - - this.config.tempDownloadDir = `${config.tempDir}/download`; - fs.ensureDirSync(this.config.tempDownloadDir); + if (!instance) { + this.config = Object.assign({}, config); + + this.config.tempDownloadDir = `${config.tempDir}/download`; + fs.ensureDirSync(this.config.tempDownloadDir); - this.config.tempPublicDir = `${config.publicDir}/tmp`; - fs.ensureDirSync(this.config.tempPublicDir); + this.config.tempPublicDir = `${config.publicDir}/tmp`; + fs.ensureDirSync(this.config.tempPublicDir); - this.down = new FileDownloader(); - this.decomp = new FileDecompressor(); - this.bookConverter = new BookConverter(this.config); + this.workerState = new WorkerState(); + this.down = new FileDownloader(); + this.decomp = new FileDecompressor(); + this.bookConverter = new BookConverter(this.config); - if (!singleCleanExecute) { this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, 60*60*1000);//1 раз в час - singleCleanExecute = true; + + instance = this; } + + return instance; } async loadBook(opts, wState) { @@ -107,8 +112,8 @@ class ReaderWorker { } loadBookUrl(opts) { - const workerId = workerState.generateWorkerId(); - const wState = workerState.getControl(workerId); + const workerId = this.workerState.generateWorkerId(); + const wState = this.workerState.getControl(workerId); wState.set({state: 'start'}); this.loadBook(opts, wState); @@ -117,10 +122,7 @@ class ReaderWorker { } async saveFile(file) { - const buf = await fs.readFile(file.path); - - const hash = crypto.createHash('sha256').update(buf).digest('hex'); - + const hash = await utils.getFileHash(file.path, 'sha256', 'hex'); const outFilename = `${this.config.uploadDir}/${hash}`; if (!await fs.pathExists(outFilename)) { diff --git a/server/core/workerState.js b/server/core/WorkerState.js similarity index 85% rename from server/core/workerState.js rename to server/core/WorkerState.js index b9526d27..a4c0c1b0 100644 --- a/server/core/workerState.js +++ b/server/core/WorkerState.js @@ -3,10 +3,18 @@ const utils = require('./utils'); const cleanInterval = 3600; //sec const cleanAfterLastModified = cleanInterval - 60; //sec +let instance = null; + +//singleton class WorkerState { constructor() { - this.states = {}; - setTimeout(this.cleanStates.bind(this), cleanInterval*1000); + if (!instance) { + this.states = {}; + this.cleanStates(); + instance = this; + } + + return instance; } generateWorkerId() { @@ -51,6 +59,4 @@ class WorkerState { } } -const workerState = new WorkerState(); - -module.exports = workerState; \ No newline at end of file +module.exports = WorkerState; \ No newline at end of file diff --git a/server/core/ZipStreamer.js b/server/core/ZipStreamer.js new file mode 100644 index 00000000..50eec0c7 --- /dev/null +++ b/server/core/ZipStreamer.js @@ -0,0 +1,78 @@ +const fs = require('fs-extra'); +const path = require('path'); + +const zipStream = require('zip-stream'); +const unzipStream = require('node-stream-zip'); + +class ZipStreamer { + constructor() { + } + + //TODO: сделать рекурсивный обход директорий, пока только файлы + //files = ['filename', 'dirname/'] + pack(zipFile, files, options, entryCallback) { + return new Promise((resolve, reject) => { (async() => { + entryCallback = (entryCallback ? entryCallback : () => {}); + const zip = new zipStream(options); + + const outputStream = fs.createWriteStream(zipFile); + + outputStream.on('error', reject); + outputStream.on('finish', async() => { + let file = {path: zipFile}; + try { + file.size = (await fs.stat(zipFile)).size; + } catch (e) { + reject(e); + } + resolve(file); + }); + + zip.on('error', reject); + zip.pipe(outputStream); + + const zipAddEntry = (filename) => { + return new Promise((resolve, reject) => { + const basename = path.basename(filename); + const source = fs.createReadStream(filename); + + zip.entry(source, {name: basename}, (err, entry) => { + if (err) reject(err); + resolve(entry); + }); + }); + }; + + for (const filename of files) { + const entry = await zipAddEntry(filename); + entryCallback({path: entry.name, size: entry.size, compressedSize: entry.csize}); + } + + zip.finish(); + })().catch(reject); }); + } + + unpack(zipFile, outputDir, entryCallback) { + return new Promise((resolve, reject) => { + entryCallback = (entryCallback ? entryCallback : () => {}); + const unzip = new unzipStream({file: zipFile}); + + let files = []; + unzip.on('extract', (en) => { + const entry = {path: en.name, size: en.size, compressedSize: en.compressedSize}; + entryCallback(entry); + files.push(entry); + }); + + unzip.on('ready', () => { + unzip.extract(null, outputDir, (err) => { + if (err) reject(err); + unzip.close(); + resolve(files); + }); + }); + }); + } +} + +module.exports = ZipStreamer; \ No newline at end of file diff --git a/server/core/getLogger.js b/server/core/getLogger.js deleted file mode 100644 index ed17b6fd..00000000 --- a/server/core/getLogger.js +++ /dev/null @@ -1,40 +0,0 @@ -const fs = require('fs-extra'); -const Logger = require('./Logger'); - -let logger = null; - -function initLogger(config) { - if (logger) - logger.close(); - - let loggerParams = null; - - if (config.loggingEnabled) { - fs.ensureDirSync(config.logDir); - loggerParams = [ - {log: 'ConsoleLog'}, - {log: 'FileLog', fileName: `${config.logDir}/${config.name}.log`}, - ]; - } - - logger = new Logger(loggerParams); - - return logger; -} - -function getLogger() { - if (logger) - return logger; - throw new Error('getLogger error: logger not initialized'); -} - -function getLog() { - const l = getLogger(); - return l.log.bind(l); -} - -module.exports = { - initLogger, - getLogger, - getLog, -}; \ No newline at end of file diff --git a/server/core/BookConverter/sax.js b/server/core/sax.js similarity index 100% rename from server/core/BookConverter/sax.js rename to server/core/sax.js diff --git a/server/core/utils.js b/server/core/utils.js index b134cf79..9591c7d7 100644 --- a/server/core/utils.js +++ b/server/core/utils.js @@ -1,6 +1,28 @@ const { spawn } = require('child_process'); const fs = require('fs-extra'); const crypto = require('crypto'); +const baseX = require('base-x'); + +const BASE36 = '0123456789abcdefghijklmnopqrstuvwxyz'; +const bs36 = baseX(BASE36); + +function toBase36(data) { + return bs36.encode(Buffer.from(data)); +} + +function fromBase36(data) { + return bs36.decode(data); +} + +function getFileHash(filename, hashName, enc) { + return new Promise((resolve, reject) => { + const hash = crypto.createHash(hashName); + const rs = fs.createReadStream(filename); + rs.on('error', reject); + rs.on('data', chunk => hash.update(chunk)); + rs.on('end', () => resolve(hash.digest(enc))); + }); +} function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); @@ -20,7 +42,7 @@ function spawnProcess(cmd, opts) { onData = (onData ? onData : () => {}); args = (args ? args : []); - return new Promise(async(resolve, reject) => { + return new Promise((resolve, reject) => { (async() => { let resolved = false; const proc = spawn(cmd, args, {detached: true}); @@ -50,10 +72,13 @@ function spawnProcess(cmd, opts) { process.kill(proc.pid); reject({status: 'killed', stdout, stderr}); } - }); + })().catch(reject); }); } module.exports = { + toBase36, + fromBase36, + getFileHash, sleep, randomHexString, touchFile, diff --git a/server/db/connManager.js b/server/db/ConnManager.js similarity index 83% rename from server/db/connManager.js rename to server/db/ConnManager.js index e5fbf3f9..5576fa47 100644 --- a/server/db/connManager.js +++ b/server/db/ConnManager.js @@ -1,20 +1,30 @@ const fs = require('fs-extra'); const SqliteConnectionPool = require('./SqliteConnectionPool'); -const log = require('../core/getLogger').getLog(); +const log = new (require('../core/AppLogger'))().log;//singleton const migrations = { 'app': require('./migrations/app'), 'readerStorage': require('./migrations/readerStorage'), }; +let instance = null; + +//singleton class ConnManager { constructor() { - this._pool = {}; + if (!instance) { + this.inited = false; + + instance = this; + } + + return instance; } async init(config) { this.config = config; + this._pool = {}; const force = null;//(config.branch == 'development' ? 'last' : null); @@ -39,6 +49,7 @@ class ConnManager { this._pool[poolConfig.poolName] = connPool; } + this.inited = true; } get pool() { @@ -46,6 +57,4 @@ class ConnManager { } } -const connManager = new ConnManager(); - -module.exports = connManager; \ No newline at end of file +module.exports = ConnManager; \ No newline at end of file diff --git a/server/dev.js b/server/dev.js index 7782d385..1ae2dfbd 100644 --- a/server/dev.js +++ b/server/dev.js @@ -1,4 +1,4 @@ -const log = require('./core/getLogger').getLog(); +const log = new (require('./core/AppLogger'))().log;//singleton function webpackDevMiddleware(app) { const webpack = require('webpack'); diff --git a/server/index.js b/server/index.js index 6006793a..73563630 100644 --- a/server/index.js +++ b/server/index.js @@ -1,21 +1,30 @@ -const config = require('./config'); -const logger = require('./core/getLogger'); -logger.initLogger(config); -const log = logger.getLog(); - -const configSaver = require('./config/configSaver'); -const argv = require('minimist')(process.argv.slice(2)); - const fs = require('fs-extra'); const path = require('path'); +const argv = require('minimist')(process.argv.slice(2)); const express = require('express'); const compression = require('compression'); -const connManager = require('./db/connManager'); - async function init() { + //config + const configManager = new (require('./config'))();//singleton + await configManager.init(); + configManager.userConfigFile = argv.config; + await configManager.load(); + const config = configManager.config; + + //logger + const appLogger = new (require('./core/AppLogger'))();//singleton + await appLogger.init(config); + const log = appLogger.log; + + //dirs + log(`${config.name} v${config.version}`); + log('Initializing'); + await fs.ensureDir(config.dataDir); await fs.ensureDir(config.uploadDir); + await fs.ensureDir(config.sharedDir); + await fs.ensureDir(config.tempDir); await fs.emptyDir(config.tempDir); @@ -26,16 +35,14 @@ async function init() { await fs.move(appNewDir, appDir); } - //загружаем конфиг из файла - await configSaver.load(config, argv.config); + //connections + const connManager = new (require('./db/ConnManager'))();//singleton + await connManager.init(config); } -async function main() { - log(`${config.name} v${config.version}`); - log('Initializing'); - await init(); - - await connManager.init(config); +async function main() { + const log = new (require('./core/AppLogger'))().log;//singleton + const config = new (require('./config'))().config;//singleton //servers for (let server of config.servers) { @@ -86,6 +93,7 @@ async function main() { (async() => { try { + await init(); await main(); } catch (e) { console.error(e);