diff --git a/build/linux.js b/build/linux.js index 8e0e3b1d..70dc4232 100644 --- a/build/linux.js +++ b/build/linux.js @@ -5,8 +5,7 @@ const stream = require('stream'); const pipeline = util.promisify(stream.pipeline); const got = require('got'); -const decompress = require('decompress'); -const decompressTargz = require('decompress-targz'); +const FileDecompressor = require('../server/core/FileDecompressor'); const distDir = path.resolve(__dirname, '../dist'); const publicDir = `${distDir}/tmp/public`; @@ -15,6 +14,8 @@ const outDir = `${distDir}/linux`; const tempDownloadDir = `${distDir}/tmp/download`; async function main() { + const decomp = new FileDecompressor(); + await fs.emptyDir(outDir); // перемещаем public на место if (await fs.pathExists(publicDir)) @@ -32,11 +33,7 @@ async function main() { console.log(`done downloading ${sqliteRemoteUrl}`); //распаковываем - await decompress(`${tempDownloadDir}/sqlite.tar.gz`, `${tempDownloadDir}`, { - plugins: [ - decompressTargz() - ] - }); + console.log(await decomp.unpackTarZZ(`${tempDownloadDir}/sqlite.tar.gz`, tempDownloadDir)); console.log('files decompressed'); } // копируем в дистрибутив @@ -53,11 +50,7 @@ async function main() { console.log(`done downloading ${ipfsRemoteUrl}`); //распаковываем - await decompress(`${tempDownloadDir}/ipfs.tar.gz`, `${tempDownloadDir}`, { - plugins: [ - decompressTargz() - ] - }); + console.log(await decomp.unpackTarZZ(`${tempDownloadDir}/ipfs.tar.gz`, tempDownloadDir)); console.log('files decompressed'); } diff --git a/build/win.js b/build/win.js index d47dfc8c..569d3f10 100644 --- a/build/win.js +++ b/build/win.js @@ -5,8 +5,7 @@ const stream = require('stream'); const pipeline = util.promisify(stream.pipeline); const got = require('got'); -const decompress = require('decompress'); -const decompressTargz = require('decompress-targz'); +const FileDecompressor = require('../server/core/FileDecompressor'); const distDir = path.resolve(__dirname, '../dist'); const publicDir = `${distDir}/tmp/public`; @@ -15,6 +14,8 @@ const outDir = `${distDir}/win`; const tempDownloadDir = `${distDir}/tmp/download`; async function main() { + const decomp = new FileDecompressor(); + await fs.emptyDir(outDir); // перемещаем public на место if (await fs.pathExists(publicDir)) @@ -32,11 +33,7 @@ async function main() { console.log(`done downloading ${sqliteRemoteUrl}`); //распаковываем - await decompress(`${tempDownloadDir}/sqlite.tar.gz`, `${tempDownloadDir}`, { - plugins: [ - decompressTargz() - ] - }); + console.log(await decomp.unpackTarZZ(`${tempDownloadDir}/sqlite.tar.gz`, tempDownloadDir)); console.log('files decompressed'); } // копируем в дистрибутив @@ -53,7 +50,7 @@ async function main() { console.log(`done downloading ${ipfsRemoteUrl}`); //распаковываем - await decompress(`${tempDownloadDir}/ipfs.zip`, `${tempDownloadDir}`); + console.log(await decomp.unpack(`${tempDownloadDir}/ipfs.zip`, tempDownloadDir)); console.log('files decompressed'); } // копируем в дистрибутив diff --git a/client/api/reader.js b/client/api/reader.js index 071a9689..c89ccec0 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -49,8 +49,9 @@ class Reader { } //проверка воркера const prevProgress = response.data.progress; + const prevState = response.data.state; response = await workerApi.post('/get-state', {workerId}); - i = (prevProgress != response.data.progress ? 1 : i); + i = (prevProgress != response.data.progress || prevState != response.data.state ? 1 : i); } } diff --git a/client/components/Reader/HelpPage/CommonHelpPage/CommonHelpPage.vue b/client/components/Reader/HelpPage/CommonHelpPage/CommonHelpPage.vue index 5013c3df..edf0e182 100644 --- a/client/components/Reader/HelpPage/CommonHelpPage/CommonHelpPage.vue +++ b/client/components/Reader/HelpPage/CommonHelpPage/CommonHelpPage.vue @@ -19,7 +19,7 @@

В качестве URL можно задавать html-страничку с книгой, либо прямую ссылку на файл из онлайн-библиотеки (например, скопировав адрес ссылки или кнопки "скачать fb2").

-

Поддерживаемые форматы: html, txt, fb2, fb2.zip

+

Поддерживаемые форматы: fb2, fb2.zip, html, txt и другие

Связаться с разработчиком: bookpauk@gmail.com

diff --git a/client/components/Reader/LoaderPage/LoaderPage.vue b/client/components/Reader/LoaderPage/LoaderPage.vue index 58515f5d..684e0e73 100644 --- a/client/components/Reader/LoaderPage/LoaderPage.vue +++ b/client/components/Reader/LoaderPage/LoaderPage.vue @@ -3,9 +3,8 @@
{{ title }} Добро пожаловать! - Поддерживаются форматы: fb2, html, txt - ...а также: rtf, doc, docx, и вскоре: pdf, epub, mobi - ...и распознается сжатие: zip, bz2, gz + Поддерживаются форматы: fb2, html, txt и сжатие: zip, bz2, gz + ...а также: rtf, doc, docx, pdf, epub, mobi
diff --git a/client/components/Reader/share/bookManager.js b/client/components/Reader/share/bookManager.js index bc954965..a1bfa326 100644 --- a/client/components/Reader/share/bookManager.js +++ b/client/components/Reader/share/bookManager.js @@ -5,10 +5,6 @@ import BookParser from './BookParser'; const maxDataSize = 500*1024*1024;//chars, not bytes -const bmCacheStore = localForage.createInstance({ - name: 'bmCacheStore' -}); - const bmMetaStore = localForage.createInstance({ name: 'bmMetaStore' }); @@ -21,15 +17,22 @@ const bmRecentStore = localForage.createInstance({ name: 'bmRecentStore' }); +const bmCacheStore = localForage.createInstance({ + name: 'bmCacheStore' +}); + class BookManager { async init(settings) { this.settings = settings; - //this.booksCached нужен только для ускорения загрузки читалки this.booksCached = await bmCacheStore.getItem('books'); if (!this.booksCached) this.booksCached = {}; this.recent = await bmCacheStore.getItem('recent'); + this.recentLast = await bmCacheStore.getItem('recent-last'); + if (this.recentLast) + this.recent[this.recentLast.key] = this.recentLast; + this.books = Object.assign({}, this.booksCached); this.recentChanged1 = true; @@ -44,7 +47,9 @@ class BookManager { } } - + //долгая загрузка из хранилища + //bmMetaStore и bmRecentStore в будущем можно будет убрать + //bmCacheStore достаточно async loadMeta(immediate) { if (!immediate) await utils.sleep(2000); @@ -57,7 +62,13 @@ class BookManager { if (keySplit.length == 2 && keySplit[0] == 'bmMeta') { let meta = await bmMetaStore.getItem(key); + const oldBook = this.books[meta.key]; this.books[meta.key] = meta; + + if (oldBook && oldBook.data && oldBook.parsed) { + this.books[meta.key].data = oldBook.data; + this.books[meta.key].parsed = oldBook.parsed; + } } } @@ -69,12 +80,14 @@ class BookManager { } await this.cleanBooks(); + await this.cleanRecentBooks(); this.booksCached = {}; for (const key in this.books) { this.booksCached[key] = this.metaOnly(this.books[key]); } await bmCacheStore.setItem('books', this.booksCached); + await bmCacheStore.setItem('recent', this.recent); } async cleanBooks() { @@ -205,8 +218,13 @@ class BookManager { this.recent[result.key] = result; await bmRecentStore.setItem(result.key, result); - await this.cleanRecentBooks(); - await bmCacheStore.setItem('recent', this.recent); + + //кэшируем, аккуратно + if (!(this.recentLast && this.recentLast.key == result.key)) { + await bmCacheStore.setItem('recent', this.recent); + } + this.recentLast = result; + await bmCacheStore.setItem('recent-last', this.recentLast); this.recentChanged1 = true; this.recentChanged2 = true; diff --git a/package-lock.json b/package-lock.json index 97c61806..bb360b8f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.4.7", + "version": "0.5.2", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -1597,9 +1597,9 @@ } }, "base64-js": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-0.0.8.tgz", - "integrity": "sha1-EQHpVE9KdrG8OybUUsqW16NeeXg=" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", + "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==" }, "bcrypt-pbkdf": { "version": "1.0.2", @@ -1622,12 +1622,23 @@ "dev": true }, "bl": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", - "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz", + "integrity": "sha512-EUAyP5UHU5hxF8BPT0LKW8gjYLhq1DQIcneOX/pL/m2Alo+OYDQAJlHq+yseMP50Os2nHXOSic6Ss3vSQeyf4A==", "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" + "readable-stream": "^3.0.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.2.0.tgz", + "integrity": "sha512-RV20kLjdmpZuTF1INEb9IA3L68Nmi+Ri7ppZqo78wj//Pn62fCoJyV9zalccNzDD/OuJpMG4f+pfMl8+L6QdGw==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } } }, "bluebird": { @@ -1809,39 +1820,14 @@ } }, "buffer": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-3.6.0.tgz", - "integrity": "sha1-pyyTb3e5a/UvX357RnGAYoVR3vs=", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.2.1.tgz", + "integrity": "sha512-c+Ko0loDaFfuPWiL02ls9Xd3GO3cPVmUobQ6t3rXNUk304u6hGq+8N/kFi+QEIKhzK3uwolVhLzszmfLmMLnqg==", "requires": { - "base64-js": "0.0.8", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" } }, - "buffer-alloc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", - "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", - "requires": { - "buffer-alloc-unsafe": "^1.1.0", - "buffer-fill": "^1.0.0" - } - }, - "buffer-alloc-unsafe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", - "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==" - }, - "buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" - }, - "buffer-fill": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", - "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=" - }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -2354,14 +2340,6 @@ "delayed-stream": "~1.0.0" } }, - "commander": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", - "integrity": "sha1-Br42f+v9oMMwqh4qBy09yXYkJdQ=", - "requires": { - "graceful-readlink": ">= 1.0.0" - } - }, "commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", @@ -3080,45 +3058,6 @@ "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", "dev": true }, - "decompress": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/decompress/-/decompress-4.2.0.tgz", - "integrity": "sha1-eu3YVCflqS2s/lVnSnxQXpbQH50=", - "requires": { - "decompress-tar": "^4.0.0", - "decompress-tarbz2": "^4.0.0", - "decompress-targz": "^4.0.0", - "decompress-unzip": "^4.0.1", - "graceful-fs": "^4.1.10", - "make-dir": "^1.0.0", - "pify": "^2.3.0", - "strip-dirs": "^2.0.0" - } - }, - "decompress-bzip2": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/decompress-bzip2/-/decompress-bzip2-4.0.0.tgz", - "integrity": "sha1-0SVMlJ4F6vYol1QoawY/3Hz/AT8=", - "requires": { - "file-type": "^4.3.0", - "seek-bzip": "^1.0.5" - }, - "dependencies": { - "file-type": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-4.4.0.tgz", - "integrity": "sha1-G2AOX8ofvcboDApwxxyNul95BsU=" - } - } - }, - "decompress-gz": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/decompress-gz/-/decompress-gz-0.0.1.tgz", - "integrity": "sha512-YMdCWdxHvPplsTbV1tvr2oFJOtAFNxqVMFnKWEmePBXl+LKG5z5bFrowzc12Jzd7O29nnzI/D1M95Asx0Qa1fg==", - "requires": { - "file-type": "^5.2.0" - } - }, "decompress-response": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", @@ -3127,63 +3066,6 @@ "mimic-response": "^1.0.0" } }, - "decompress-tar": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/decompress-tar/-/decompress-tar-4.1.1.tgz", - "integrity": "sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ==", - "requires": { - "file-type": "^5.2.0", - "is-stream": "^1.1.0", - "tar-stream": "^1.5.2" - } - }, - "decompress-tarbz2": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/decompress-tarbz2/-/decompress-tarbz2-4.1.1.tgz", - "integrity": "sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A==", - "requires": { - "decompress-tar": "^4.1.0", - "file-type": "^6.1.0", - "is-stream": "^1.1.0", - "seek-bzip": "^1.0.5", - "unbzip2-stream": "^1.0.9" - }, - "dependencies": { - "file-type": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-6.2.0.tgz", - "integrity": "sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==" - } - } - }, - "decompress-targz": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/decompress-targz/-/decompress-targz-4.1.1.tgz", - "integrity": "sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w==", - "requires": { - "decompress-tar": "^4.1.1", - "file-type": "^5.2.0", - "is-stream": "^1.1.0" - } - }, - "decompress-unzip": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/decompress-unzip/-/decompress-unzip-4.0.1.tgz", - "integrity": "sha1-3qrM39FK6vhVePczroIQ+bSEj2k=", - "requires": { - "file-type": "^3.8.0", - "get-stream": "^2.2.0", - "pify": "^2.3.0", - "yauzl": "^2.4.2" - }, - "dependencies": { - "file-type": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", - "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=" - } - } - }, "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", @@ -4172,6 +4054,43 @@ } } }, + "extract-zip": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz", + "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=", + "requires": { + "concat-stream": "1.6.2", + "debug": "2.6.9", + "mkdirp": "0.5.1", + "yauzl": "2.4.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "fd-slicer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", + "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=", + "requires": { + "pend": "~1.2.0" + } + }, + "yauzl": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", + "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", + "requires": { + "fd-slicer": "~1.0.1" + } + } + } + }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -4213,14 +4132,6 @@ "integrity": "sha512-483XLLxTVIwWK3QTrMGRqUfUpoOs/0hbQrl2oz4J0pAcm3A3bu84wxTFqGqkJzewCLdME38xJLJAxBABfQT8sQ==", "dev": true }, - "fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", - "requires": { - "pend": "~1.2.0" - } - }, "fg-loadcss": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fg-loadcss/-/fg-loadcss-2.1.0.tgz", @@ -4261,11 +4172,6 @@ "schema-utils": "^1.0.0" } }, - "file-type": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/file-type/-/file-type-5.2.0.tgz", - "integrity": "sha1-LdvqfHP/42No365J3DOMBYwritY=" - }, "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -5057,15 +4963,6 @@ "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==", "dev": true }, - "get-stream": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-2.3.1.tgz", - "integrity": "sha1-Xzj5PzRgCWZu4BUKBUFn+Rvdld4=", - "requires": { - "object-assign": "^4.0.1", - "pinkie-promise": "^2.0.0" - } - }, "get-value": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", @@ -5220,11 +5117,6 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" }, - "graceful-readlink": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", - "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=" - }, "har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -5967,11 +5859,6 @@ "is-extglob": "^2.1.1" } }, - "is-natural-number": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/is-natural-number/-/is-natural-number-4.0.1.tgz", - "integrity": "sha1-q5124dtM7VHjXeDHLr7PCfc0zeg=" - }, "is-number": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", @@ -6031,7 +5918,8 @@ "is-stream": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", + "dev": true }, "is-svg": { "version": "3.0.0", @@ -6336,6 +6224,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, "requires": { "pify": "^3.0.0" }, @@ -6343,7 +6232,8 @@ "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true } } }, @@ -7413,20 +7303,17 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, - "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" - }, "pinkie": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", - "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=", + "dev": true }, "pinkie-promise": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "dev": true, "requires": { "pinkie": "^2.0.0" } @@ -10047,14 +9934,6 @@ "ajv-keywords": "^3.1.0" } }, - "seek-bzip": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.5.tgz", - "integrity": "sha1-z+kXyz0nS8/6x5J1ivUxc+sfq9w=", - "requires": { - "commander": "~2.8.1" - } - }, "semver": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", @@ -10563,14 +10442,6 @@ "ansi-regex": "^2.0.0" } }, - "strip-dirs": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/strip-dirs/-/strip-dirs-2.1.0.tgz", - "integrity": "sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g==", - "requires": { - "is-natural-number": "^4.0.1" - } - }, "strip-eof": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", @@ -10770,18 +10641,50 @@ "yallist": "^3.0.2" } }, - "tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "tar-fs": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz", + "integrity": "sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA==", "requires": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", + "chownr": "^1.1.1", + "mkdirp": "^0.5.1", + "pump": "^3.0.0", + "tar-stream": "^2.0.0" + }, + "dependencies": { + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + } + } + }, + "tar-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.0.1.tgz", + "integrity": "sha512-I6OJF7wE62BC6zNPdHDtseK0D0187PBjbKSLYY4ffvVkBM6tyBn2O9plDvVM2229/mozfEL/X3++qSvYYQE2xw==", + "requires": { + "bl": "^3.0.0", + "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.2.0.tgz", + "integrity": "sha512-RV20kLjdmpZuTF1INEb9IA3L68Nmi+Ri7ppZqo78wj//Pn62fCoJyV9zalccNzDD/OuJpMG4f+pfMl8+L6QdGw==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } } }, "terser": { @@ -11036,11 +10939,6 @@ "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=", "dev": true }, - "to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==" - }, "to-fast-properties": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz", @@ -11201,12 +11099,12 @@ "dev": true }, "unbzip2-stream": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.1.tgz", - "integrity": "sha512-fIZnvdjblYs7Cru/xC6tCPVhz7JkYcVQQkePwMLyQELzYTds2Xn8QefPVnvdVhhZqubxNA1cASXEH5wcK0Bucw==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz", + "integrity": "sha512-fUlAF7U9Ah1Q6EieQ4x4zLNejrRvDWUYmxXUpN3uziFYCHapjWFaCAnreY9bGgxzaMCFAPPpYNng57CypwJVhg==", "requires": { - "buffer": "^3.0.1", - "through": "^2.3.6" + "buffer": "^5.2.1", + "through": "^2.3.8" } }, "union-value": { @@ -12003,15 +11901,6 @@ "camelcase": "^5.0.0", "decamelize": "^1.2.0" } - }, - "yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", - "requires": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } } } } diff --git a/package.json b/package.json index 1ba693c9..021feee9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.5.2", + "version": "0.5.3", "engines": { "node": ">=10.0.0" }, @@ -29,7 +29,6 @@ "clean-webpack-plugin": "^1.0.0", "copy-webpack-plugin": "^4.6.0", "css-loader": "^1.0.0", - "decompress-targz": "^4.1.1", "disable-output-webpack-plugin": "^1.0.1", "element-theme-chalk": "^2.4.11", "eslint": "^5.11.1", @@ -59,11 +58,9 @@ "axios": "^0.18.0", "chardet": "^0.7.0", "compression": "^1.7.3", - "decompress": "^4.2.0", - "decompress-bzip2": "^4.0.0", - "decompress-gz": "0.0.1", "element-ui": "^2.4.11", "express": "^4.16.4", + "extract-zip": "^1.6.7", "fg-loadcss": "^2.1.0", "fs-extra": "^7.0.1", "got": "^9.5.1", @@ -76,6 +73,8 @@ "path-browserify": "^1.0.0", "sql-template-strings": "^2.2.2", "sqlite": "^3.0.0", + "tar-fs": "^2.0.0", + "unbzip2-stream": "^1.3.3", "vue": "^2.5.21", "vue-router": "^3.0.2", "vuex": "^3.0.1", diff --git a/server/core/BookConverter/ConvertBase.js b/server/core/BookConverter/ConvertBase.js index ca78b442..ef06884e 100644 --- a/server/core/BookConverter/ConvertBase.js +++ b/server/core/BookConverter/ConvertBase.js @@ -5,6 +5,8 @@ const chardet = require('chardet'); const textUtils = require('./textUtils'); const utils = require('../utils'); +let execConverterCounter = 0; + class ConvertBase { constructor(config) { this.config = config; @@ -30,7 +32,11 @@ class ConvertBase { } async execConverter(path, args, onData) { + execConverterCounter++; try { + if (execConverterCounter > 10) + throw new Error('Слишком большая очередь конвертирования. Пожалуйста, попробуйте позже.'); + const result = await utils.spawnProcess(path, {args, onData}); if (result.code != 0) { let error = result.code; @@ -46,6 +52,8 @@ class ConvertBase { } else { throw new Error(e); } + } finally { + execConverterCounter--; } } diff --git a/server/core/BookConverter/ConvertDoc.js b/server/core/BookConverter/ConvertDoc.js index d3571b0f..ce627eb8 100644 --- a/server/core/BookConverter/ConvertDoc.js +++ b/server/core/BookConverter/ConvertDoc.js @@ -18,13 +18,13 @@ class ConvertDoc extends ConvertDocX { const {inputFiles, callback} = opts; - const outFile = `${inputFiles.fileListDir}/${path.basename(inputFiles.sourceFile)}`; + const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const docFile = `${outFile}.doc`; const docxFile = `${outFile}.docx`; const fb2File = `${outFile}.fb2`; await fs.copy(inputFiles.sourceFile, docFile); - await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.fileListDir, docFile]); + await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, docFile]); return await super.convert(docxFile, fb2File, callback); } diff --git a/server/core/BookConverter/ConvertDocX.js b/server/core/BookConverter/ConvertDocX.js index 8978475f..11ab67f7 100644 --- a/server/core/BookConverter/ConvertDocX.js +++ b/server/core/BookConverter/ConvertDocX.js @@ -6,12 +6,11 @@ const ConvertBase = require('./ConvertBase'); class ConvertDocX extends ConvertBase { check(data, opts) { const {inputFiles} = opts; - if (this.config.useExternalBookConverter && inputFiles.sourceFileType && inputFiles.sourceFileType.ext == 'zip') { //ищем файл '[Content_Types].xml' - for (const file of inputFiles.fileList) { - if (file == '[Content_Types].xml') { + for (const file of inputFiles.files) { + if (file.path == '[Content_Types].xml') { return true; } } @@ -37,7 +36,7 @@ class ConvertDocX extends ConvertBase { const {inputFiles, callback} = opts; - const outFile = `${inputFiles.fileListDir}/${path.basename(inputFiles.sourceFile)}`; + const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const docxFile = `${outFile}.docx`; const fb2File = `${outFile}.fb2`; diff --git a/server/core/BookConverter/ConvertEpub.js b/server/core/BookConverter/ConvertEpub.js new file mode 100644 index 00000000..79b25b8e --- /dev/null +++ b/server/core/BookConverter/ConvertEpub.js @@ -0,0 +1,49 @@ +const fs = require('fs-extra'); +const path = require('path'); + +const ConvertBase = require('./ConvertBase'); + +class ConvertEpub extends ConvertBase { + async check(data, opts) { + const {inputFiles} = opts; + + if (this.config.useExternalBookConverter && + inputFiles.sourceFileType && inputFiles.sourceFileType.ext == 'zip') { + //ищем файл 'mimetype' + for (const file of inputFiles.files) { + if (file.path == 'mimetype') { + const mt = await fs.readFile(`${inputFiles.filesDir}/${file.path}`); + if (mt.toString().trim() == 'application/epub+zip') + return true; + break; + } + } + } + + return false; + } + + async run(data, opts) { + if (!await this.check(data, opts)) + return false; + await this.checkExternalConverterPresent(); + + const {inputFiles, callback} = opts; + + const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; + const epubFile = `${outFile}.epub`; + const fb2File = `${outFile}.fb2`; + + await fs.copy(inputFiles.sourceFile, epubFile); + + let perc = 0; + await this.execConverter(this.calibrePath, [epubFile, fb2File], () => { + perc = (perc < 100 ? perc + 5 : 50); + callback(perc); + }); + + return await fs.readFile(fb2File); + } +} + +module.exports = ConvertEpub; diff --git a/server/core/BookConverter/ConvertHtml.js b/server/core/BookConverter/ConvertHtml.js index 9596cff8..ce28f9d4 100644 --- a/server/core/BookConverter/ConvertHtml.js +++ b/server/core/BookConverter/ConvertHtml.js @@ -18,11 +18,18 @@ class ConvertHtml extends ConvertBase { } async run(data, opts) { - const checkResult = this.check(data, opts); - if (!checkResult) - return false; + let isText = false; + if (!opts.skipCheck) { + const checkResult = this.check(data, opts); + if (!checkResult) + return false; + + isText = checkResult.isText; + } else { + isText = opts.isText; + } + const {cutTitle} = opts; - let {isText} = checkResult; let titleInfo = {}; let desc = {_n: 'description', 'title-info': titleInfo}; let pars = []; @@ -45,14 +52,14 @@ class ConvertHtml extends ConvertBase { newParagraph(); const l = pars.length; - if (pars[l - 1]._t == '') - text = text.trimLeft(); pars[l - 1]._t += text; //посчитаем отступы у текста, чтобы выделить потом параграфы const lines = text.split('\n'); for (let line of lines) { - line = repCrLfTab(line) + if (line.trim() == '') + continue; + line = repCrLfTab(line); let l = 0; while (l < line.length && line[l] == ' ') { @@ -67,7 +74,7 @@ class ConvertHtml extends ConvertBase { const newPara = new Set(['tr', 'br', 'br/', 'dd', 'p', 'title', '/title', 'h1', 'h2', 'h3', '/h1', '/h2', '/h3']); const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars - if (!cutCounter) { + if (!cutCounter && !(cutTitle && inTitle)) { growParagraph(text); } @@ -102,12 +109,27 @@ class ConvertHtml extends ConvertBase { //подозрение на чистый текст, надо разбить на параграфы if (isText || pars.length < buf.length/2000) { let total = 0; + let count = 1; for (let i = 0; i < spaceCounter.length; i++) { - total += (spaceCounter[i] ? spaceCounter[i] : 0); + const sc = (spaceCounter[i] ? spaceCounter[i] : 0); + if (sc) count++; + total += sc; + } + + let d = 0; + const mid = total/count; + for (let i = 0; i < spaceCounter.length; i++) { + const sc = (spaceCounter[i] ? spaceCounter[i] : 0); + if (sc > mid) d++; + } + + let i = 0; + //если разброс не слишком большой, выделяем параграфы + if (d < 10 && spaceCounter.length) { + total /= 20; + i = spaceCounter.length - 1; + while (i > 0 && (!spaceCounter[i] || spaceCounter[i] < total)) i--; } - total /= 10; - let i = spaceCounter.length - 1; - while (i > 0 && (!spaceCounter[i] || spaceCounter[i] < total)) i--; const parIndent = (i > 0 ? i : 0); diff --git a/server/core/BookConverter/ConvertMobi.js b/server/core/BookConverter/ConvertMobi.js new file mode 100644 index 00000000..e8d6172b --- /dev/null +++ b/server/core/BookConverter/ConvertMobi.js @@ -0,0 +1,37 @@ +const fs = require('fs-extra'); +const path = require('path'); + +const ConvertBase = require('./ConvertBase'); + +class ConvertMobi extends ConvertBase { + async check(data, opts) { + const {inputFiles} = opts; + + return (this.config.useExternalBookConverter && + inputFiles.sourceFileType && inputFiles.sourceFileType.ext == 'mobi'); + } + + async run(data, opts) { + if (!await this.check(data, opts)) + return false; + await this.checkExternalConverterPresent(); + + const {inputFiles, callback} = opts; + + const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; + const mobiFile = `${outFile}.mobi`; + const fb2File = `${outFile}.fb2`; + + await fs.copy(inputFiles.sourceFile, mobiFile); + + let perc = 0; + await this.execConverter(this.calibrePath, [mobiFile, fb2File], () => { + perc = (perc < 100 ? perc + 5 : 50); + callback(perc); + }); + + return await fs.readFile(fb2File); + } +} + +module.exports = ConvertMobi; diff --git a/server/core/BookConverter/ConvertPdf.js b/server/core/BookConverter/ConvertPdf.js new file mode 100644 index 00000000..236d0a1a --- /dev/null +++ b/server/core/BookConverter/ConvertPdf.js @@ -0,0 +1,127 @@ +const fs = require('fs-extra'); + +const sax = require('./sax'); +const utils = require('../utils'); +const ConvertHtml = require('./ConvertHtml'); + +class ConvertPdf extends ConvertHtml { + check(data, opts) { + const {inputFiles} = opts; + + return this.config.useExternalBookConverter && + inputFiles.sourceFileType && inputFiles.sourceFileType.ext == 'pdf'; + } + + async run(notUsed, opts) { + if (!this.check(notUsed, opts)) + return false; + await this.checkExternalConverterPresent(); + + const {inputFiles, callback} = opts; + + const outFile = `${inputFiles.filesDir}/${utils.randomHexString(10)}.xml`; + + //конвертируем в xml + let perc = 0; + await this.execConverter(this.pdfToHtmlPath, ['-c', '-s', '-xml', inputFiles.sourceFile, outFile], () => { + perc = (perc < 80 ? perc + 10 : 40); + callback(perc); + }); + callback(80); + + const data = await fs.readFile(outFile); + callback(90); + + //парсим xml + let lines = []; + let inText = false; + let title = ''; + let prevTop = 0; + let i = -1; + + const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars + if (!cutCounter && inText) { + lines[i].text += text + ' '; + if (i < 2) + title += text + ' '; + } + }; + + const onStartNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars + if (!cutCounter) { + if (tag == 'text' && !inText) { + let attrs = sax.getAttrsSync(tail); + const line = { + text: '', + top: parseInt((attrs.top && attrs.top.value ? attrs.top.value : null), 10), + left: parseInt((attrs.left && attrs.left.value ? attrs.left.value : null), 10), + width: parseInt((attrs.width && attrs.width.value ? attrs.width.value : null), 10), + height: parseInt((attrs.height && attrs.height.value ? attrs.height.value : null), 10), + }; + + if (line.width !== '0' || line.height !== '0') { + inText = true; + if (isNaN(line.top) || isNaN(prevTop) || (Math.abs(prevTop - line.top) > 3)) { + i++; + lines[i] = line; + } + prevTop = line.top; + } + } + } + }; + + const onEndNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars + if (tag == 'text') + inText = false; + }; + + let buf = this.decode(data).toString(); + sax.parseSync(buf, { + onStartNode, onEndNode, onTextNode + }); + + //найдем параграфы и отступы + const indents = []; + for (const line of lines) { + if (!isNaN(line.left)) { + indents[line.left] = 1; + } + } + + let j = 0; + for (let i = 0; i < indents.length; i++) { + if (indents[i]) { + j++; + indents[i] = j; + } + } + indents[0] = 0; + + //формируем текст + let text = `${title}`; + let concat = ''; + let sp = ''; + for (const line of lines) { + if (concat == '') { + const left = line.left || 0; + sp = ' '.repeat(indents[left]); + } + + let t = line.text.trim(); + if (t.substr(-1) == '-') { + t = t.substr(0, t.length - 1); + concat += t; + } else { + text += sp + concat + t + "\n"; + concat = ''; + } + } + if (concat) + text += sp + concat + "\n"; + + return await super.run(Buffer.from(text), {skipCheck: true, isText: true, cutTitle: true}); + } +} + +module.exports = ConvertPdf; diff --git a/server/core/BookConverter/ConvertRtf.js b/server/core/BookConverter/ConvertRtf.js index 6e6eda31..61f489f7 100644 --- a/server/core/BookConverter/ConvertRtf.js +++ b/server/core/BookConverter/ConvertRtf.js @@ -18,13 +18,13 @@ class ConvertRtf extends ConvertDocX { const {inputFiles, callback} = opts; - const outFile = `${inputFiles.fileListDir}/${path.basename(inputFiles.sourceFile)}`; + const outFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}`; const rtfFile = `${outFile}.rtf`; const docxFile = `${outFile}.docx`; const fb2File = `${outFile}.fb2`; await fs.copy(inputFiles.sourceFile, rtfFile); - await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.fileListDir, rtfFile]); + await this.execConverter(this.sofficePath, ['--headless', '--convert-to', 'docx', '--outdir', inputFiles.filesDir, rtfFile]); return await super.convert(docxFile, fb2File, callback); } diff --git a/server/core/BookConverter/index.js b/server/core/BookConverter/index.js index 6249b911..433c94b8 100644 --- a/server/core/BookConverter/index.js +++ b/server/core/BookConverter/index.js @@ -3,9 +3,12 @@ const FileDetector = require('../FileDetector'); //порядок важен const convertClassFactory = [ + require('./ConvertEpub'), + require('./ConvertPdf'), require('./ConvertRtf'), require('./ConvertDocX'), require('./ConvertDoc'), + require('./ConvertMobi'), require('./ConvertFb2'), require('./ConvertSamlib'), require('./ConvertHtml'), @@ -25,13 +28,6 @@ class BookConverter { const selectedFileType = await this.detector.detectFile(inputFiles.selectedFile); const data = await fs.readFile(inputFiles.selectedFile); - let selectedFileType2 = null; - let data2 = null; - if (inputFiles.nesting) { - selectedFileType2 = await this.detector.detectFile(inputFiles.nesting.selectedFile); - data2 = await fs.readFile(inputFiles.nesting.selectedFile); - } - let result = false; for (const convert of this.convertFactory) { result = await convert.run(data, {inputFiles, url, callback, dataType: selectedFileType}); @@ -39,14 +35,10 @@ class BookConverter { await fs.writeFile(outputFile, result); break; } + } - if (inputFiles.nesting) { - result = await convert.run(data2, {inputFiles: inputFiles.nesting, url, callback, dataType: selectedFileType2}); - if (result) { - await fs.writeFile(outputFile, result); - break; - } - } + if (!result && inputFiles.nesting) { + result = await this.convertToFb2(inputFiles.nesting, outputFile, url, callback); } if (!result) { @@ -58,6 +50,7 @@ class BookConverter { } callback(100); + return result; } } diff --git a/server/core/FileDecompressor.js b/server/core/FileDecompressor.js index 9948147b..59cdcbff 100644 --- a/server/core/FileDecompressor.js +++ b/server/core/FileDecompressor.js @@ -2,11 +2,11 @@ const fs = require('fs-extra'); const zlib = require('zlib'); const crypto = require('crypto'); const path = require('path'); -const utils = require('./utils'); -const decompress = require('decompress'); -const decompressGz = require('decompress-gz'); -const decompressBzip2 = require('decompress-bzip2'); +const extractZip = require('extract-zip'); +const unbzip2Stream = require('unbzip2-stream'); +const tar = require('tar-fs') +const utils = require('./utils'); const FileDetector = require('./FileDetector'); class FileDecompressor { @@ -14,72 +14,188 @@ class FileDecompressor { this.detector = new FileDetector(); } - async decompressFile(filename, outputDir) { + async decompressNested(filename, outputDir) { + await fs.ensureDir(outputDir); + const fileType = await this.detector.detectFile(filename); let result = { sourceFile: filename, sourceFileType: fileType, selectedFile: filename, - fileListDir: outputDir, - fileList: [] + filesDir: outputDir, + files: [] }; - if (!fileType || !(fileType.ext == 'zip' || fileType.ext == 'bz2' || fileType.ext == 'gz')) { + if (!fileType || !(fileType.ext == 'zip' || fileType.ext == 'bz2' || fileType.ext == 'gz' || fileType.ext == 'tar')) { return result; } - //дурной decompress, поэтому в 2 этапа - //этап 1 - let files = []; - try { - files = await decompress(filename, outputDir); - } catch (e) { - // - } + result.files = await this.decompressTarZZ(fileType.ext, filename, outputDir); - //этап 2 - if (files.length == 0) { - try { - files = await decompress(filename, outputDir, { - inputFile: filename, - plugins: [ - decompressGz(), - decompressBzip2({path: path.basename(filename)}), - ] - }); - } catch (e) { - // - } - } - let sel = filename; - let fileList = []; let max = 0; - if (files.length) { + if (result.files.length) { //ищем файл с максимальным размером - for (let file of files) { - fileList.push(file.path); - if (file.data.length > max) { + for (let file of result.files) { + if (file.size > max) { sel = `${outputDir}/${file.path}`; - max = file.data.length; + max = file.size; } } } - //дурной decompress - if (sel != filename) - await fs.chmod(sel, 0o664); - result.selectedFile = sel; - result.fileList = fileList; if (sel != filename) { - result.nesting = await this.decompressFile(sel, `${outputDir}/${utils.randomHexString(10)}`); + result.nesting = await this.decompressNested(sel, `${outputDir}/${utils.randomHexString(10)}`); } return result; } + async unpack(filename, outputDir) { + const fileType = await this.detector.detectFile(filename); + if (!fileType) + throw new Error('Не удалось определить формат файла'); + + return await this.decompress(fileType.ext, filename, outputDir); + } + + async unpackTarZZ(filename, outputDir) { + const fileType = await this.detector.detectFile(filename); + if (!fileType) + throw new Error('Не удалось определить формат файла'); + + return await this.decompressTarZZ(fileType.ext, filename, outputDir); + } + + async decompressTarZZ(fileExt, filename, outputDir) { + const files = await this.decompress(fileExt, filename, outputDir); + if (fileExt == 'tar' || files.length != 1) + return files; + + const tarFilename = `${outputDir}/${files[0].path}`; + const fileType = await this.detector.detectFile(tarFilename); + if (!fileType || fileType.ext != 'tar') + return files; + + const newTarFilename = `${outputDir}/${utils.randomHexString(30)}`; + await fs.rename(tarFilename, newTarFilename); + + const tarFiles = await this.decompress('tar', newTarFilename, outputDir); + await fs.remove(newTarFilename); + + return tarFiles; + } + + async decompress(fileExt, filename, outputDir) { + let files = []; + + switch (fileExt) { + case 'zip': + files = await this.unZip(filename, outputDir); + break; + case 'bz2': + files = await this.unBz2(filename, outputDir); + break; + case 'gz': + files = await this.unGz(filename, outputDir); + break; + case 'tar': + files = await this.unTar(filename, outputDir); + break; + default: + throw new Error(`FileDecompressor: неизвестный формат файла '${fileExt}'`); + } + + return files; + } + + async unZip(filename, outputDir) { + return new Promise((resolve, reject) => { + const files = []; + extractZip(filename, { + dir: outputDir, + onEntry: (entry) => { + files.push({path: entry.fileName, size: entry.uncompressedSize}); + } + }, (err) => { + if (err) + reject(err); + resolve(files); + }); + }); + } + + unBz2(filename, outputDir) { + return this.decompressByStream(unbzip2Stream(), filename, outputDir); + } + + unGz(filename, outputDir) { + return this.decompressByStream(zlib.createGunzip(), filename, outputDir); + } + + unTar(filename, outputDir) { + return new Promise((resolve, reject) => { + const files = []; + + const tarExtract = tar.extract(outputDir, { + map: (header) => { + files.push({path: header.name, size: header.size}); + return header; + } + }); + + tarExtract.on('finish', () => { + resolve(files); + }); + + tarExtract.on('error', (err) => { + reject(err); + }); + + const inputStream = fs.createReadStream(filename); + inputStream.on('error', (err) => { + reject(err); + }); + + inputStream.pipe(tarExtract); + }); + } + + decompressByStream(stream, filename, outputDir) { + return new Promise(async(resolve, reject) => { + const file = {path: path.parse(filename).name}; + let outFilename = `${outputDir}/${file.path}`; + if (await fs.pathExists(outFilename)) { + file.path = `${utils.randomHexString(10)}-${file.path}`; + outFilename = `${outputDir}/${file.path}`; + } + + const inputStream = fs.createReadStream(filename); + const outputStream = fs.createWriteStream(outFilename); + + outputStream.on('finish', async() => { + try { + file.size = (await fs.stat(outFilename)).size; + } catch (e) { + reject(e); + } + resolve([file]); + }); + + inputStream.on('error', (err) => { + reject(err); + }); + + outputStream.on('error', (err) => { + reject(err); + }); + + inputStream.pipe(stream).pipe(outputStream); + }); + } + async gzipBuffer(buf) { return new Promise((resolve, reject) => { zlib.gzip(buf, {level: 1}, (err, result) => { diff --git a/server/core/FileDetector/signatures.json b/server/core/FileDetector/signatures.json index 691df440..a267fc06 100644 --- a/server/core/FileDetector/signatures.json +++ b/server/core/FileDetector/signatures.json @@ -128,21 +128,10 @@ { "type": "equal", "start": 3, "end": 4, "bytes": "08" } ] }, - { "type": "notEqual", "start": 36, "end": 58, "bytes": "70656170706c69636174696f6e2f657075622b7a6970" }, { "type": "notEqual", "start": 30, "end": 50, "bytes": "4d4554412d494e462f6d6f7a696c6c612e727361" } ] }, - { - "type": "epub", - "ext": "epub", - "mime": "application/epub+zip", - "rules": [ - { "type": "equal", "start": 0, "end": 4, "bytes": "504b0304" }, - { "type": "equal", "start": 36, "end": 58, "bytes": "70656170706c69636174696f6e2f657075622b7a6970" } - ] - }, - { "type": "xpi", "ext": "xpi", @@ -722,6 +711,15 @@ ] } ] + }, + + { + "type": "mobi", + "ext": "mobi", + "mime": "application/x-mobipocket-ebook", + "rules": [ + { "type": "equal", "start": 64, "end": 68, "bytes": "4d4f4249" } + ] } ] diff --git a/server/core/ReaderWorker.js b/server/core/ReaderWorker.js index ed05a93d..46b758f3 100644 --- a/server/core/ReaderWorker.js +++ b/server/core/ReaderWorker.js @@ -63,7 +63,14 @@ class ReaderWorker { //decompress wState.set({state: 'decompress', step: 2, progress: 0}); decompDir = `${this.config.tempDownloadDir}/${decompDirname}`; - const decompFiles = await this.decomp.decompressFile(downloadedFilename, decompDir); + let decompFiles = {}; + try { + decompFiles = await this.decomp.decompressNested(downloadedFilename, decompDir); + } catch (e) { + if (this.config.branch == 'development') + console.error(e); + throw new Error('Ошибка распаковки'); + } wState.set({progress: 100}); //конвертирование в fb2