diff --git a/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue b/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue index 044cff35..efebcc51 100644 --- a/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue +++ b/client/components/Reader/HelpPage/DonateHelpPage/DonateHelpPage.vue @@ -12,6 +12,18 @@ +
+ +
{{ paypalAddress }} + + + + +
+
+
{{ bitcoinAddress }} @@ -53,6 +65,7 @@ export default @Component({ }) class DonateHelpPage extends Vue { yandexAddress = '410018702323056'; + paypalAddress = 'bookpauk@gmail.com'; bitcoinAddress = '3EbgZ7MK1UVaN38Gty5DCBtS4PknM4Ut85'; litecoinAddress = 'MP39Riec4oSNB3XMjiquKoLWxbufRYNXxZ'; moneroAddress = '8BQPnvHcPSHM5gMQsmuypDgx9NNsYqwXKfDDuswEyF2Q2ewQSfd2pkK6ydH2wmMyq2JViZvy9DQ35hLMx7g72mFWNJTPtnz'; diff --git a/client/components/Reader/HelpPage/DonateHelpPage/assets/paypal.png b/client/components/Reader/HelpPage/DonateHelpPage/assets/paypal.png new file mode 100644 index 00000000..17e6418c Binary files /dev/null and b/client/components/Reader/HelpPage/DonateHelpPage/assets/paypal.png differ diff --git a/client/components/Reader/LoaderPage/PasteTextPage/PasteTextPage.vue b/client/components/Reader/LoaderPage/PasteTextPage/PasteTextPage.vue index 557768a2..89f73fe6 100644 --- a/client/components/Reader/LoaderPage/PasteTextPage/PasteTextPage.vue +++ b/client/components/Reader/LoaderPage/PasteTextPage/PasteTextPage.vue @@ -68,7 +68,7 @@ class PasteTextPage extends Vue { } loadBuffer() { - this.$emit('load-buffer', {buffer: `${this.bookTitle}${this.$refs.textArea.value}`}); + this.$emit('load-buffer', {buffer: `${utils.escapeXml(this.bookTitle)}${this.$refs.textArea.value}`}); this.close(); } diff --git a/client/components/Reader/versionHistory.js b/client/components/Reader/versionHistory.js index 98d1ced4..dac29b83 100644 --- a/client/components/Reader/versionHistory.js +++ b/client/components/Reader/versionHistory.js @@ -1,4 +1,16 @@ export const versionHistory = [ +{ + showUntil: '2020-02-05', + header: '0.8.4 (2020-02-06)', + content: +` + +` +}, + { showUntil: '2020-01-27', header: '0.8.3 (2020-01-28)', diff --git a/client/share/utils.js b/client/share/utils.js index 1d0357da..a171a42f 100644 --- a/client/share/utils.js +++ b/client/share/utils.js @@ -193,4 +193,13 @@ export function parseQuery(str) { query[first] = [query[first], second]; } return query; +} + +export function escapeXml(str) { + return str.replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') + ; } \ No newline at end of file diff --git a/docs/omnireader/deploy.sh b/docs/omnireader/deploy.sh index 15efc847..ba033a77 100755 --- a/docs/omnireader/deploy.sh +++ b/docs/omnireader/deploy.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash npm run build:linux sudo -u www-data cp -r ../../dist/linux/* /home/liberama diff --git a/docs/omnireader/run_server.sh b/docs/omnireader/run_server.sh index 9a2e74fd..e28c196e 100755 --- a/docs/omnireader/run_server.sh +++ b/docs/omnireader/run_server.sh @@ -1,3 +1,11 @@ -#!/bin/sh +#!/bin/bash -sudo -H -u www-data sh -c "cd /var/www; /home/liberama/liberama" \ No newline at end of file +sudo -H -u www-data bash -c "\ +while true; do\ + trap '' 2;\ + cd /var/www;\ + /home/liberama/liberama;\ + trap 2;\ + echo \"Restart after 5 sec. Press Ctrl+C to exit.\";\ + sleep 5;\ +done;" diff --git a/package-lock.json b/package-lock.json index 2c51bf6f..533368ff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.8.3", + "version": "0.8.4", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -7428,11 +7428,6 @@ "semver": "^5.3.0" } }, - "node-stream-zip": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.8.2.tgz", - "integrity": "sha512-zwP2F/R28Oqtl0gOLItk5QjJ6jEU8XO4kaUMgeqvCyXPgdCZlm8T/5qLMiNy+moJCBCiMQAaX7aVMRhT0t2vkQ==" - }, "nopt": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", diff --git a/package.json b/package.json index 8e65f54b..799be440 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.8.3", + "version": "0.8.4", "author": "Book Pauk ", "license": "CC0-1.0", "repository": "bookpauk/liberama", @@ -72,7 +72,6 @@ "lodash": "^4.17.15", "minimist": "^1.2.0", "multer": "^1.4.2", - "node-stream-zip": "^1.8.2", "pako": "^1.0.10", "path-browserify": "^1.0.0", "quasar": "^1.8.5", diff --git a/server/core/AppLogger.js b/server/core/AppLogger.js index 71ec3831..af6799ca 100644 --- a/server/core/AppLogger.js +++ b/server/core/AppLogger.js @@ -25,7 +25,8 @@ class AppLogger { loggerParams = [ {log: 'ConsoleLog'}, {log: 'FileLog', fileName: `${config.logDir}/${config.name}.log`}, - {log: 'FileLog', fileName: `${config.logDir}/${config.name}.err.log`, exclude: [LM_OK, LM_INFO]}, + {log: 'FileLog', fileName: `${config.logDir}/${config.name}.err.log`, exclude: [LM_OK, LM_INFO, LM_TOTAL]}, + {log: 'FileLog', fileName: `${config.logDir}/${config.name}.fatal.log`, exclude: [LM_OK, LM_INFO, LM_WARN, LM_ERR, LM_TOTAL]},//LM_FATAL only ]; } diff --git a/server/core/FileDecompressor.js b/server/core/FileDecompressor.js index cc7a4c0f..8a601744 100644 --- a/server/core/FileDecompressor.js +++ b/server/core/FileDecompressor.js @@ -3,11 +3,13 @@ const zlib = require('zlib'); const path = require('path'); const unbzip2Stream = require('unbzip2-stream'); const tar = require('tar-fs'); -const ZipStreamer = require('./ZipStreamer'); +const iconv = require('iconv-lite'); +const ZipStreamer = require('./Zip/ZipStreamer'); const appLogger = new (require('./AppLogger'))();//singleton -const utils = require('./utils'); const FileDetector = require('./FileDetector'); +const textUtils = require('./Reader/BookConverter/textUtils'); +const utils = require('./utils'); class FileDecompressor { constructor(limitFileSize = 0) { @@ -114,7 +116,25 @@ class FileDecompressor { async unZip(filename, outputDir) { const zip = new ZipStreamer(); - return await zip.unpack(filename, outputDir, null, this.limitFileSize); + try { + return await zip.unpack(filename, outputDir, { + limitFileSize: this.limitFileSize, + limitFileCount: 1000 + }); + } catch (e) { + fs.emptyDir(outputDir); + return await zip.unpack(filename, outputDir, { + limitFileSize: this.limitFileSize, + limitFileCount: 1000, + decodeEntryNameCallback: (nameRaw) => { + const enc = textUtils.getEncodingLite(nameRaw); + if (enc.indexOf('ISO-8859') < 0) { + return iconv.decode(nameRaw, enc); + } + return nameRaw; + } + }); + } } unBz2(filename, outputDir) { diff --git a/server/core/LibSharedStorage/MegaStorage.js b/server/core/LibSharedStorage/MegaStorage.js index c59b13d6..718534d3 100644 --- a/server/core/LibSharedStorage/MegaStorage.js +++ b/server/core/LibSharedStorage/MegaStorage.js @@ -3,7 +3,7 @@ const fs = require('fs-extra'); const path = require('path'); const log = new (require('../AppLogger'))().log;//singleton -const ZipStreamer = require('../ZipStreamer'); +const ZipStreamer = require('../Zip/ZipStreamer'); const utils = require('../utils'); diff --git a/server/core/Logger.js b/server/core/Logger.js index f01375cb..db5d122b 100644 --- a/server/core/Logger.js +++ b/server/core/Logger.js @@ -226,12 +226,12 @@ class Logger { // catch ctrl+c event and exit normally process.on('SIGINT', () => { - this.log(LM_WARN, 'Ctrl-C pressed, exiting...'); + this.log(LM_FATAL, 'Ctrl-C pressed, exiting...'); process.exit(2); }); process.on('SIGTERM', () => { - this.log(LM_WARN, 'Kill signal, exiting...'); + this.log(LM_FATAL, 'Kill signal, exiting...'); process.exit(2); }); diff --git a/server/core/Reader/BookConverter/ConvertBase.js b/server/core/Reader/BookConverter/ConvertBase.js index 0c21bd12..34aafe67 100644 --- a/server/core/Reader/BookConverter/ConvertBase.js +++ b/server/core/Reader/BookConverter/ConvertBase.js @@ -1,6 +1,5 @@ const fs = require('fs-extra'); const iconv = require('iconv-lite'); -const chardet = require('chardet'); const he = require('he'); const LimitedQueue = require('../../LimitedQueue'); @@ -77,16 +76,6 @@ class ConvertBase { decode(data) { let selected = textUtils.getEncoding(data); - if (selected == 'ISO-8859-5') { - const charsetAll = chardet.detectAll(data.slice(0, 20000)); - for (const charset of charsetAll) { - if (charset.name.indexOf('ISO-8859') < 0) { - selected = charset.name; - break; - } - } - } - if (selected.toLowerCase() != 'utf-8') return iconv.decode(data, selected); else diff --git a/server/core/Reader/BookConverter/ConvertHtml.js b/server/core/Reader/BookConverter/ConvertHtml.js index 4131436f..8df01185 100644 --- a/server/core/Reader/BookConverter/ConvertHtml.js +++ b/server/core/Reader/BookConverter/ConvertHtml.js @@ -6,6 +6,7 @@ class ConvertHtml extends ConvertBase { check(data, opts) { const {dataType} = opts; + //html? if (dataType && (dataType.ext == 'html' || dataType.ext == 'xml')) return {isText: false}; @@ -14,6 +15,11 @@ class ConvertHtml extends ConvertBase { return {isText: true}; } + //из буфера обмена? + if (data.toString().indexOf('') == 0) { + return {isText: false}; + } + return false; } diff --git a/server/core/Reader/BookConverter/textUtils.js b/server/core/Reader/BookConverter/textUtils.js index 80ee401e..f7f2c185 100644 --- a/server/core/Reader/BookConverter/textUtils.js +++ b/server/core/Reader/BookConverter/textUtils.js @@ -1,4 +1,23 @@ -function getEncoding(buf, returnAll) { +const chardet = require('chardet'); + +function getEncoding(buf) { + let selected = getEncodingLite(buf); + + if (selected == 'ISO-8859-5') { + const charsetAll = chardet.detectAll(buf.slice(0, 20000)); + for (const charset of charsetAll) { + if (charset.name.indexOf('ISO-8859') < 0) { + selected = charset.name; + break; + } + } + } + + return selected; +} + + +function getEncodingLite(buf, returnAll) { const lowerCase = 3; const upperCase = 1; @@ -106,5 +125,6 @@ function checkIfText(buf) { module.exports = { getEncoding, + getEncodingLite, checkIfText, } \ No newline at end of file diff --git a/server/core/ZipStreamer.js b/server/core/Zip/ZipStreamer.js similarity index 67% rename from server/core/ZipStreamer.js rename to server/core/Zip/ZipStreamer.js index c761eaa9..1ac9ef9f 100644 --- a/server/core/ZipStreamer.js +++ b/server/core/Zip/ZipStreamer.js @@ -2,7 +2,7 @@ const fs = require('fs-extra'); const path = require('path'); const zipStream = require('zip-stream'); -const unzipStream = require('node-stream-zip'); +const unzipStream = require('./node_stream_zip'); class ZipStreamer { constructor() { @@ -52,9 +52,15 @@ class ZipStreamer { })().catch(reject); }); } - unpack(zipFile, outputDir, entryCallback, limitFileSize = 0) { + unpack(zipFile, outputDir, options, entryCallback) { return new Promise((resolve, reject) => { entryCallback = (entryCallback ? entryCallback : () => {}); + const { + limitFileSize = 0, + limitFileCount = 0, + decodeEntryNameCallback = false, + } = options; + const unzip = new unzipStream({file: zipFile}); unzip.on('error', reject); @@ -67,23 +73,41 @@ class ZipStreamer { }); unzip.on('ready', () => { - if (limitFileSize) { - for (const entry of Object.values(unzip.entries())) { - if (!entry.isDirectory && entry.size > limitFileSize) { + if (limitFileCount || limitFileSize || decodeEntryNameCallback) { + const entries = Object.values(unzip.entries()); + if (limitFileCount && entries.length > limitFileCount) { + reject('Слишком много файлов'); + return; + } + + for (const entry of entries) { + if (limitFileSize && !entry.isDirectory && entry.size > limitFileSize) { reject('Файл слишком большой'); return; } + + if (decodeEntryNameCallback) { + entry.name = (decodeEntryNameCallback(entry.nameRaw)).toString(); + } } } unzip.extract(null, outputDir, (err) => { - if (err) reject(err); - unzip.close(); - resolve(files); + if (err) { + reject(err); + return; + } + try { + unzip.close(); + resolve(files); + } catch (e) { + reject(e); + } }); }); }); } + } module.exports = ZipStreamer; \ No newline at end of file diff --git a/server/core/Zip/node_stream_zip.js b/server/core/Zip/node_stream_zip.js new file mode 100644 index 00000000..d60cc996 --- /dev/null +++ b/server/core/Zip/node_stream_zip.js @@ -0,0 +1,1055 @@ +/** + * @license node-stream-zip | (c) 2015 Antelle | https://github.com/antelle/node-stream-zip/blob/master/LICENSE + * Portions copyright https://github.com/cthackers/adm-zip | https://raw.githubusercontent.com/cthackers/adm-zip/master/LICENSE + */ + +// region Deps + +var + util = require('util'), + fs = require('fs'), + path = require('path'), + events = require('events'), + zlib = require('zlib'), + stream = require('stream'); + +// endregion + +// region Constants + +var consts = { + /* The local file header */ + LOCHDR : 30, // LOC header size + LOCSIG : 0x04034b50, // "PK\003\004" + LOCVER : 4, // version needed to extract + LOCFLG : 6, // general purpose bit flag + LOCHOW : 8, // compression method + LOCTIM : 10, // modification time (2 bytes time, 2 bytes date) + LOCCRC : 14, // uncompressed file crc-32 value + LOCSIZ : 18, // compressed size + LOCLEN : 22, // uncompressed size + LOCNAM : 26, // filename length + LOCEXT : 28, // extra field length + + /* The Data descriptor */ + EXTSIG : 0x08074b50, // "PK\007\008" + EXTHDR : 16, // EXT header size + EXTCRC : 4, // uncompressed file crc-32 value + EXTSIZ : 8, // compressed size + EXTLEN : 12, // uncompressed size + + /* The central directory file header */ + CENHDR : 46, // CEN header size + CENSIG : 0x02014b50, // "PK\001\002" + CENVEM : 4, // version made by + CENVER : 6, // version needed to extract + CENFLG : 8, // encrypt, decrypt flags + CENHOW : 10, // compression method + CENTIM : 12, // modification time (2 bytes time, 2 bytes date) + CENCRC : 16, // uncompressed file crc-32 value + CENSIZ : 20, // compressed size + CENLEN : 24, // uncompressed size + CENNAM : 28, // filename length + CENEXT : 30, // extra field length + CENCOM : 32, // file comment length + CENDSK : 34, // volume number start + CENATT : 36, // internal file attributes + CENATX : 38, // external file attributes (host system dependent) + CENOFF : 42, // LOC header offset + + /* The entries in the end of central directory */ + ENDHDR : 22, // END header size + ENDSIG : 0x06054b50, // "PK\005\006" + ENDSIGFIRST : 0x50, + ENDSUB : 8, // number of entries on this disk + ENDTOT : 10, // total number of entries + ENDSIZ : 12, // central directory size in bytes + ENDOFF : 16, // offset of first CEN header + ENDCOM : 20, // zip file comment length + MAXFILECOMMENT : 0xFFFF, + + /* The entries in the end of ZIP64 central directory locator */ + ENDL64HDR : 20, // ZIP64 end of central directory locator header size + ENDL64SIG : 0x07064b50, // ZIP64 end of central directory locator signature + ENDL64SIGFIRST : 0x50, + ENDL64OFS : 8, // ZIP64 end of central directory offset + + /* The entries in the end of ZIP64 central directory */ + END64HDR : 56, // ZIP64 end of central directory header size + END64SIG : 0x06064b50, // ZIP64 end of central directory signature + END64SIGFIRST : 0x50, + END64SUB : 24, // number of entries on this disk + END64TOT : 32, // total number of entries + END64SIZ : 40, + END64OFF : 48, + + /* Compression methods */ + STORED : 0, // no compression + SHRUNK : 1, // shrunk + REDUCED1 : 2, // reduced with compression factor 1 + REDUCED2 : 3, // reduced with compression factor 2 + REDUCED3 : 4, // reduced with compression factor 3 + REDUCED4 : 5, // reduced with compression factor 4 + IMPLODED : 6, // imploded + // 7 reserved + DEFLATED : 8, // deflated + ENHANCED_DEFLATED: 9, // enhanced deflated + PKWARE : 10,// PKWare DCL imploded + // 11 reserved + BZIP2 : 12, // compressed using BZIP2 + // 13 reserved + LZMA : 14, // LZMA + // 15-17 reserved + IBM_TERSE : 18, // compressed using IBM TERSE + IBM_LZ77 : 19, //IBM LZ77 z + + /* General purpose bit flag */ + FLG_ENC : 0, // encrypted file + FLG_COMP1 : 1, // compression option + FLG_COMP2 : 2, // compression option + FLG_DESC : 4, // data descriptor + FLG_ENH : 8, // enhanced deflation + FLG_STR : 16, // strong encryption + FLG_LNG : 1024, // language encoding + FLG_MSK : 4096, // mask header values + FLG_ENTRY_ENC : 1, + + /* 4.5 Extensible data fields */ + EF_ID : 0, + EF_SIZE : 2, + + /* Header IDs */ + ID_ZIP64 : 0x0001, + ID_AVINFO : 0x0007, + ID_PFS : 0x0008, + ID_OS2 : 0x0009, + ID_NTFS : 0x000a, + ID_OPENVMS : 0x000c, + ID_UNIX : 0x000d, + ID_FORK : 0x000e, + ID_PATCH : 0x000f, + ID_X509_PKCS7 : 0x0014, + ID_X509_CERTID_F : 0x0015, + ID_X509_CERTID_C : 0x0016, + ID_STRONGENC : 0x0017, + ID_RECORD_MGT : 0x0018, + ID_X509_PKCS7_RL : 0x0019, + ID_IBM1 : 0x0065, + ID_IBM2 : 0x0066, + ID_POSZIP : 0x4690, + + EF_ZIP64_OR_32 : 0xffffffff, + EF_ZIP64_OR_16 : 0xffff +}; + +// endregion + +// region StreamZip + +var StreamZip = function(config) { + var + fd, + fileSize, + chunkSize, + ready = false, + that = this, + op, + centralDirectory, + closed, + + entries = config.storeEntries !== false ? {} : null, + fileName = config.file; + + open(); + + function open() { + fs.open(fileName, 'r', function(err, f) { + if (err) + return that.emit('error', err); + fd = f; + fs.fstat(fd, function(err, stat) { + if (err) + return that.emit('error', err); + fileSize = stat.size; + chunkSize = config.chunkSize || Math.round(fileSize / 1000); + chunkSize = Math.max(Math.min(chunkSize, Math.min(128*1024, fileSize)), Math.min(1024, fileSize)); + readCentralDirectory(); + }); + }); + } + + function readUntilFoundCallback(err, bytesRead) { + if (err || !bytesRead) + return that.emit('error', err || 'Archive read error'); + var + buffer = op.win.buffer, + pos = op.lastPos, + bufferPosition = pos - op.win.position, + minPos = op.minPos; + while (--pos >= minPos && --bufferPosition >= 0) { + if (buffer.length - bufferPosition >= 4 && + buffer[bufferPosition] === op.firstByte) { // quick check first signature byte + if (buffer.readUInt32LE(bufferPosition) === op.sig) { + op.lastBufferPosition = bufferPosition; + op.lastBytesRead = bytesRead; + op.complete(); + return; + } + } + } + if (pos === minPos) { + return that.emit('error', 'Bad archive'); + } + op.lastPos = pos + 1; + op.chunkSize *= 2; + if (pos <= minPos) + return that.emit('error', 'Bad archive'); + var expandLength = Math.min(op.chunkSize, pos - minPos); + op.win.expandLeft(expandLength, readUntilFoundCallback); + + } + + function readCentralDirectory() { + var totalReadLength = Math.min(consts.ENDHDR + consts.MAXFILECOMMENT, fileSize); + op = { + win: new FileWindowBuffer(fd), + totalReadLength: totalReadLength, + minPos: fileSize - totalReadLength, + lastPos: fileSize, + chunkSize: Math.min(1024, chunkSize), + firstByte: consts.ENDSIGFIRST, + sig: consts.ENDSIG, + complete: readCentralDirectoryComplete + }; + op.win.read(fileSize - op.chunkSize, op.chunkSize, readUntilFoundCallback); + } + + function readCentralDirectoryComplete() { + var buffer = op.win.buffer; + var pos = op.lastBufferPosition; + try { + centralDirectory = new CentralDirectoryHeader(); + centralDirectory.read(buffer.slice(pos, pos + consts.ENDHDR)); + centralDirectory.headerOffset = op.win.position + pos; + if (centralDirectory.commentLength) + that.comment = buffer.slice(pos + consts.ENDHDR, + pos + consts.ENDHDR + centralDirectory.commentLength).toString(); + else + that.comment = null; + that.entriesCount = centralDirectory.volumeEntries; + that.centralDirectory = centralDirectory; + if (centralDirectory.volumeEntries === consts.EF_ZIP64_OR_16 && centralDirectory.totalEntries === consts.EF_ZIP64_OR_16 + || centralDirectory.size === consts.EF_ZIP64_OR_32 || centralDirectory.offset === consts.EF_ZIP64_OR_32) { + readZip64CentralDirectoryLocator(); + } else { + op = {}; + readEntries(); + } + } catch (err) { + that.emit('error', err); + } + } + + function readZip64CentralDirectoryLocator() { + var length = consts.ENDL64HDR; + if (op.lastBufferPosition > length) { + op.lastBufferPosition -= length; + readZip64CentralDirectoryLocatorComplete(); + } else { + op = { + win: op.win, + totalReadLength: length, + minPos: op.win.position - length, + lastPos: op.win.position, + chunkSize: op.chunkSize, + firstByte: consts.ENDL64SIGFIRST, + sig: consts.ENDL64SIG, + complete: readZip64CentralDirectoryLocatorComplete + }; + op.win.read(op.lastPos - op.chunkSize, op.chunkSize, readUntilFoundCallback); + } + } + + function readZip64CentralDirectoryLocatorComplete() { + var buffer = op.win.buffer; + var locHeader = new CentralDirectoryLoc64Header(); + locHeader.read(buffer.slice(op.lastBufferPosition, op.lastBufferPosition + consts.ENDL64HDR)); + var readLength = fileSize - locHeader.headerOffset; + op = { + win: op.win, + totalReadLength: readLength, + minPos: locHeader.headerOffset, + lastPos: op.lastPos, + chunkSize: op.chunkSize, + firstByte: consts.END64SIGFIRST, + sig: consts.END64SIG, + complete: readZip64CentralDirectoryComplete + }; + op.win.read(fileSize - op.chunkSize, op.chunkSize, readUntilFoundCallback); + } + + function readZip64CentralDirectoryComplete() { + var buffer = op.win.buffer; + var zip64cd = new CentralDirectoryZip64Header(); + zip64cd.read(buffer.slice(op.lastBufferPosition, op.lastBufferPosition + consts.END64HDR)); + that.centralDirectory.volumeEntries = zip64cd.volumeEntries; + that.centralDirectory.totalEntries = zip64cd.totalEntries; + that.centralDirectory.size = zip64cd.size; + that.centralDirectory.offset = zip64cd.offset; + that.entriesCount = zip64cd.volumeEntries; + op = {}; + readEntries(); + } + + function readEntries() { + op = { + win: new FileWindowBuffer(fd), + pos: centralDirectory.offset, + chunkSize: chunkSize, + entriesLeft: centralDirectory.volumeEntries + }; + op.win.read(op.pos, Math.min(chunkSize, fileSize - op.pos), readEntriesCallback); + } + + function readEntriesCallback(err, bytesRead) { + if (err || !bytesRead) + return that.emit('error', err || 'Entries read error'); + var + buffer = op.win.buffer, + bufferPos = op.pos - op.win.position, + bufferLength = buffer.length, + entry = op.entry; + try { + while (op.entriesLeft > 0) { + if (!entry) { + entry = new ZipEntry(); + entry.readHeader(buffer, bufferPos); + entry.headerOffset = op.win.position + bufferPos; + op.entry = entry; + op.pos += consts.CENHDR; + bufferPos += consts.CENHDR; + } + var entryHeaderSize = entry.fnameLen + entry.extraLen + entry.comLen; + var advanceBytes = entryHeaderSize + (op.entriesLeft > 1 ? consts.CENHDR : 0); + if (bufferLength - bufferPos < advanceBytes) { + op.win.moveRight(chunkSize, readEntriesCallback, bufferPos); + op.move = true; + return; + } + entry.read(buffer, bufferPos); + if (!config.skipEntryNameValidation) { + entry.validateName(); + } + if (entries) + entries[entry.name] = entry; + that.emit('entry', entry); + op.entry = entry = null; + op.entriesLeft--; + op.pos += entryHeaderSize; + bufferPos += entryHeaderSize; + } + that.emit('ready'); + } catch (err) { + that.emit('error', err); + } + } + + function checkEntriesExist() { + if (!entries) + throw new Error('storeEntries disabled'); + } + + Object.defineProperty(this, 'ready', { get: function() { return ready; } }); + + this.entry = function(name) { + checkEntriesExist(); + return entries[name]; + }; + + this.entries = function() { + checkEntriesExist(); + return entries; + }; + + this.stream = function(entry, callback) { + return this.openEntry(entry, function(err, entry) { + if (err) + return callback(err); + var offset = dataOffset(entry); + var entryStream = new EntryDataReaderStream(fd, offset, entry.compressedSize); + if (entry.method === consts.STORED) { + } else if (entry.method === consts.DEFLATED || entry.method === consts.ENHANCED_DEFLATED) { + entryStream = entryStream.pipe(zlib.createInflateRaw()); + } else { + return callback('Unknown compression method: ' + entry.method); + } + if (canVerifyCrc(entry)) + entryStream = entryStream.pipe(new EntryVerifyStream(entryStream, entry.crc, entry.size)); + callback(null, entryStream); + }, false); + }; + + this.entryDataSync = function(entry) { + var err = null; + this.openEntry(entry, function(e, en) { + err = e; + entry = en; + }, true); + if (err) + throw err; + var + data = Buffer.alloc(entry.compressedSize), + bytesRead; + new FsRead(fd, data, 0, entry.compressedSize, dataOffset(entry), function(e, br) { + err = e; + bytesRead = br; + }).read(true); + if (err) + throw err; + if (entry.method === consts.STORED) { + } else if (entry.method === consts.DEFLATED || entry.method === consts.ENHANCED_DEFLATED) { + data = zlib.inflateRawSync(data); + } else { + throw new Error('Unknown compression method: ' + entry.method); + } + if (data.length !== entry.size) + throw new Error('Invalid size'); + if (canVerifyCrc(entry)) { + var verify = new CrcVerify(entry.crc, entry.size); + verify.data(data); + } + return data; + }; + + this.openEntry = function(entry, callback, sync) { + if (typeof entry === 'string') { + checkEntriesExist(); + entry = entries[entry]; + if (!entry) + return callback('Entry not found'); + } + if (!entry.isFile) + return callback('Entry is not file'); + if (!fd) + return callback('Archive closed'); + var buffer = Buffer.alloc(consts.LOCHDR); + new FsRead(fd, buffer, 0, buffer.length, entry.offset, function(err) { + if (err) + return callback(err); + var readEx; + try { + entry.readDataHeader(buffer); + if (entry.encrypted) { + readEx = 'Entry encrypted'; + } + } catch (ex) { + readEx = ex + } + callback(readEx, entry); + }).read(sync); + }; + + function dataOffset(entry) { + return entry.offset + consts.LOCHDR + entry.fnameLen + entry.extraLen; + } + + function canVerifyCrc(entry) { + // if bit 3 (0x08) of the general-purpose flags field is set, then the CRC-32 and file sizes are not known when the header is written + return (entry.flags & 0x8) != 0x8; + } + + function extract(entry, outPath, callback) { + that.stream(entry, function (err, stm) { + if (err) { + callback(err); + } else { + var fsStm, errThrown; + stm.on('error', function(err) { + errThrown = err; + if (fsStm) { + stm.unpipe(fsStm); + fsStm.close(function () { + callback(err); + }); + } + }); + fs.open(outPath, 'w', function(err, fdFile) { + if (err) + return callback(err || errThrown); + if (errThrown) { + fs.close(fd, function() { + callback(errThrown); + }); + return; + } + fsStm = fs.createWriteStream(outPath, { fd: fdFile }); + fsStm.on('finish', function() { + that.emit('extract', entry, outPath); + if (!errThrown) + callback(); + }); + stm.pipe(fsStm); + }); + } + }); + } + + function createDirectories(baseDir, dirs, callback) { + if (!dirs.length) + return callback(); + var dir = dirs.shift(); + dir = path.join(baseDir, path.join.apply(path, dir)); + fs.mkdir(dir, function(err) { + if (err && err.code !== 'EEXIST') + return callback(err); + createDirectories(baseDir, dirs, callback); + }); + } + + function extractFiles(baseDir, baseRelPath, files, callback, extractedCount) { + if (!files.length) + return callback(null, extractedCount); + var file = files.shift(); + var targetPath = path.join(baseDir, file.name.replace(baseRelPath, '')); + extract(file, targetPath, function (err) { + if (err) + return callback(err, extractedCount); + extractFiles(baseDir, baseRelPath, files, callback, extractedCount + 1); + }); + } + + this.extract = function(entry, outPath, callback) { + var entryName = entry || ''; + if (typeof entry === 'string') { + entry = this.entry(entry); + if (entry) { + entryName = entry.name; + } else { + if (entryName.length && entryName[entryName.length - 1] !== '/') + entryName += '/'; + } + } + if (!entry || entry.isDirectory) { + var files = [], dirs = [], allDirs = {}; + for (var e in entries) { + if (Object.prototype.hasOwnProperty.call(entries, e) && e.lastIndexOf(entryName, 0) === 0) { + var relPath = e.replace(entryName, ''); + var childEntry = entries[e]; + if (childEntry.isFile) { + files.push(childEntry); + relPath = path.dirname(relPath); + } + if (relPath && !allDirs[relPath] && relPath !== '.') { + allDirs[relPath] = true; + var parts = relPath.split('/').filter(function (f) { return f; }); + if (parts.length) + dirs.push(parts); + while (parts.length > 1) { + parts = parts.slice(0, parts.length - 1); + var partsPath = parts.join('/'); + if (allDirs[partsPath] || partsPath === '.') { + break; + } + allDirs[partsPath] = true; + dirs.push(parts); + } + } + } + } + dirs.sort(function(x, y) { return x.length - y.length; }); + if (dirs.length) { + createDirectories(outPath, dirs, function (err) { + if (err) + callback(err); + else + extractFiles(outPath, entryName, files, callback, 0); + }); + } else { + extractFiles(outPath, entryName, files, callback, 0); + } + } else { + fs.stat(outPath, function(err, stat) { + if (stat && stat.isDirectory()) + extract(entry, path.join(outPath, path.basename(entry.name)), callback); + else + extract(entry, outPath, callback); + }); + } + }; + + this.close = function(callback) { + if (closed) { + if (callback) + callback(); + } else { + closed = true; + fs.close(fd, function(err) { + fd = null; + if (callback) + callback(err); + }); + } + }; + + var originalEmit = events.EventEmitter.prototype.emit; + this.emit = function() { + if (!closed) { + return originalEmit.apply(this, arguments); + } + }; +}; + +StreamZip.setFs = function(customFs) { + fs = customFs; +}; + +util.inherits(StreamZip, events.EventEmitter); + +// endregion + +// region CentralDirectoryHeader + +var CentralDirectoryHeader = function() { +}; + +CentralDirectoryHeader.prototype.read = function(data) { + if (data.length != consts.ENDHDR || data.readUInt32LE(0) != consts.ENDSIG) + throw new Error('Invalid central directory'); + // number of entries on this volume + this.volumeEntries = data.readUInt16LE(consts.ENDSUB); + // total number of entries + this.totalEntries = data.readUInt16LE(consts.ENDTOT); + // central directory size in bytes + this.size = data.readUInt32LE(consts.ENDSIZ); + // offset of first CEN header + this.offset = data.readUInt32LE(consts.ENDOFF); + // zip file comment length + this.commentLength = data.readUInt16LE(consts.ENDCOM); +}; + +// endregion + +// region CentralDirectoryLoc64Header + +var CentralDirectoryLoc64Header = function() { +}; + +CentralDirectoryLoc64Header.prototype.read = function(data) { + if (data.length != consts.ENDL64HDR || data.readUInt32LE(0) != consts.ENDL64SIG) + throw new Error('Invalid zip64 central directory locator'); + // ZIP64 EOCD header offset + this.headerOffset = Util.readUInt64LE(data, consts.ENDSUB); +}; + +// endregion + +// region CentralDirectoryZip64Header + +var CentralDirectoryZip64Header = function() { +}; + +CentralDirectoryZip64Header.prototype.read = function(data) { + if (data.length != consts.END64HDR || data.readUInt32LE(0) != consts.END64SIG) + throw new Error('Invalid central directory'); + // number of entries on this volume + this.volumeEntries = Util.readUInt64LE(data, consts.END64SUB); + // total number of entries + this.totalEntries = Util.readUInt64LE(data, consts.END64TOT); + // central directory size in bytes + this.size = Util.readUInt64LE(data, consts.END64SIZ); + // offset of first CEN header + this.offset = Util.readUInt64LE(data, consts.END64OFF); +}; + +// endregion + +// region ZipEntry + +var ZipEntry = function() { +}; + +function toBits(dec, size) { + var b = (dec >>> 0).toString(2); + while (b.length < size) + b = '0' + b; + return b.split(''); +} + +function parseZipTime(timebytes, datebytes) { + var timebits = toBits(timebytes, 16); + var datebits = toBits(datebytes, 16); + + var mt = { + h: parseInt(timebits.slice(0,5).join(''), 2), + m: parseInt(timebits.slice(5,11).join(''), 2), + s: parseInt(timebits.slice(11,16).join(''), 2) * 2, + Y: parseInt(datebits.slice(0,7).join(''), 2) + 1980, + M: parseInt(datebits.slice(7,11).join(''), 2), + D: parseInt(datebits.slice(11,16).join(''), 2), + }; + var dt_str = [mt.Y, mt.M, mt.D].join('-') + ' ' + [mt.h, mt.m, mt.s].join(':') + ' GMT+0'; + return new Date(dt_str).getTime(); +} + +ZipEntry.prototype.readHeader = function(data, offset) { + // data should be 46 bytes and start with "PK 01 02" + if (data.length < offset + consts.CENHDR || data.readUInt32LE(offset) != consts.CENSIG) { + throw new Error('Invalid entry header'); + } + // version made by + this.verMade = data.readUInt16LE(offset + consts.CENVEM); + // version needed to extract + this.version = data.readUInt16LE(offset + consts.CENVER); + // encrypt, decrypt flags + this.flags = data.readUInt16LE(offset + consts.CENFLG); + // compression method + this.method = data.readUInt16LE(offset + consts.CENHOW); + // modification time (2 bytes time, 2 bytes date) + var timebytes = data.readUInt16LE(offset + consts.CENTIM); + var datebytes = data.readUInt16LE(offset + consts.CENTIM + 2); + this.time = parseZipTime(timebytes, datebytes); + + // uncompressed file crc-32 value + this.crc = data.readUInt32LE(offset + consts.CENCRC); + // compressed size + this.compressedSize = data.readUInt32LE(offset + consts.CENSIZ); + // uncompressed size + this.size = data.readUInt32LE(offset + consts.CENLEN); + // filename length + this.fnameLen = data.readUInt16LE(offset + consts.CENNAM); + // extra field length + this.extraLen = data.readUInt16LE(offset + consts.CENEXT); + // file comment length + this.comLen = data.readUInt16LE(offset + consts.CENCOM); + // volume number start + this.diskStart = data.readUInt16LE(offset + consts.CENDSK); + // internal file attributes + this.inattr = data.readUInt16LE(offset + consts.CENATT); + // external file attributes + this.attr = data.readUInt32LE(offset + consts.CENATX); + // LOC header offset + this.offset = data.readUInt32LE(offset + consts.CENOFF); +}; + +ZipEntry.prototype.readDataHeader = function(data) { + // 30 bytes and should start with "PK\003\004" + if (data.readUInt32LE(0) != consts.LOCSIG) { + throw new Error('Invalid local header'); + } + // version needed to extract + this.version = data.readUInt16LE(consts.LOCVER); + // general purpose bit flag + this.flags = data.readUInt16LE(consts.LOCFLG); + // compression method + this.method = data.readUInt16LE(consts.LOCHOW); + // modification time (2 bytes time ; 2 bytes date) + var timebytes = data.readUInt16LE(consts.LOCTIM); + var datebytes = data.readUInt16LE(consts.LOCTIM + 2); + this.time = parseZipTime(timebytes, datebytes); + + // uncompressed file crc-32 value + this.crc = data.readUInt32LE(consts.LOCCRC) || this.crc; + // compressed size + var compressedSize = data.readUInt32LE(consts.LOCSIZ); + if (compressedSize && compressedSize !== consts.EF_ZIP64_OR_32) { + this.compressedSize = compressedSize; + } + // uncompressed size + var size = data.readUInt32LE(consts.LOCLEN); + if (size && size !== consts.EF_ZIP64_OR_32) { + this.size = size; + } + // filename length + this.fnameLen = data.readUInt16LE(consts.LOCNAM); + // extra field length + this.extraLen = data.readUInt16LE(consts.LOCEXT); +}; + +ZipEntry.prototype.read = function(data, offset) { + this.nameRaw = data.slice(offset, offset += this.fnameLen); + this.name = this.nameRaw.toString(); + var lastChar = data[offset - 1]; + this.isDirectory = (lastChar == 47) || (lastChar == 92); + + if (this.extraLen) { + this.readExtra(data, offset); + offset += this.extraLen; + } + this.comment = this.comLen ? data.slice(offset, offset + this.comLen).toString() : null; +}; + +ZipEntry.prototype.validateName = function() { + if (/\\|^\w+:|^\/|(^|\/)\.\.(\/|$)/.test(this.name)) { + throw new Error('Malicious entry: ' + this.name); + } +}; + +ZipEntry.prototype.readExtra = function(data, offset) { + var signature, size, maxPos = offset + this.extraLen; + while (offset < maxPos) { + signature = data.readUInt16LE(offset); + offset += 2; + size = data.readUInt16LE(offset); + offset += 2; + if (consts.ID_ZIP64 === signature) { + this.parseZip64Extra(data, offset, size); + } + offset += size; + } +}; + +ZipEntry.prototype.parseZip64Extra = function(data, offset, length) { + if (length >= 8 && this.size === consts.EF_ZIP64_OR_32) { + this.size = Util.readUInt64LE(data, offset); + offset += 8; length -= 8; + } + if (length >= 8 && this.compressedSize === consts.EF_ZIP64_OR_32) { + this.compressedSize = Util.readUInt64LE(data, offset); + offset += 8; length -= 8; + } + if (length >= 8 && this.offset === consts.EF_ZIP64_OR_32) { + this.offset = Util.readUInt64LE(data, offset); + offset += 8; length -= 8; + } + if (length >= 4 && this.diskStart === consts.EF_ZIP64_OR_16) { + this.diskStart = data.readUInt32LE(offset); + // offset += 4; length -= 4; + } +}; + +Object.defineProperty(ZipEntry.prototype, 'encrypted', { + get: function() { return (this.flags & consts.FLG_ENTRY_ENC) == consts.FLG_ENTRY_ENC; } +}); + +Object.defineProperty(ZipEntry.prototype, 'isFile', { + get: function() { return !this.isDirectory; } +}); + +// endregion + +// region FsRead + +var FsRead = function(fd, buffer, offset, length, position, callback) { + this.fd = fd; + this.buffer = buffer; + this.offset = offset; + this.length = length; + this.position = position; + this.callback = callback; + this.bytesRead = 0; + this.waiting = false; +}; + +FsRead.prototype.read = function(sync) { + if (StreamZip.debug) { + console.log('read', this.position, this.bytesRead, this.length, this.offset); + } + this.waiting = true; + var err; + if (sync) { + try { + var bytesRead = fs.readSync(this.fd, this.buffer, this.offset + this.bytesRead, + this.length - this.bytesRead, this.position + this.bytesRead); + } catch (e) { + err = e; + } + this.readCallback(sync, err, err ? bytesRead : null); + } else { + fs.read(this.fd, this.buffer, this.offset + this.bytesRead, + this.length - this.bytesRead, this.position + this.bytesRead, + this.readCallback.bind(this, sync)); + } +}; + +FsRead.prototype.readCallback = function(sync, err, bytesRead) { + if (typeof bytesRead === 'number') + this.bytesRead += bytesRead; + if (err || !bytesRead || this.bytesRead === this.length) { + this.waiting = false; + return this.callback(err, this.bytesRead); + } else { + this.read(sync); + } +}; + +// endregion + +// region FileWindowBuffer + +var FileWindowBuffer = function(fd) { + this.position = 0; + this.buffer = Buffer.alloc(0); + + var fsOp = null; + + this.checkOp = function() { + if (fsOp && fsOp.waiting) + throw new Error('Operation in progress'); + }; + + this.read = function(pos, length, callback) { + this.checkOp(); + if (this.buffer.length < length) + this.buffer = Buffer.alloc(length); + this.position = pos; + fsOp = new FsRead(fd, this.buffer, 0, length, this.position, callback).read(); + }; + + this.expandLeft = function(length, callback) { + this.checkOp(); + this.buffer = Buffer.concat([Buffer.alloc(length), this.buffer]); + this.position -= length; + if (this.position < 0) + this.position = 0; + fsOp = new FsRead(fd, this.buffer, 0, length, this.position, callback).read(); + }; + + this.expandRight = function(length, callback) { + this.checkOp(); + var offset = this.buffer.length; + this.buffer = Buffer.concat([this.buffer, Buffer.alloc(length)]); + fsOp = new FsRead(fd, this.buffer, offset, length, this.position + offset, callback).read(); + }; + + this.moveRight = function(length, callback, shift) { + this.checkOp(); + if (shift) { + this.buffer.copy(this.buffer, 0, shift); + } else { + shift = 0; + } + this.position += shift; + fsOp = new FsRead(fd, this.buffer, this.buffer.length - shift, shift, this.position + this.buffer.length - shift, callback).read(); + }; +}; + +// endregion + +// region EntryDataReaderStream + +var EntryDataReaderStream = function(fd, offset, length) { + stream.Readable.prototype.constructor.call(this); + this.fd = fd; + this.offset = offset; + this.length = length; + this.pos = 0; + this.readCallback = this.readCallback.bind(this); +}; + +util.inherits(EntryDataReaderStream, stream.Readable); + +EntryDataReaderStream.prototype._read = function(n) { + var buffer = Buffer.alloc(Math.min(n, this.length - this.pos)); + if (buffer.length) { + fs.read(this.fd, buffer, 0, buffer.length, this.offset + this.pos, this.readCallback); + } else { + this.push(null); + } +}; + +EntryDataReaderStream.prototype.readCallback = function(err, bytesRead, buffer) { + this.pos += bytesRead; + if (err) { + this.emit('error', err); + this.push(null); + } else if (!bytesRead) { + this.push(null); + } else { + if (bytesRead !== buffer.length) + buffer = buffer.slice(0, bytesRead); + this.push(buffer); + } +}; + +// endregion + +// region EntryVerifyStream + +var EntryVerifyStream = function(baseStm, crc, size) { + stream.Transform.prototype.constructor.call(this); + this.verify = new CrcVerify(crc, size); + var that = this; + baseStm.on('error', function(e) { + that.emit('error', e); + }); +}; + +util.inherits(EntryVerifyStream, stream.Transform); + +EntryVerifyStream.prototype._transform = function(data, encoding, callback) { + var err; + try { + this.verify.data(data); + } catch (e) { + err = e; + } + callback(err, data); +}; + +// endregion + +// region CrcVerify + +var CrcVerify = function(crc, size) { + this.crc = crc; + this.size = size; + this.state = { + crc: ~0, + size: 0 + }; +}; + +CrcVerify.prototype.data = function(data) { + var crcTable = CrcVerify.getCrcTable(); + var crc = this.state.crc, off = 0, len = data.length; + while (--len >= 0) + crc = crcTable[(crc ^ data[off++]) & 0xff] ^ (crc >>> 8); + this.state.crc = crc; + this.state.size += data.length; + if (this.state.size >= this.size) { + var buf = Buffer.alloc(4); + buf.writeInt32LE(~this.state.crc & 0xffffffff, 0); + crc = buf.readUInt32LE(0); + if (crc !== this.crc) + throw new Error('Invalid CRC'); + if (this.state.size !== this.size) + throw new Error('Invalid size'); + } +}; + +CrcVerify.getCrcTable = function() { + var crcTable = CrcVerify.crcTable; + if (!crcTable) { + CrcVerify.crcTable = crcTable = []; + var b = Buffer.alloc(4); + for (var n = 0; n < 256; n++) { + var c = n; + for (var k = 8; --k >= 0; ) + if ((c & 1) != 0) { c = 0xedb88320 ^ (c >>> 1); } else { c = c >>> 1; } + if (c < 0) { + b.writeInt32LE(c, 0); + c = b.readUInt32LE(0); + } + crcTable[n] = c; + } + } + return crcTable; +}; + +// endregion + +// region Util + +var Util = { + readUInt64LE: function(buffer, offset) { + return (buffer.readUInt32LE(offset + 4) * 0x0000000100000000) + buffer.readUInt32LE(offset); + } +}; + +// endregion + +// region exports + +module.exports = StreamZip; + +// endregion