diff --git a/server/core/FileDecompressor.js b/server/core/FileDecompressor.js index 95a4a46b..993f93ef 100644 --- a/server/core/FileDecompressor.js +++ b/server/core/FileDecompressor.js @@ -1,6 +1,5 @@ const fs = require('fs-extra'); const zlib = require('zlib'); -const crypto = require('crypto'); const path = require('path'); const unbzip2Stream = require('unbzip2-stream'); const tar = require('tar-fs'); @@ -208,15 +207,26 @@ class FileDecompressor { }); } - async gzipFileIfNotExists(filename, outDir) { - const buf = await fs.readFile(filename); + async gzipFile(inputFile, outputFile) { + return new Promise((resolve, reject) => { + const gzip = zlib.createGzip({level: 1}); + const input = fs.createReadStream(inputFile); + const output = fs.createWriteStream(outputFile); - const hash = crypto.createHash('sha256').update(buf).digest('hex'); + input.pipe(gzip).pipe(output).on('finish', (err) => { + if (err) reject(err); + else resolve(); + }); + }); + } + + async gzipFileIfNotExists(filename, outDir) { + const hash = await utils.getFileHash(filename, 'sha256', 'hex'); const outFilename = `${outDir}/${hash}`; if (!await fs.pathExists(outFilename)) { - await fs.writeFile(outFilename, await this.gzipBuffer(buf)) + await this.gzipFile(filename, outFilename); } else { await utils.touchFile(outFilename); } diff --git a/server/core/ReaderWorker.js b/server/core/ReaderWorker.js index 24038693..1191e21f 100644 --- a/server/core/ReaderWorker.js +++ b/server/core/ReaderWorker.js @@ -1,6 +1,5 @@ const fs = require('fs-extra'); const path = require('path'); -const crypto = require('crypto'); const workerState = require('./workerState'); const FileDownloader = require('./FileDownloader'); @@ -9,6 +8,8 @@ const BookConverter = require('./BookConverter'); const utils = require('./utils'); const log = require('./getLogger').getLog(); +const LibSharedStorage = require('./LibSharedStorage'); + let singleCleanExecute = false; class ReaderWorker { @@ -30,6 +31,12 @@ class ReaderWorker { this.periodicCleanDir(this.config.uploadDir, this.config.maxUploadPublicDirSize, 60*60*1000);//1 раз в час singleCleanExecute = true; } + + (async() => { + const libSharedStorage = new LibSharedStorage(); + await libSharedStorage.init(config); + libSharedStorage.filenameToStoragePath('/home/sizikov/Downloads/15/1.zip'); + })(); } async loadBook(opts, wState) { @@ -117,10 +124,7 @@ class ReaderWorker { } async saveFile(file) { - const buf = await fs.readFile(file.path); - - const hash = crypto.createHash('sha256').update(buf).digest('hex'); - + const hash = await utils.getFileHash(file.path, 'sha256', 'hex'); const outFilename = `${this.config.uploadDir}/${hash}`; if (!await fs.pathExists(outFilename)) { diff --git a/server/core/utils.js b/server/core/utils.js index b134cf79..f1ba1125 100644 --- a/server/core/utils.js +++ b/server/core/utils.js @@ -1,6 +1,28 @@ const { spawn } = require('child_process'); const fs = require('fs-extra'); const crypto = require('crypto'); +const baseX = require('base-x'); + +const BASE36 = '0123456789abcdefghijklmnopqrstuvwxyz'; +const bs36 = baseX(BASE36); + +function toBase36(data) { + return bs36.encode(Buffer.from(data)); +} + +function fromBase36(data) { + return bs36.decode(data); +} + +function getFileHash(filename, hashName, enc) { + return new Promise((resolve, reject) => { + const hash = crypto.createHash(hashName); + const rs = fs.createReadStream(filename); + rs.on('error', reject); + rs.on('data', chunk => hash.update(chunk)); + rs.on('end', () => resolve(hash.digest(enc))); + }); +} function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); @@ -54,6 +76,9 @@ function spawnProcess(cmd, opts) { } module.exports = { + toBase36, + fromBase36, + getFileHash, sleep, randomHexString, touchFile,