diff --git a/server/core/LibSharedStorage/MegaStorage.js b/server/core/LibSharedStorage/MegaStorage.js index 4fddbed1..c59b13d6 100644 --- a/server/core/LibSharedStorage/MegaStorage.js +++ b/server/core/LibSharedStorage/MegaStorage.js @@ -1,12 +1,43 @@ +const _ = require('lodash'); const fs = require('fs-extra'); const path = require('path'); + +const log = new (require('../AppLogger'))().log;//singleton const ZipStreamer = require('../ZipStreamer'); const utils = require('../utils'); +const zeroStats = { + zipFilesCount: 0, + descFilesCount: 0, + zipFilesSize: 0, + descFilesSize: 0, +}; + +let instance = null; + +//singleton class MegaStorage { constructor() { - this.inited = false; + if (!instance) { + this.inited = false; + + this.debouncedSaveStats = _.debounce(() => { + this.saveStats().catch((e) => { + log(LM_ERR, `MegaStorage::saveStats ${e.message}`); + //process.exit(1); + }); + }, 5000, {'maxWait':6000}); + + process.on('exit', () => { + this.saveStatsSync(); + }); + + + instance = this; + } + + return instance; } async init(config) { @@ -17,12 +48,12 @@ class MegaStorage { await fs.ensureDir(this.megaStorageDir); this.readingFiles = false; - this.stats = {}; + this.stats = _.cloneDeep(zeroStats); if (await fs.pathExists(this.statsPath)) { this.stats = Object.assign({}, - JSON.parse(await fs.readFile(this.statsPath, 'utf8')), - this.stats + this.stats, + JSON.parse(await fs.readFile(this.statsPath, 'utf8')) ); } @@ -31,7 +62,7 @@ class MegaStorage { async nameHash(filename) { if (!this.inited) - throw new Error('MegaStorage::not inited'); + throw new Error('not inited'); const hash = utils.toBase36(await utils.getFileHash(filename, 'sha1')); const hashPath = `${hash.substr(0, 2)}/${hash.substr(2, 2)}/${hash}`; const fullHashPath = `${this.megaStorageDir}/${hashPath}`; @@ -51,11 +82,18 @@ class MegaStorage { async addFile(nameHash, desc = null, force = false) { if (!this.inited) - throw new Error('MegaStorage::not inited'); + throw new Error('not inited'); if (await this.checkFileExists(nameHash) && !force) return false; await fs.ensureDir(path.dirname(nameHash.zipPath)); + let oldZipSize = 0; + let newZipCount = 1; + if (await fs.pathExists(nameHash.zipPath)) { + oldZipSize = (await fs.stat(nameHash.zipPath)).size; + newZipCount = 0; + } + const zip = new ZipStreamer(); let entry = {}; let resultFile = await zip.pack(nameHash.zipPath, [nameHash.filename], {zlib: {level: this.compressLevel}}, (ent) => { @@ -64,22 +102,40 @@ class MegaStorage { if (desc) { desc = Object.assign({}, desc, {fileSize: entry.size, zipFileSize: resultFile.size}); - this.updateDesc(nameHash, desc); + await this.updateDesc(nameHash, desc); } + + this.stats.zipFilesSize += -oldZipSize + resultFile.size; + this.stats.zipFilesCount += newZipCount; + this.needSaveStats = true; + + this.debouncedSaveStats(); return desc; } async updateDesc(nameHash, desc) { - await fs.writeFile(nameHash.descPath, JSON.stringify(desc, null, 2)); + let oldDescSize = 0; + let newDescCount = 1; + if (await fs.pathExists(nameHash.descPath)) { + oldDescSize = (await fs.stat(nameHash.descPath)).size; + newDescCount = 0; + } + + const data = JSON.stringify(desc, null, 2); + await fs.writeFile(nameHash.descPath, data); + + this.stats.descFilesSize += -oldDescSize + data.length; + this.stats.descFilesCount += newDescCount; + this.needSaveStats = true; + + this.debouncedSaveStats(); } async _findFiles(callback, dir) { if (!callback || !this.readingFiles) return; - if (!dir) - dir = this.megaStorageDir; - let result; + let result = true; const files = await fs.readdir(dir, { withFileTypes: true }); for (const file of files) { if (!this.readingFiles) @@ -88,17 +144,17 @@ class MegaStorage { if (file.isDirectory()) result = await this._findFiles(callback, found); else - callback(found); + await callback(found); } return result; } - async startFindFiles(callback, dir) { + async startFindFiles(callback) { if (!this.inited) - throw new Error('MegaStorage::not inited'); + throw new Error('not inited'); this.readingFiles = true; try { - return await this._findFiles(callback, dir); + return await this._findFiles(callback, this.megaStorageDir); } finally { this.readingFiles = false; } @@ -108,9 +164,45 @@ class MegaStorage { this.readingFiles = false; } + async saveStats() { + if (this.needSaveStats) { + await fs.writeFile(this.statsPath, JSON.stringify(this.stats, null, 2)); + this.needSaveStats = false; + } + } + + saveStatsSync() { + if (this.needSaveStats) { + fs.writeFileSync(this.statsPath, JSON.stringify(this.stats, null, 2)); + this.needSaveStats = false; + } + } + async getStats(gather = false) { if (!this.inited) throw new Error('MegaStorage::not inited'); + if (!gather || this.readingFiles) + return this.stats; + + let stats = _.cloneDeep(zeroStats); + const result = await this.startFindFiles(async(entry) => { + if (path.extname(entry) == '.zip') { + stats.zipFilesSize += (await fs.stat(entry)).size; + stats.zipFilesCount++; + } + + if (path.extname(entry) == '.desc') { + stats.descFilesSize += (await fs.stat(entry)).size; + stats.descFilesCount++; + } + }); + + if (result) { + this.stats = stats; + this.needSaveStats = true; + this.debouncedSaveStats(); + } + return this.stats; } }