liberama/server/core/LibSharedStorage/MegaStorage.js

const _ = require('lodash');
const fs = require('fs-extra');
const path = require('path');

const log = new (require('../AppLogger'))().log;//singleton
const ZipStreamer = require('../ZipStreamer');

const utils = require('../utils');

const zeroStats = {
    zipFilesCount: 0,
    descFilesCount: 0,
    zipFilesSize: 0,
    descFilesSize: 0,
};

let instance = null;

//singleton
class MegaStorage {
    constructor() {
        if (!instance) {
            this.inited = false;

            this.debouncedSaveStats = _.debounce(() => {
                this.saveStats().catch((e) => {
                    log(LM_ERR, `MegaStorage::saveStats ${e.message}`);
                    //process.exit(1);
                });
            }, 5000, {'maxWait':6000});

            process.on('exit', () => {
                this.saveStatsSync();
            });


            instance = this;
        }

        return instance;
    }

    async init(config) {
        this.config = config;
        this.megaStorageDir = config.megaStorageDir;
        this.statsPath = `${this.megaStorageDir}/stats.json`;
        this.compressLevel = (config.compressLevel ? config.compressLevel : 4);
        await fs.ensureDir(this.megaStorageDir);

        this.readingFiles = false;
        this.stats = _.cloneDeep(zeroStats);

        if (await fs.pathExists(this.statsPath)) {
            this.stats = Object.assign({},
                this.stats,
                JSON.parse(await fs.readFile(this.statsPath, 'utf8'))
            );
        }

        this.inited = true;
    }

    async nameHash(filename) {
        if (!this.inited)
            throw new Error('not inited');
        const hash = utils.toBase36(await utils.getFileHash(filename, 'sha1'));
        const hashPath = `${hash.substr(0, 2)}/${hash.substr(2, 2)}/${hash}`;
        const fullHashPath = `${this.megaStorageDir}/${hashPath}`;
        return {
            filename,
            hash,
            hashPath,
            fullHashPath,
            zipPath: `${fullHashPath}.zip`,
            descPath: `${fullHashPath}.desc`,
        };
    }

    async checkFileExists(nameHash) {
        return await fs.pathExists(nameHash.zipPath);
    }

    async addFile(nameHash, desc = null, force = false) {
        if (!this.inited)
            throw new Error('not inited');
        if (await this.checkFileExists(nameHash) && !force)
            return false;

        await fs.ensureDir(path.dirname(nameHash.zipPath));
        let oldZipSize = 0;
        let newZipCount = 1;
        if (await fs.pathExists(nameHash.zipPath)) {
            oldZipSize = (await fs.stat(nameHash.zipPath)).size;
            newZipCount = 0;
        }

        const zip = new ZipStreamer();
        let entry = {};
        let resultFile = await zip.pack(nameHash.zipPath, [nameHash.filename], {zlib: {level: this.compressLevel}}, (ent) => {
            entry = ent;
        });

        if (desc) {
            desc = Object.assign({}, desc, {fileSize: entry.size, zipFileSize: resultFile.size});
            await this.updateDesc(nameHash, desc);
        }

        this.stats.zipFilesSize += -oldZipSize + resultFile.size;
        this.stats.zipFilesCount += newZipCount;
        this.needSaveStats = true;

        this.debouncedSaveStats();
        return desc;
    }

    async updateDesc(nameHash, desc) {
        let oldDescSize = 0;
        let newDescCount = 1;
        if (await fs.pathExists(nameHash.descPath)) {
            oldDescSize = (await fs.stat(nameHash.descPath)).size;
            newDescCount = 0;
        }

        const data = JSON.stringify(desc, null, 2);
        await fs.writeFile(nameHash.descPath, data);

        this.stats.descFilesSize += -oldDescSize + data.length;
        this.stats.descFilesCount += newDescCount;
        this.needSaveStats = true;

        this.debouncedSaveStats();
    }

    async _findFiles(callback, dir) {
        if (!callback || !this.readingFiles)
            return;

        let result = true;
        const files = await fs.readdir(dir, { withFileTypes: true });
        for (const file of files) {
            if (!this.readingFiles)
                return;
            const found = path.resolve(dir, file.name);
            if (file.isDirectory())
                result = await this._findFiles(callback, found);
            else
                await callback(found);
        }
        return result;
    }

    async startFindFiles(callback) {
        if (!this.inited)
            throw new Error('not inited');
        this.readingFiles = true;
        try {
            return await this._findFiles(callback, this.megaStorageDir);
        } finally {
            this.readingFiles = false;
        }
    }

    async stopFindFiles() {
        this.readingFiles = false;
    }

    async saveStats() {
        if (this.needSaveStats) {
            await fs.writeFile(this.statsPath, JSON.stringify(this.stats, null, 2));
            this.needSaveStats = false;
        }
    }

    saveStatsSync() {
        if (this.needSaveStats) {
            fs.writeFileSync(this.statsPath, JSON.stringify(this.stats, null, 2));
            this.needSaveStats = false;
        }
    }

    async getStats(gather = false) {
        if (!this.inited)
            throw new Error('MegaStorage::not inited');
        if (!gather || this.readingFiles)
            return this.stats;

        let stats = _.cloneDeep(zeroStats);
        const result = await this.startFindFiles(async(entry) => {
            if (path.extname(entry) == '.zip') {
                stats.zipFilesSize += (await fs.stat(entry)).size;
                stats.zipFilesCount++;
            }

            if (path.extname(entry) == '.desc') {
                stats.descFilesSize += (await fs.stat(entry)).size;
                stats.descFilesCount++;
            }
        });

        if (result) {
            this.stats = stats;
            this.needSaveStats = true;
            this.debouncedSaveStats();
        }
        return this.stats;
    }
}

module.exports = MegaStorage;