diff --git a/server/config/base.js b/server/config/base.js index 7760a3ce..5e581f91 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -39,6 +39,11 @@ module.exports = { ], jembaDb: [ + { + dbName: 'app', + thread: true, + openAll: true, + }, { dbName: 'reader-storage', thread: true, diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 16c79a89..a51565ef 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -7,6 +7,8 @@ const FileDownloader = require('../FileDownloader'); const FileDecompressor = require('../FileDecompressor'); const BookConverter = require('./BookConverter'); const RemoteStorage = require('../RemoteStorage'); +const JembaConnManager = require('../../db/JembaConnManager');//singleton +const ayncExit = new (require('../AsyncExit'))(); const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton @@ -33,6 +35,9 @@ class ReaderWorker { this.decomp = new FileDecompressor(3*config.maxUploadFileSize); this.bookConverter = new BookConverter(this.config); + this.connManager = new JembaConnManager(); + this.appDb = this.connManager.db['app']; + this.remoteStorage = false; if (config.remoteStorage) { this.remoteStorage = new RemoteStorage( @@ -248,12 +253,7 @@ class ReaderWorker { } async cleanDir(dir, remoteDir, maxSize, moveToRemote) { - if (!this.remoteSent) - this.remoteSent = {}; - if (!this.remoteSent[remoteDir]) - this.remoteSent[remoteDir] = {}; - - const sent = this.remoteSent[remoteDir]; + const sent = this.remoteSent; const list = await fs.readdir(dir); @@ -267,12 +267,17 @@ class ReaderWorker { files.push({name: filePath, stat}); } } + log(`clean dir ${dir}, maxSize=${maxSize}, found ${files.length} files, total size=${size}`); files.sort((a, b) => a.stat.mtimeMs - b.stat.mtimeMs); + //удаленное хранилище if (moveToRemote && this.remoteStorage) { + const foundFiles = new Set(); for (const file of files) { + foundFiles.add(file.name); + if (sent[file.name]) continue; @@ -280,11 +285,23 @@ class ReaderWorker { try { log(`remoteStorage.putFile ${remoteDir}/${path.basename(file.name)}`); await this.remoteStorage.putFile(file.name, remoteDir); + sent[file.name] = true; + await this.appDb.insert({table: 'remote_sent', ignore: true, rows: [{id: file.name, remoteDir}]}); } catch (e) { log(LM_ERR, e.stack); } } + + //почистим remoteSent и БД + //несколько неоптимально, таскает все записи из БД + const rows = await this.appDb.select({table: 'remote_sent'}); + for (const row of rows) { + if (row.remoteDir === remoteDir && !foundFiles.has(row.id)) { + delete sent[row.id]; + await this.appDb.delete({table: 'remote_sent', where: `@@id(${this.appDb.esc(row.id)})`}); + } + } } let i = 0; @@ -298,27 +315,44 @@ class ReaderWorker { || (moveToRemote && this.remoteStorage && sent[oldFile]) || size > maxSize*1.5) { await fs.remove(oldFile); - delete sent[oldFile]; j++; } size -= file.stat.size; i++; } + log(`removed ${j} files`); } async periodicCleanDir(cleanConfig) { - while (1) {// eslint-disable-line no-constant-condition - for (const [remoteDir, config] of Object.entries(cleanConfig)) { - try { - await this.cleanDir(config.dir, remoteDir, config.maxSize, config.moveToRemote); - } catch(e) { - log(LM_ERR, e.stack); + try { + if (!this.remoteSent) + this.remoteSent = {}; + + //инициализация this.remoteSent + if (this.remoteStorage) { + const rows = await this.appDb.select({table: 'remote_sent'}); + for (const row of rows) { + this.remoteSent[row.id] = true; } } - await utils.sleep(cleanDirPeriod); + while (1) {// eslint-disable-line no-constant-condition + + for (const [remoteDir, config] of Object.entries(cleanConfig)) { + try { + await this.cleanDir(config.dir, remoteDir, config.maxSize, config.moveToRemote); + } catch(e) { + log(LM_ERR, e.stack); + } + } + + await utils.sleep(cleanDirPeriod); + } + } catch (e) { + log(LM_FATAL, e.message); + ayncExit.exit(1); } } diff --git a/server/core/RemoteStorage.js b/server/core/RemoteStorage.js index f380bb5d..9af5a13d 100644 --- a/server/core/RemoteStorage.js +++ b/server/core/RemoteStorage.js @@ -13,10 +13,10 @@ class RemoteStorage { this.wsc = new WebSocketConnection(config.url); } - async wsQuery(query) { + async wsRequest(query) { const response = await this.wsc.message( - await this.wsc.send(Object.assign({accessToken: this.accessToken}, query), 30), - 300 + await this.wsc.send(Object.assign({accessToken: this.accessToken}, query), 600), + 600 ); if (response.error) throw new Error(response.error); @@ -24,19 +24,19 @@ class RemoteStorage { } async wsStat(fileName) { - return await this.wsQuery({action: 'get-stat', fileName}); + return await this.wsRequest({action: 'get-stat', fileName}); } async wsGetFile(fileName) { - return this.wsQuery({action: 'get-file', fileName}); + return this.wsRequest({action: 'get-file', fileName}); } async wsPutFile(fileName, data) {//data base64 encoded string - return this.wsQuery({action: 'put-file', fileName, data}); + return this.wsRequest({action: 'put-file', fileName, data}); } async wsDelFile(fileName) { - return this.wsQuery({action: 'del-file', fileName}); + return this.wsRequest({action: 'del-file', fileName}); } makeRemoteFileName(fileName, dir = '') { diff --git a/server/db/jembaMigrations/app/001-create.js b/server/db/jembaMigrations/app/001-create.js new file mode 100644 index 00000000..d1451d0a --- /dev/null +++ b/server/db/jembaMigrations/app/001-create.js @@ -0,0 +1,12 @@ +module.exports = { + up: [ + ['create', { + table: 'remote_sent' + }], + ], + down: [ + ['drop', { + table: 'remote_sent' + }], + ] +}; diff --git a/server/db/jembaMigrations/app/index.js b/server/db/jembaMigrations/app/index.js new file mode 100644 index 00000000..85941af0 --- /dev/null +++ b/server/db/jembaMigrations/app/index.js @@ -0,0 +1,6 @@ +module.exports = { + table: 'migration1', + data: [ + {id: 1, name: 'create', data: require('./001-create')} + ] +} \ No newline at end of file diff --git a/server/db/jembaMigrations/index.js b/server/db/jembaMigrations/index.js index 21cd2dcd..b5726b5e 100644 --- a/server/db/jembaMigrations/index.js +++ b/server/db/jembaMigrations/index.js @@ -1,4 +1,5 @@ module.exports = { + 'app': require('./app'), 'reader-storage': require('./reader-storage'), 'book-update-server': require('./book-update-server'), };