From 6ddb97d43eacddbaa1205c3785f97d099954fe42 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 17 Jul 2022 17:11:34 +0700 Subject: [PATCH 1/3] =?UTF-8?q?=D0=A2=D1=8E=D0=BD=D0=B8=D0=BD=D0=B3=20?= =?UTF-8?q?=D1=82=D0=B0=D0=B9=D0=BC=D0=B0=D1=83=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/RemoteStorage.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/core/RemoteStorage.js b/server/core/RemoteStorage.js index f380bb5d..0807801f 100644 --- a/server/core/RemoteStorage.js +++ b/server/core/RemoteStorage.js @@ -15,8 +15,8 @@ class RemoteStorage { async wsQuery(query) { const response = await this.wsc.message( - await this.wsc.send(Object.assign({accessToken: this.accessToken}, query), 30), - 300 + await this.wsc.send(Object.assign({accessToken: this.accessToken}, query), 600), + 600 ); if (response.error) throw new Error(response.error); From e65a8a13eae89c6c1c000081c7a97f21bb55ff2b Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 17 Jul 2022 20:04:23 +0700 Subject: [PATCH 2/3] =?UTF-8?q?=D0=A0=D0=B5=D1=84=D0=B0=D0=BA=D1=82=D0=BE?= =?UTF-8?q?=D1=80=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/RemoteStorage.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/core/RemoteStorage.js b/server/core/RemoteStorage.js index 0807801f..9af5a13d 100644 --- a/server/core/RemoteStorage.js +++ b/server/core/RemoteStorage.js @@ -13,7 +13,7 @@ class RemoteStorage { this.wsc = new WebSocketConnection(config.url); } - async wsQuery(query) { + async wsRequest(query) { const response = await this.wsc.message( await this.wsc.send(Object.assign({accessToken: this.accessToken}, query), 600), 600 @@ -24,19 +24,19 @@ class RemoteStorage { } async wsStat(fileName) { - return await this.wsQuery({action: 'get-stat', fileName}); + return await this.wsRequest({action: 'get-stat', fileName}); } async wsGetFile(fileName) { - return this.wsQuery({action: 'get-file', fileName}); + return this.wsRequest({action: 'get-file', fileName}); } async wsPutFile(fileName, data) {//data base64 encoded string - return this.wsQuery({action: 'put-file', fileName, data}); + return this.wsRequest({action: 'put-file', fileName, data}); } async wsDelFile(fileName) { - return this.wsQuery({action: 'del-file', fileName}); + return this.wsRequest({action: 'del-file', fileName}); } makeRemoteFileName(fileName, dir = '') { From 4fd9d579e0d0f6a88d26e0b523e5841b65ff0479 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Sun, 17 Jul 2022 21:10:52 +0700 Subject: [PATCH 3/3] =?UTF-8?q?=D0=9D=D0=B5=D0=B1=D0=BE=D0=BB=D1=8C=D1=88?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=B4=D0=BE=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=BA?= =?UTF-8?q?=D0=B8=20remoteSent,=20=D0=BE=D0=BF=D1=82=D0=B8=D0=BC=D0=B8?= =?UTF-8?q?=D0=B7=D0=B0=D1=86=D0=B8=D1=8F=20=D0=BE=D1=82=D0=BF=D1=80=D0=B0?= =?UTF-8?q?=D0=B2=D0=BA=D0=B8=20=D1=84=D0=B0=D0=B9=D0=BB=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/config/base.js | 5 ++ server/core/Reader/ReaderWorker.js | 62 ++++++++++++++++----- server/db/jembaMigrations/app/001-create.js | 12 ++++ server/db/jembaMigrations/app/index.js | 6 ++ server/db/jembaMigrations/index.js | 1 + 5 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 server/db/jembaMigrations/app/001-create.js create mode 100644 server/db/jembaMigrations/app/index.js diff --git a/server/config/base.js b/server/config/base.js index 7760a3ce..5e581f91 100644 --- a/server/config/base.js +++ b/server/config/base.js @@ -39,6 +39,11 @@ module.exports = { ], jembaDb: [ + { + dbName: 'app', + thread: true, + openAll: true, + }, { dbName: 'reader-storage', thread: true, diff --git a/server/core/Reader/ReaderWorker.js b/server/core/Reader/ReaderWorker.js index 16c79a89..a51565ef 100644 --- a/server/core/Reader/ReaderWorker.js +++ b/server/core/Reader/ReaderWorker.js @@ -7,6 +7,8 @@ const FileDownloader = require('../FileDownloader'); const FileDecompressor = require('../FileDecompressor'); const BookConverter = require('./BookConverter'); const RemoteStorage = require('../RemoteStorage'); +const JembaConnManager = require('../../db/JembaConnManager');//singleton +const ayncExit = new (require('../AsyncExit'))(); const utils = require('../utils'); const log = new (require('../AppLogger'))().log;//singleton @@ -33,6 +35,9 @@ class ReaderWorker { this.decomp = new FileDecompressor(3*config.maxUploadFileSize); this.bookConverter = new BookConverter(this.config); + this.connManager = new JembaConnManager(); + this.appDb = this.connManager.db['app']; + this.remoteStorage = false; if (config.remoteStorage) { this.remoteStorage = new RemoteStorage( @@ -248,12 +253,7 @@ class ReaderWorker { } async cleanDir(dir, remoteDir, maxSize, moveToRemote) { - if (!this.remoteSent) - this.remoteSent = {}; - if (!this.remoteSent[remoteDir]) - this.remoteSent[remoteDir] = {}; - - const sent = this.remoteSent[remoteDir]; + const sent = this.remoteSent; const list = await fs.readdir(dir); @@ -267,12 +267,17 @@ class ReaderWorker { files.push({name: filePath, stat}); } } + log(`clean dir ${dir}, maxSize=${maxSize}, found ${files.length} files, total size=${size}`); files.sort((a, b) => a.stat.mtimeMs - b.stat.mtimeMs); + //удаленное хранилище if (moveToRemote && this.remoteStorage) { + const foundFiles = new Set(); for (const file of files) { + foundFiles.add(file.name); + if (sent[file.name]) continue; @@ -280,11 +285,23 @@ class ReaderWorker { try { log(`remoteStorage.putFile ${remoteDir}/${path.basename(file.name)}`); await this.remoteStorage.putFile(file.name, remoteDir); + sent[file.name] = true; + await this.appDb.insert({table: 'remote_sent', ignore: true, rows: [{id: file.name, remoteDir}]}); } catch (e) { log(LM_ERR, e.stack); } } + + //почистим remoteSent и БД + //несколько неоптимально, таскает все записи из БД + const rows = await this.appDb.select({table: 'remote_sent'}); + for (const row of rows) { + if (row.remoteDir === remoteDir && !foundFiles.has(row.id)) { + delete sent[row.id]; + await this.appDb.delete({table: 'remote_sent', where: `@@id(${this.appDb.esc(row.id)})`}); + } + } } let i = 0; @@ -298,27 +315,44 @@ class ReaderWorker { || (moveToRemote && this.remoteStorage && sent[oldFile]) || size > maxSize*1.5) { await fs.remove(oldFile); - delete sent[oldFile]; j++; } size -= file.stat.size; i++; } + log(`removed ${j} files`); } async periodicCleanDir(cleanConfig) { - while (1) {// eslint-disable-line no-constant-condition - for (const [remoteDir, config] of Object.entries(cleanConfig)) { - try { - await this.cleanDir(config.dir, remoteDir, config.maxSize, config.moveToRemote); - } catch(e) { - log(LM_ERR, e.stack); + try { + if (!this.remoteSent) + this.remoteSent = {}; + + //инициализация this.remoteSent + if (this.remoteStorage) { + const rows = await this.appDb.select({table: 'remote_sent'}); + for (const row of rows) { + this.remoteSent[row.id] = true; } } - await utils.sleep(cleanDirPeriod); + while (1) {// eslint-disable-line no-constant-condition + + for (const [remoteDir, config] of Object.entries(cleanConfig)) { + try { + await this.cleanDir(config.dir, remoteDir, config.maxSize, config.moveToRemote); + } catch(e) { + log(LM_ERR, e.stack); + } + } + + await utils.sleep(cleanDirPeriod); + } + } catch (e) { + log(LM_FATAL, e.message); + ayncExit.exit(1); } } diff --git a/server/db/jembaMigrations/app/001-create.js b/server/db/jembaMigrations/app/001-create.js new file mode 100644 index 00000000..d1451d0a --- /dev/null +++ b/server/db/jembaMigrations/app/001-create.js @@ -0,0 +1,12 @@ +module.exports = { + up: [ + ['create', { + table: 'remote_sent' + }], + ], + down: [ + ['drop', { + table: 'remote_sent' + }], + ] +}; diff --git a/server/db/jembaMigrations/app/index.js b/server/db/jembaMigrations/app/index.js new file mode 100644 index 00000000..85941af0 --- /dev/null +++ b/server/db/jembaMigrations/app/index.js @@ -0,0 +1,6 @@ +module.exports = { + table: 'migration1', + data: [ + {id: 1, name: 'create', data: require('./001-create')} + ] +} \ No newline at end of file diff --git a/server/db/jembaMigrations/index.js b/server/db/jembaMigrations/index.js index 21cd2dcd..b5726b5e 100644 --- a/server/db/jembaMigrations/index.js +++ b/server/db/jembaMigrations/index.js @@ -1,4 +1,5 @@ module.exports = { + 'app': require('./app'), 'reader-storage': require('./reader-storage'), 'book-update-server': require('./book-update-server'), };