From 5122cda6dbd11fcf0bd8b7b172e1235d7c0fbf99 Mon Sep 17 00:00:00 2001 From: Book Pauk Date: Fri, 8 Mar 2019 18:05:58 +0700 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D1=8F?= =?UTF-8?q?=D0=B5=D0=BC=20=D0=BC=D0=B8=D0=B3=D1=80=D0=B0=D1=86=D0=B8=D0=B8?= =?UTF-8?q?=20=D0=B2=20=D0=91=D0=94=20sqlite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/core/SqliteConnectionPool.js | 90 ---------- server/db/SqliteConnectionPool.js | 182 ++++++++++++++++++++ server/{core => db}/connManager.js | 17 ++ server/db/migrations/app/index.js | 5 + server/db/migrations/readerStorage/index.js | 5 + server/index.js | 4 +- 6 files changed, 210 insertions(+), 93 deletions(-) delete mode 100644 server/core/SqliteConnectionPool.js create mode 100644 server/db/SqliteConnectionPool.js rename server/{core => db}/connManager.js (50%) create mode 100644 server/db/migrations/app/index.js create mode 100644 server/db/migrations/readerStorage/index.js diff --git a/server/core/SqliteConnectionPool.js b/server/core/SqliteConnectionPool.js deleted file mode 100644 index 162b80bd..00000000 --- a/server/core/SqliteConnectionPool.js +++ /dev/null @@ -1,90 +0,0 @@ -const utils = require('./utils'); -const sqlite = require('sqlite'); - -const waitingDelay = 100; //ms - -class SqliteConnectionPool { - constructor() { - this.closed = true; - } - - async open(connCount, dbFileName) { - if (!Number.isInteger(connCount) || connCount <= 0) - return; - this.connections = []; - this.taken = new Set(); - this.freed = new Set(); - - for (let i = 0; i < connCount; i++) { - let client = await sqlite.open(dbFileName); - client.configure('busyTimeout', 10000); //ms - - client.ret = () => { - this.taken.delete(i); - this.freed.add(i); - }; - - this.freed.add(i); - this.connections[i] = client; - } - this.closed = false; - } - - _setImmediate() { - return new Promise((resolve) => { - setImmediate(() => { - return resolve(); - }); - }); - } - - async get() { - if (this.closed) - return; - - let freeConnIndex = this.freed.values().next().value; - if (freeConnIndex == null) { - if (waitingDelay) - await utils.sleep(waitingDelay); - return await this._setImmediate().then(() => this.get()); - } - - this.freed.delete(freeConnIndex); - this.taken.add(freeConnIndex); - - return this.connections[freeConnIndex]; - } - - async run(query) { - const dbh = await this.get(); - try { - let result = await dbh.run(query); - dbh.ret(); - return result; - } catch (e) { - dbh.ret(); - throw e; - } - } - - async all(query) { - const dbh = await this.get(); - try { - let result = await dbh.all(query); - dbh.ret(); - return result; - } catch (e) { - dbh.ret(); - throw e; - } - } - - async close() { - for (let i = 0; i < this.connections.length; i++) { - await this.connections[i].close(); - } - this.closed = true; - } -} - -module.exports = SqliteConnectionPool; \ No newline at end of file diff --git a/server/db/SqliteConnectionPool.js b/server/db/SqliteConnectionPool.js new file mode 100644 index 00000000..11c2d633 --- /dev/null +++ b/server/db/SqliteConnectionPool.js @@ -0,0 +1,182 @@ +const sqlite = require('sqlite'); +const SQL = require('sql-template-strings'); + +const utils = require('../core/utils'); + +const waitingDelay = 100; //ms + +class SqliteConnectionPool { + constructor() { + this.closed = true; + } + + async open(connCount, dbFileName) { + if (!Number.isInteger(connCount) || connCount <= 0) + return; + this.connections = []; + this.taken = new Set(); + this.freed = new Set(); + + for (let i = 0; i < connCount; i++) { + let client = await sqlite.open(dbFileName); + client.configure('busyTimeout', 10000); //ms + + client.ret = () => { + this.taken.delete(i); + this.freed.add(i); + }; + + this.freed.add(i); + this.connections[i] = client; + } + this.closed = false; + } + + _setImmediate() { + return new Promise((resolve) => { + setImmediate(() => { + return resolve(); + }); + }); + } + + async get() { + if (this.closed) + return; + + let freeConnIndex = this.freed.values().next().value; + if (freeConnIndex == null) { + if (waitingDelay) + await utils.sleep(waitingDelay); + return await this._setImmediate().then(() => this.get()); + } + + this.freed.delete(freeConnIndex); + this.taken.add(freeConnIndex); + + return this.connections[freeConnIndex]; + } + + async run(query) { + const dbh = await this.get(); + try { + let result = await dbh.run(query); + dbh.ret(); + return result; + } catch (e) { + dbh.ret(); + throw e; + } + } + + async all(query) { + const dbh = await this.get(); + try { + let result = await dbh.all(query); + dbh.ret(); + return result; + } catch (e) { + dbh.ret(); + throw e; + } + } + + async exec(query) { + const dbh = await this.get(); + try { + let result = await dbh.exec(query); + dbh.ret(); + return result; + } catch (e) { + dbh.ret(); + throw e; + } + } + + async close() { + for (let i = 0; i < this.connections.length; i++) { + await this.connections[i].close(); + } + this.closed = true; + } + + // Modified from node-sqlite/.../src/Database.js + async migrate(migs, table, force) { + const migrations = migs.sort((a, b) => Math.sign(a.id - b.id)); + + if (!migrations.length) { + throw new Error('No migration data'); + } + + migrations.map(migration => { + const data = migration.data; + const [up, down] = data.split(/^--\s+?down\b/mi); + if (!down) { + const message = `The ${migration.filename} file does not contain '-- Down' separator.`; + throw new Error(message); + } else { + /* eslint-disable no-param-reassign */ + migration.up = up.replace(/^-- .*?$/gm, '').trim();// Remove comments + migration.down = down.trim(); // and trim whitespaces + } + }); + + // Create a database table for migrations meta data if it doesn't exist + await this.run(`CREATE TABLE IF NOT EXISTS "${table}" ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + up TEXT NOT NULL, + down TEXT NOT NULL +)`); + + // Get the list of already applied migrations + let dbMigrations = await this.all( + `SELECT id, name, up, down FROM "${table}" ORDER BY id ASC`, + ); + + // Undo migrations that exist only in the database but not in migs, + // also undo the last migration if the `force` option was set to `last`. + const lastMigration = migrations[migrations.length - 1]; + for (const migration of dbMigrations.slice().sort((a, b) => Math.sign(b.id - a.id))) { + if (!migrations.some(x => x.id === migration.id) || + (force === 'last' && migration.id === lastMigration.id)) { + await this.run('BEGIN'); + try { + await this.exec(migration.down); + await this.run(SQL`DELETE FROM "`.append(table).append(SQL`" WHERE id = ${migration.id}`)); + await this.run('COMMIT'); + dbMigrations = dbMigrations.filter(x => x.id !== migration.id); + } catch (err) { + await this.run('ROLLBACK'); + throw err; + } + } else { + break; + } + } + + // Apply pending migrations + let applied = []; + const lastMigrationId = dbMigrations.length ? dbMigrations[dbMigrations.length - 1].id : 0; + for (const migration of migrations) { + if (migration.id > lastMigrationId) { + await this.run('BEGIN'); + try { + await this.exec(migration.up); + await this.run(SQL`INSERT INTO "`.append(table).append( + SQL`" (id, name, up, down) VALUES (${migration.id}, ${migration.name}, ${migration.up}, ${migration.down})`) + ); + await this.run('COMMIT'); + applied.push(migration.id); + } catch (err) { + await this.run('ROLLBACK'); + throw err; + } + } + } + + return applied; + } +} + +module.exports = SqliteConnectionPool; \ No newline at end of file diff --git a/server/core/connManager.js b/server/db/connManager.js similarity index 50% rename from server/core/connManager.js rename to server/db/connManager.js index 43eab563..be3e6786 100644 --- a/server/core/connManager.js +++ b/server/db/connManager.js @@ -1,4 +1,10 @@ const SqliteConnectionPool = require('./SqliteConnectionPool'); +const log = require('../core/getLogger').getLog(); + +const migrations = { + 'app': require('./migrations/app'), + 'readerStorage': require('./migrations/readerStorage'), +}; class ConnManager { constructor() { @@ -7,11 +13,22 @@ class ConnManager { async init(config) { this.config = config; + + const force = (config.branch == 'development' ? 'last' : null); + for (const poolConfig of this.config.db) { const dbFileName = this.config.dataDir + '/' + poolConfig.fileName; const connPool = new SqliteConnectionPool(); await connPool.open(poolConfig.connCount, dbFileName); + log(`Opened database "${poolConfig.poolName}"`); + //миграции + const migs = migrations[poolConfig.poolName]; + if (migs && migs.data.length) { + const applied = await connPool.migrate(migs.data, migs.table, force); + log(`Applied ${applied} migrations to "${poolConfig.poolName}"`); + } + this._pool[poolConfig.poolName] = connPool; } } diff --git a/server/db/migrations/app/index.js b/server/db/migrations/app/index.js new file mode 100644 index 00000000..ef075b89 --- /dev/null +++ b/server/db/migrations/app/index.js @@ -0,0 +1,5 @@ +module.exports = { + table: 'migration1', + data: [ + ] +} \ No newline at end of file diff --git a/server/db/migrations/readerStorage/index.js b/server/db/migrations/readerStorage/index.js new file mode 100644 index 00000000..ef075b89 --- /dev/null +++ b/server/db/migrations/readerStorage/index.js @@ -0,0 +1,5 @@ +module.exports = { + table: 'migration1', + data: [ + ] +} \ No newline at end of file diff --git a/server/index.js b/server/index.js index f62d9179..2e05db75 100644 --- a/server/index.js +++ b/server/index.js @@ -11,7 +11,7 @@ const path = require('path'); const express = require('express'); const compression = require('compression'); -const connManager = require('./core/connManager'); +const connManager = require('./db/connManager'); async function init() { await fs.ensureDir(config.dataDir); @@ -35,9 +35,7 @@ async function main() { log('Initializing'); await init(); - log('Opening databases'); await connManager.init(config); - log(`Opened databases: ${Object.keys(connManager.pool).join(', ')}`); //servers for (let server of config.servers) {