diff --git a/server/core/BookUpdateChecker/BUCClient.js b/server/core/BookUpdateChecker/BUCClient.js index 96c6780d..3379c00c 100644 --- a/server/core/BookUpdateChecker/BUCClient.js +++ b/server/core/BookUpdateChecker/BUCClient.js @@ -51,7 +51,7 @@ class BUCClient { try { this.fromCheckTime = await this.findMaxCheckTime(); - this.periodicSync();//no await + //this.periodicSync();//no await log(`BUC Client started`); } catch (e) { diff --git a/server/core/BookUpdateChecker/BUCServer.js b/server/core/BookUpdateChecker/BUCServer.js index 4a9c6457..e83bea14 100644 --- a/server/core/BookUpdateChecker/BUCServer.js +++ b/server/core/BookUpdateChecker/BUCServer.js @@ -34,7 +34,7 @@ class BUCServer { this.fillCheckQueuePeriod = 10*1000;//период пополнения очереди this.periodicCheckWait = 500;//пауза, если нечего делать - this.cleanQueryInterval = 100*1000;//10*minuteMs;//интервал очистки устаревших + this.cleanQueryInterval = 300*dayMs;//интервал очистки устаревших this.oldQueryInterval = 5*minuteMs;//интервал устаревания запроса на обновление this.checkingInterval = 30*1000;//интервал проверки обновления одного и того же файла this.sameHostCheckInterval = 1000;//интервал проверки файла на том же сайте, не менее @@ -127,10 +127,10 @@ class BUCServer { rows = await db.select({table: 'buc', count: true}); log(LM_WARN, `'buc' table length: ${rows[0].count}`); - +/* rows = await db.select({table: 'buc'}); console.log(rows); - +*/ now = Date.now(); //выборка кандидатов rows = await db.select({ @@ -192,22 +192,39 @@ console.log(rows); this.hostChecking[url.hostname] = true; try { - const downdata = await this.down.load(row.id); - const hash = await utils.getBufHash(downdata, 'sha256', 'hex'); + let unchanged = true; + let size = 0; + let hash = ''; + + const headers = await this.down.head(row.id); + const modTime = headers['last-modified'] + + if (!modTime || !row.modTime || (modTime !== row.modTime)) { + const downdata = await this.down.load(row.id); + + size = downdata.length; + hash = await utils.getBufHash(downdata, 'sha256', 'hex'); + unchanged = false; + } await db.update({ table: 'buc', mod: `(r) => { r.checkTime = ${db.esc(Date.now())}; - r.size = ${db.esc(downdata.length)}; - r.checkSum = ${db.esc(hash)}; + r.modTime = ${(unchanged ? 'r.modTime' : db.esc(modTime))}; + r.size = ${(unchanged ? 'r.size' : db.esc(size))}; + r.checkSum = ${(unchanged ? 'r.checkSum' : db.esc(hash))}; r.state = 0; r.error = ''; }`, where: `@@id(${db.esc(row.id)})` }); - log(`checked ${row.id} > size ${downdata.length}`); + if (unchanged) { + log(`checked ${row.id} > unchanged`); + } else { + log(`checked ${row.id} > size ${size}`); + } } catch (e) { await db.update({ table: 'buc', @@ -239,23 +256,7 @@ console.log(rows); try { //обнуляем все статусы await this.db.update({table: 'buc', mod: `(r) => r.state = 0`}); -/* -await this.db.insert({ - table: 'buc', - replace: true, - rows: [ - { - id: 'http://old.omnireader.ru/test.txt', // book URL - queryTime: Date.now(), - checkTime: 0, // 0 - never checked - size: 0, - checkSum: '', //sha256 - state: 0, // 0 - not processing, 1 - processing - error: '', - } - ], -}); -*/ + this.fillCheckQueue();//no await //10 потоков diff --git a/server/core/FileDownloader.js b/server/core/FileDownloader.js index 0e54e21e..a308bb63 100644 --- a/server/core/FileDownloader.js +++ b/server/core/FileDownloader.js @@ -1,5 +1,7 @@ const axios = require('axios'); +const userAgent = 'Mozilla/5.0 (X11; HasCodingOs 1.0; Linux x64) AppleWebKit/637.36 (KHTML, like Gecko) Chrome/70.0.3112.101 Safari/637.36 HasBrowser/5.0'; + class FileDownloader { constructor(limitDownloadSize = 0) { this.limitDownloadSize = limitDownloadSize; @@ -10,7 +12,7 @@ class FileDownloader { const options = { headers: { - 'user-agent': 'Mozilla/5.0 (X11; HasCodingOs 1.0; Linux x64) AppleWebKit/637.36 (KHTML, like Gecko) Chrome/70.0.3112.101 Safari/637.36 HasBrowser/5.0' + 'user-agent': userAgent }, responseType: 'stream', }; @@ -62,6 +64,17 @@ class FileDownloader { } } + async head(url) { + const options = { + headers: { + 'user-agent': userAgent + }, + }; + + const res = await axios.head(url, options); + return res.headers; + } + streamToBuffer(stream, progress) { return new Promise((resolve, reject) => {