Работа над BookUpdateChecker

server/core/BookUpdateChecker/BUCClient.js

@@ -51,7 +51,7 @@ class BUCClient {
 
         try {
             this.fromCheckTime = await this.findMaxCheckTime();
-            this.periodicSync();//no await
+            //this.periodicSync();//no await
 
             log(`BUC Client started`);
         } catch (e) {

server/core/BookUpdateChecker/BUCServer.js

@@ -34,7 +34,7 @@ class BUCServer {
                 this.fillCheckQueuePeriod = 10*1000;//период пополнения очереди
                 this.periodicCheckWait = 500;//пауза, если нечего делать
 
-                this.cleanQueryInterval = 100*1000;//10*minuteMs;//интервал очистки устаревших
+                this.cleanQueryInterval = 300*dayMs;//интервал очистки устаревших
                 this.oldQueryInterval = 5*minuteMs;//интервал устаревания запроса на обновление
                 this.checkingInterval = 30*1000;//интервал проверки обновления одного и того же файла
                 this.sameHostCheckInterval = 1000;//интервал проверки файла на том же сайте, не менее
@@ -127,10 +127,10 @@ class BUCServer {
 
                 rows = await db.select({table: 'buc', count: true});
                 log(LM_WARN, `'buc' table length: ${rows[0].count}`);
-
+/*
 rows = await db.select({table: 'buc'});
 console.log(rows);
-
+*/
                 now = Date.now();
                 //выборка кандидатов
                 rows = await db.select({
@@ -192,22 +192,39 @@ console.log(rows);
                     this.hostChecking[url.hostname] = true;
 
                     try {
-                        const downdata = await this.down.load(row.id);
+                        let unchanged = true;
-                        const hash = await utils.getBufHash(downdata, 'sha256', 'hex');
+                        let size = 0;
+                        let hash = '';
+
+                        const headers = await this.down.head(row.id);
+                        const modTime = headers['last-modified']
+
+                        if (!modTime || !row.modTime || (modTime !== row.modTime)) {
+                            const downdata = await this.down.load(row.id);
+
+                            size = downdata.length;
+                            hash = await utils.getBufHash(downdata, 'sha256', 'hex');
+                            unchanged = false;
+                        }
 
                         await db.update({
                             table: 'buc',
                             mod: `(r) => {
                                 r.checkTime = ${db.esc(Date.now())};
-                                r.size = ${db.esc(downdata.length)};
+                                r.modTime = ${(unchanged ? 'r.modTime' : db.esc(modTime))};
-                                r.checkSum = ${db.esc(hash)};
+                                r.size = ${(unchanged ? 'r.size' : db.esc(size))};
+                                r.checkSum = ${(unchanged ? 'r.checkSum' : db.esc(hash))};
                                 r.state = 0;
                                 r.error = '';
                             }`,
                             where: `@@id(${db.esc(row.id)})`
                         });
 
-                        log(`checked ${row.id} > size ${downdata.length}`);
+                        if (unchanged) {
+                            log(`checked ${row.id} > unchanged`);
+                        } else {
+                            log(`checked ${row.id} > size ${size}`);
+                        }
                     } catch (e) {
                         await db.update({
                             table: 'buc',
@@ -239,23 +256,7 @@ console.log(rows);
         try {
             //обнуляем все статусы
             await this.db.update({table: 'buc', mod: `(r) => r.state = 0`});
-/*
+
-await this.db.insert({
-    table: 'buc',
-    replace: true,
-    rows: [
-        {
-            id: 'http://old.omnireader.ru/test.txt', // book URL
-            queryTime: Date.now(),
-            checkTime: 0, // 0 - never checked
-            size: 0,
-            checkSum: '', //sha256
-            state: 0, // 0 - not processing, 1 - processing
-            error: '',
-        }
-    ],
-});
-*/
             this.fillCheckQueue();//no await
 
             //10 потоков

server/core/FileDownloader.js

@@ -1,5 +1,7 @@
 const axios = require('axios');
 
+const userAgent = 'Mozilla/5.0 (X11; HasCodingOs 1.0; Linux x64) AppleWebKit/637.36 (KHTML, like Gecko) Chrome/70.0.3112.101 Safari/637.36 HasBrowser/5.0';
+
 class FileDownloader {
     constructor(limitDownloadSize = 0) {
         this.limitDownloadSize = limitDownloadSize;
@@ -10,7 +12,7 @@ class FileDownloader {
 
         const options = {
             headers: {
-                'user-agent': 'Mozilla/5.0 (X11; HasCodingOs 1.0; Linux x64) AppleWebKit/637.36 (KHTML, like Gecko) Chrome/70.0.3112.101 Safari/637.36 HasBrowser/5.0'
+                'user-agent': userAgent
             },
             responseType: 'stream',
         };
@@ -62,6 +64,17 @@ class FileDownloader {
         }
     }
 
+    async head(url) {
+        const options = {
+            headers: {
+                'user-agent': userAgent
+            },
+        };
+
+        const res = await axios.head(url, options);
+        return res.headers;
+    }
+
     streamToBuffer(stream, progress) {
         return new Promise((resolve, reject) => {