diff --git a/client/api/reader.js b/client/api/reader.js index 8031e10a..5b1b8ab4 100644 --- a/client/api/reader.js +++ b/client/api/reader.js @@ -11,7 +11,7 @@ const workerApi = axios.create({ class Reader { async loadBook(url, callback) { - const refreshPause = 200; + const refreshPause = 300; if (!callback) callback = () => {}; let response = await api.post('/load-book', {type: 'url', url}); diff --git a/client/components/Reader/HistoryPage/HistoryPage.vue b/client/components/Reader/HistoryPage/HistoryPage.vue index 7cd98ebb..176b56e8 100644 --- a/client/components/Reader/HistoryPage/HistoryPage.vue +++ b/client/components/Reader/HistoryPage/HistoryPage.vue @@ -226,7 +226,8 @@ class HistoryPage extends Vue { this.updateTableData(); const newRecent = bookManager.mostRecentBook(); - if (this.mostRecentBook != newRecent) + + if (!(this.mostRecentBook && newRecent && this.mostRecentBook.key == newRecent.key)) this.$emit('load-book', newRecent); this.mostRecentBook = newRecent; diff --git a/client/components/Reader/Reader.vue b/client/components/Reader/Reader.vue index 40f9d51e..dc14f10e 100644 --- a/client/components/Reader/Reader.vue +++ b/client/components/Reader/Reader.vue @@ -656,7 +656,7 @@ class Reader extends Vue { // если есть в локальном кэше if (bookParsed) { - await bookManager.setRecentBook(Object.assign({bookPos, bookPosSeen, bookPosPercent}, bookManager.metaOnly(bookParsed))); + await bookManager.setRecentBook(Object.assign({bookPos, bookPosSeen, bookPosPercent}, bookParsed)); this.mostRecentBook(); this.addAction(bookPos); this.loaderActive = false; @@ -699,7 +699,7 @@ class Reader extends Vue { }); // добавляем в историю - await bookManager.setRecentBook(Object.assign({bookPos, bookPosSeen, bookPosPercent}, bookManager.metaOnly(addedBook))); + await bookManager.setRecentBook(Object.assign({bookPos, bookPosSeen, bookPosPercent}, addedBook)); this.mostRecentBook(); this.addAction(bookPos); this.updateRoute(true); diff --git a/client/components/Reader/SettingsPage/SettingsPage.vue b/client/components/Reader/SettingsPage/SettingsPage.vue index bfe0bfc6..477ad5cb 100644 --- a/client/components/Reader/SettingsPage/SettingsPage.vue +++ b/client/components/Reader/SettingsPage/SettingsPage.vue @@ -207,12 +207,12 @@ - Инлайн в центр + Инлайн в центр - Ширина не более размера экрана + Ширина не более размера экрана @@ -225,7 +225,7 @@ уменьшено с сохранением пропорций так, чтобы
помещаться в указанное количество строк. - + diff --git a/client/components/Reader/share/BookParser.js b/client/components/Reader/share/BookParser.js index 3358aaaa..bc6df03b 100644 --- a/client/components/Reader/share/BookParser.js +++ b/client/components/Reader/share/BookParser.js @@ -317,7 +317,7 @@ export default class BookParser { if (!text) return; - text = text.replace(/[\t\n\r]/g, ' '); + text = text.replace(/[\t\n\r\xa0]/g, ' '); switch (path) { case '/fictionbook/description/title-info/author/first-name': @@ -655,7 +655,6 @@ export default class BookParser { let style = {}; let ofs = 0;//смещение от начала параграфа para.offset let imgW = 0; - // тут начинается самый замес, перенос по слогам и стилизация, а также изображения for (const part of parts) { style = part.style; @@ -665,7 +664,7 @@ export default class BookParser { parsed.visible = this.showImages; let bin = this.binary[part.image.id]; if (!bin) - bin = {h: 0, w: 0}; + bin = {h: 1, w: 1}; let lineCount = this.imageHeightLines; let c = Math.ceil(bin.h/this.lineHeight); diff --git a/client/components/Reader/share/bookManager.js b/client/components/Reader/share/bookManager.js index 12215372..bc954965 100644 --- a/client/components/Reader/share/bookManager.js +++ b/client/components/Reader/share/bookManager.js @@ -5,6 +5,10 @@ import BookParser from './BookParser'; const maxDataSize = 500*1024*1024;//chars, not bytes +const bmCacheStore = localForage.createInstance({ + name: 'bmCacheStore' +}); + const bmMetaStore = localForage.createInstance({ name: 'bmMetaStore' }); @@ -20,11 +24,31 @@ const bmRecentStore = localForage.createInstance({ class BookManager { async init(settings) { this.settings = settings; - this.books = {}; - this.recent = {}; + + //this.booksCached нужен только для ускорения загрузки читалки + this.booksCached = await bmCacheStore.getItem('books'); + if (!this.booksCached) + this.booksCached = {}; + this.recent = await bmCacheStore.getItem('recent'); + this.books = Object.assign({}, this.booksCached); + this.recentChanged1 = true; this.recentChanged2 = true; + if (!this.books || !this.recent) { + this.books = {}; + this.recent = {}; + await this.loadMeta(true); + } else { + this.loadMeta(false); + } + } + + + async loadMeta(immediate) { + if (!immediate) + await utils.sleep(2000); + let len = await bmMetaStore.length(); for (let i = 0; i < len; i++) { const key = await bmMetaStore.key(i); @@ -45,6 +69,12 @@ class BookManager { } await this.cleanBooks(); + + this.booksCached = {}; + for (const key in this.books) { + this.booksCached[key] = this.metaOnly(this.books[key]); + } + await bmCacheStore.setItem('books', this.booksCached); } async cleanBooks() { @@ -80,9 +110,11 @@ class BookManager { const result = await this.parseBook(meta, newBook.data, callback); this.books[meta.key] = result; + this.booksCached[meta.key] = this.metaOnly(result); await bmMetaStore.setItem(`bmMeta-${meta.key}`, this.metaOnly(result)); await bmDataStore.setItem(`bmData-${meta.key}`, result.data); + await bmCacheStore.setItem('books', this.booksCached); return result; } @@ -127,11 +159,15 @@ class BookManager { await bmDataStore.removeItem(`bmData-${meta.key}`); delete this.books[meta.key]; + delete this.booksCached[meta.key]; + + await bmCacheStore.setItem('books', this.booksCached); } async parseBook(meta, data, callback) { if (!this.books) await this.init(); + const parsed = new BookParser(this.settings); const parsedMeta = await parsed.parse(data, callback); @@ -159,7 +195,7 @@ class BookManager { async setRecentBook(value, noTouch) { if (!this.recent) await this.init(); - const result = Object.assign({}, value); + const result = this.metaOnly(value); if (!noTouch) Object.assign(result, {touchTime: Date.now()}); @@ -170,6 +206,7 @@ class BookManager { await bmRecentStore.setItem(result.key, result); await this.cleanRecentBooks(); + await bmCacheStore.setItem('recent', this.recent); this.recentChanged1 = true; this.recentChanged2 = true; @@ -188,6 +225,8 @@ class BookManager { await bmRecentStore.removeItem(value.key); delete this.recent[value.key]; + await bmCacheStore.setItem('recent', this.recent); + this.recentChanged1 = true; this.recentChanged2 = true; } diff --git a/package.json b/package.json index f36e036f..7ca81322 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Liberama", - "version": "0.4.6", + "version": "0.4.7", "engines": { "node": ">=10.0.0" }, diff --git a/server/core/BookConverter/ConvertBase.js b/server/core/BookConverter/ConvertBase.js new file mode 100644 index 00000000..460d91c3 --- /dev/null +++ b/server/core/BookConverter/ConvertBase.js @@ -0,0 +1,96 @@ +const iconv = require('iconv-lite'); +const chardet = require('chardet'); +const textUtils = require('./textUtils'); + +class ConvertBase { + constructor(config) { + this.config = config; + } + + run(data, opts) {// eslint-disable-line no-unused-vars + //override + } + + decode(data) { + let selected = textUtils.getEncoding(data); + + if (selected == 'ISO-8859-5') { + const charsetAll = chardet.detectAll(data.slice(0, 20000)); + for (const charset of charsetAll) { + if (charset.name.indexOf('ISO-8859') < 0) { + selected = charset.name; + break; + } + } + } + + if (selected.toLowerCase() != 'utf-8') + return iconv.decode(data, selected); + else + return data; + } + + repSpaces(text) { + return text.replace(/ |[\t\n\r]/g, ' '); + } + + formatFb2(fb2) { + let out = ''; + out += ''; + out += this.formatFb2Node(fb2); + out += ''; + return out; + } + + formatFb2Node(node, name) { + let out = ''; + + if (Array.isArray(node)) { + for (const n of node) { + out += this.formatFb2Node(n); + } + } else if (typeof node == 'string') { + if (name) + out += `<${name}>${this.repSpaces(node)}`; + else + out += this.repSpaces(node); + } else { + if (node._n) + name = node._n; + + let attrs = ''; + if (node._attrs) { + for (let attrName in node._attrs) { + attrs += ` ${attrName}="${node._attrs[attrName]}"`; + } + } + + let tOpen = ''; + let tBody = ''; + let tClose = ''; + if (name) + tOpen += `<${name}${attrs}>`; + if (node.hasOwnProperty('_t')) + tBody += this.repSpaces(node._t); + + for (let nodeName in node) { + if (nodeName && nodeName[0] == '_' && nodeName != '_a') + continue; + + const n = node[nodeName]; + tBody += this.formatFb2Node(n, nodeName); + } + + if (name) + tClose += ``; + + if (attrs == '' && name == 'p' && tBody.trim() == '') + out += '' + else + out += `${tOpen}${tBody}${tClose}`; + } + return out; + } +} + +module.exports = ConvertBase; \ No newline at end of file diff --git a/server/core/BookConverter/ConvertFb2.js b/server/core/BookConverter/ConvertFb2.js new file mode 100644 index 00000000..e4a84866 --- /dev/null +++ b/server/core/BookConverter/ConvertFb2.js @@ -0,0 +1,41 @@ +const ConvertBase = require('./ConvertBase'); +const iconv = require('iconv-lite'); + +class ConvertFb2 extends ConvertBase { + check(data, opts) { + const {fileType} = opts; + + return (fileType && fileType.ext == 'xml' && data.toString().indexOf('= 0); + } + + run(data, opts) { + if (!this.check(data, opts)) + return false; + + return this.checkEncoding(data); + } + + checkEncoding(data) { + let result = data; + + const left = data.indexOf('= 0) { + const right = data.indexOf('?>', left); + if (right >= 0) { + const head = data.slice(left, right + 2).toString(); + const m = head.match(/encoding="(.*)"/); + if (m) { + let encoding = m[1].toLowerCase(); + if (encoding != 'utf-8') { + result = iconv.decode(data, encoding); + result = Buffer.from(result.toString().replace(m[0], 'encoding="utf-8"')); + } + } + } + } + + return result; + } +} + +module.exports = ConvertFb2; diff --git a/server/core/BookConverter/ConvertHtml.js b/server/core/BookConverter/ConvertHtml.js new file mode 100644 index 00000000..fa85937d --- /dev/null +++ b/server/core/BookConverter/ConvertHtml.js @@ -0,0 +1,162 @@ +const ConvertBase = require('./ConvertBase'); +const sax = require('./sax'); +const textUtils = require('./textUtils'); + +class ConvertHtml extends ConvertBase { + check(data, opts) { + const {fileType} = opts; + + if (fileType && (fileType.ext == 'html' || fileType.ext == 'xml')) + return {isText: false}; + + //может это чистый текст? + if (textUtils.checkIfText(data)) { + return {isText: true}; + } + + return false; + } + + run(data, opts) { + const checkResult = this.check(data, opts); + if (!checkResult) + return false; + + let {isText} = checkResult; + let titleInfo = {}; + let desc = {_n: 'description', 'title-info': titleInfo}; + let pars = []; + let body = {_n: 'body', section: {_a: []}}; + let fb2 = [desc, body]; + + let title = ''; + let inTitle = false; + + let spaceCounter = []; + + const repCrLfTab = (text) => text.replace(/[\n\r]/g, '').replace(/\t/g, ' '); + + const newParagraph = () => { + pars.push({_n: 'p', _t: ''}); + }; + + const growParagraph = (text) => { + if (!pars.length) + newParagraph(); + + const l = pars.length; + if (pars[l - 1]._t == '') + text = text.trimLeft(); + pars[l - 1]._t += text; + + //посчитаем отступы у текста, чтобы выделить потом параграфы + const lines = text.split('\n'); + for (let line of lines) { + line = repCrLfTab(line) + + let l = 0; + while (l < line.length && line[l] == ' ') { + l++; + } + if (!spaceCounter[l]) + spaceCounter[l] = 0; + spaceCounter[l]++; + } + }; + + const newPara = new Set(['tr', 'br', 'br/', 'dd', 'p', 'title', '/title', 'h1', 'h2', 'h3', '/h1', '/h2', '/h3']); + + const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars + if (!cutCounter) { + growParagraph(text); + } + + if (inTitle && !title) + title = text; + }; + + const onStartNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars + if (!cutCounter) { + if (newPara.has(tag)) + newParagraph(); + } + + if (tag == 'title') + inTitle = true; + }; + + const onEndNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars + if (tag == 'title') + inTitle = false; + }; + + let buf = this.decode(data).toString(); + + sax.parseSync(buf, { + onStartNode, onEndNode, onTextNode, + innerCut: new Set(['head', 'script', 'style', 'binary']) + }); + + titleInfo['book-title'] = title; + + //подозрение на чистый текст, надо разбить на параграфы + if (isText || pars.length < buf.length/2000) { + let total = 0; + for (let i = 0; i < spaceCounter.length; i++) { + total += (spaceCounter[i] ? spaceCounter[i] : 0); + } + total /= 10; + let i = spaceCounter.length - 1; + while (i > 0 && (!spaceCounter[i] || spaceCounter[i] < total)) i--; + + const parIndent = (i > 0 ? i : 0); + + let newPars = []; + const newPar = () => { + newPars.push({_n: 'p', _t: ''}); + }; + + const growPar = (text) => { + if (!newPars.length) + newPar(); + + const l = newPars.length; + newPars[l - 1]._t += text; + } + + i = 0; + for (const par of pars) { + if (i > 0) + newPar(); + i++; + + const lines = par._t.split('\n'); + for (let line of lines) { + line = repCrLfTab(line); + + let l = 0; + while (l < line.length && line[l] == ' ') { + l++; + } + + if (l >= parIndent) + newPar(); + growPar(line.trim() + ' '); + } + } + + body.section._a[0] = newPars; + } else { + body.section._a[0] = pars; + } + + //убираем лишнее + for (let i = 0; i < pars.length; i++) + pars[i]._t = this.repSpaces(pars[i]._t).trim(); + + return this.formatFb2(fb2); + } + +} + +module.exports = ConvertHtml; diff --git a/server/core/BookConverter/ConvertSamlib.js b/server/core/BookConverter/ConvertSamlib.js new file mode 100644 index 00000000..40ebd10b --- /dev/null +++ b/server/core/BookConverter/ConvertSamlib.js @@ -0,0 +1,275 @@ +const _ = require('lodash'); +const URL = require('url').URL; + +const sax = require('./sax'); +const ConvertBase = require('./ConvertBase'); + +class ConvertSamlib extends ConvertBase { + check(data, opts) { + const {url} = opts; + + const parsedUrl = new URL(url); + if (parsedUrl.hostname == 'samlib.ru' || + parsedUrl.hostname == 'budclub.ru' || + parsedUrl.hostname == 'zhurnal.lib.ru') { + return {hostname: parsedUrl.hostname}; + } + + return false; + } + + run(data, opts) { + const checkResult = this.check(data, opts); + if (!checkResult) + return false; + + const {hostname} = checkResult; + let titleInfo = {}; + let desc = {_n: 'description', 'title-info': titleInfo}; + let pars = []; + let body = {_n: 'body', section: {_a: pars}}; + let fb2 = [desc, body]; + + let inSubtitle = false; + let inJustify = true; + let inImage = false; + let isFirstPara = false; + let path = ''; + let tag = '';// eslint-disable-line no-unused-vars + + let inText = false; + let textFound = false; + let node = {_a: pars}; + + let inPara = false; + let italic = false; + let bold = false; + + const openTag = (name, attrs) => { + if (name == 'p') + inPara = true; + let n = {_n: name, _attrs: attrs, _a: [], _p: node}; + node._a.push(n); + node = n; + }; + + const closeTag = (name) => { + if (name == 'p') + inPara = false; + if (node._p) { + const exact = (node._n == name); + node = node._p; + if (!exact) + closeTag(name); + } + }; + + const growParagraph = (text) => { + if (!node._p) { + if (text.trim() != '') + openTag('p'); + else + return; + } + if (node._n == 'p' && node._a.length == 0) + text = text.trimLeft(); + node._a.push({_t: text}); + }; + + const onStartNode = (elemName, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars + if (elemName == '') + return; + if (!inText) { + path += '/' + elemName; + tag = elemName; + } else { + switch (elemName) { + case 'li': + case 'p': + case 'dd': + case 'br': + if (!(inSubtitle && isFirstPara)) { + if (inPara) + closeTag('p'); + openTag('p'); + } + isFirstPara = false; + break; + case 'h1': + case 'h2': + case 'h3': + if (inPara) + closeTag('p'); + openTag('p'); + bold = true; + break; + case 'i': + case 'em': + italic = true; + break; + case 'b': + case 'strong': + bold = true; + break; + case 'div': + if (inPara) + closeTag('p'); + if (tail.indexOf('align="center"') >= 0) { + openTag('subtitle'); + inSubtitle = true; + isFirstPara = true; + } + + if (tail.indexOf('align="justify"') >= 0) { + openTag('p'); + inJustify = true; + } + + break; + case 'img': { + if (inPara) + closeTag('p'); + const attrs = sax.getAttrsSync(tail); + if (attrs.src && attrs.src.value) { + let href = attrs.src.value; + if (href[0] == '/') + href = `http://${hostname}${href}`; + openTag('image', {href}); + inImage = true; + } + break; + } + } + } + }; + + const onEndNode = (elemName, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars + if (!inText) { + const oldPath = path; + let t = ''; + do { + let i = path.lastIndexOf('/'); + t = path.substr(i + 1); + path = path.substr(0, i); + } while (t != elemName && path); + + if (t != elemName) { + path = oldPath; + } + + let i = path.lastIndexOf('/'); + tag = path.substr(i + 1); + } else { + switch (elemName) { + case 'li': + case 'p': + case 'dd': + closeTag('p'); + break; + case 'h1': + case 'h2': + case 'h3': + closeTag('p'); + bold = false; + break; + case 'i': + case 'em': + italic = false; + break; + case 'b': + case 'strong': + bold = false; + break; + case 'div': + if (inSubtitle) { + closeTag('subtitle'); + inSubtitle = false; + isFirstPara = false; + } + + if (inJustify) { + closeTag('p'); + inJustify = false; + } + break; + case 'img': + if (inImage) + closeTag('image'); + inImage = false; + break; + } + } + }; + + const onComment = (text) => {// eslint-disable-line no-unused-vars + if (text == '--------- Собственно произведение -------------') { + inText = true; + textFound = true; + } + if (text == '-----------------------------------------------') + inText = false; + }; + + const onTextNode = (text) => {// eslint-disable-line no-unused-vars + if (text && text.trim() == '') + text = (text.indexOf(' ') >= 0 ? ' ' : ''); + + if (!text) + return; + + switch (path) { + case '/html/body/center/h2': + titleInfo['book-title'] = text; + return; + case '/html/body/div/h3': + if (!titleInfo.author) + titleInfo.author = {}; + text = text.replace(':', '').trim().split(' '); + if (text[0]) + titleInfo.author['last-name'] = text[0]; + if (text[1]) + titleInfo.author['first-name'] = text[1]; + if (text[2]) + titleInfo.author['middle-name'] = text[2]; + return; + } + + let tOpen = (bold ? '' : ''); + tOpen += (italic ? '' : ''); + let tClose = (italic ? '' : ''); + tClose += (bold ? '' : ''); + + if (inText) + growParagraph(`${tOpen}${text}${tClose}`); + }; + + sax.parseSync(this.decode(data).toString().replace(/ /g, ' '), { + onStartNode, onEndNode, onTextNode, onComment, + innerCut: new Set(['head', 'script', 'style']) + }); + + //текст не найден на странице, обработать корректно не получилось + if (!textFound) + return false; + + const title = (titleInfo['book-title'] ? titleInfo['book-title'] : ''); + let author = ''; + if (titleInfo.author) { + author = _.compact([ + (titleInfo.author['last-name'] ? titleInfo.author['last-name'] : ''), + (titleInfo.author['first-name'] ? titleInfo.author['first-name'] : ''), + (titleInfo.author['middle-name'] ? titleInfo.author['middle-name'] : ''), + ]).join(' '); + } + + pars.unshift({_n: 'title', _a: [ + {_n: 'p', _t: author}, {_n: 'p', _t: ''}, + {_n: 'p', _t: title}, {_n: 'p', _t: ''}, + ]}) + + return this.formatFb2(fb2); + } + +} + +module.exports = ConvertSamlib; diff --git a/server/core/BookConverter/index.js b/server/core/BookConverter/index.js index 657ed90b..8d9a6fd3 100644 --- a/server/core/BookConverter/index.js +++ b/server/core/BookConverter/index.js @@ -1,536 +1,45 @@ const fs = require('fs-extra'); -const URL = require('url').URL; -const iconv = require('iconv-lite'); -const chardet = require('chardet'); -const _ = require('lodash'); -const sax = require('./sax'); -const textUtils = require('./textUtils'); - const FileDetector = require('../FileDetector'); -const repSpaces = (text) => text.replace(/ |[\t\n\r]/g, ' '); -const repSpaces2 = (text) => text.replace(/[\n\r]/g, ''); -const repSpaces3 = (text) => text.replace(/ /g, ' '); +//порядок важен +const convertClassFactory = [ + require('./ConvertFb2'), + require('./ConvertSamlib'), + require('./ConvertHtml'), +]; class BookConverter { - constructor() { + constructor(config) { this.detector = new FileDetector(); + + this.convertFactory = []; + for (const convertClass of convertClassFactory) { + this.convertFactory.push(new convertClass(config)); + } } async convertToFb2(inputFile, outputFile, url, callback) { const fileType = await this.detector.detectFile(inputFile); const data = await fs.readFile(inputFile); - callback(100); - - if (fileType && (fileType.ext == 'html' || fileType.ext == 'xml')) { - if (data.toString().indexOf('= 0) { - await fs.writeFile(outputFile, this.checkEncoding(data)); - return; + let result = false; + for (const convert of this.convertFactory) { + result = convert.run(data, {inputFile, url, callback, fileType}); + if (result) { + await fs.writeFile(outputFile, result); + break; } + } - const parsedUrl = new URL(url); - if (parsedUrl.hostname == 'samlib.ru' || - parsedUrl.hostname == 'budclub.ru' || - parsedUrl.hostname == 'zhurnal.lib.ru') { - await fs.writeFile(outputFile, this.convertSamlib(data, parsedUrl.hostname)); - return; - } - - await fs.writeFile(outputFile, this.convertHtml(data)); - return; - } else { + if (!result) { if (fileType) throw new Error(`Этот формат файла не поддерживается: ${fileType.mime}`); else { - //может это чистый текст? - if (textUtils.checkIfText(data)) { - await fs.writeFile(outputFile, this.convertHtml(data, true)); - return; - } - throw new Error(`Не удалось определить формат файла: ${url}`); } } - } - decode(data) { - let selected = textUtils.getEncoding(data); - - if (selected == 'ISO-8859-5') { - const charsetAll = chardet.detectAll(data.slice(0, 20000)); - for (const charset of charsetAll) { - if (charset.name.indexOf('ISO-8859') < 0) { - selected = charset.name; - break; - } - } - } - - if (selected.toLowerCase() != 'utf-8') - return iconv.decode(data, selected); - else - return data; - } - - checkEncoding(data) { - let result = data; - - const left = data.indexOf('= 0) { - const right = data.indexOf('?>', left); - if (right >= 0) { - const head = data.slice(left, right + 2).toString(); - const m = head.match(/encoding="(.*)"/); - if (m) { - let encoding = m[1].toLowerCase(); - if (encoding != 'utf-8') { - result = iconv.decode(data, encoding); - result = Buffer.from(result.toString().replace(m[0], 'encoding="utf-8"')); - } - } - } - } - - return result; - } - - convertHtml(data, isText) { - let titleInfo = {}; - let desc = {_n: 'description', 'title-info': titleInfo}; - let pars = []; - let body = {_n: 'body', section: {_a: []}}; - let fb2 = [desc, body]; - - let title = ''; - let inTitle = false; - - let spaceCounter = []; - - const newParagraph = () => { - pars.push({_n: 'p', _t: ''}); - }; - - const growParagraph = (text) => { - if (!pars.length) - newParagraph(); - - const l = pars.length; - if (pars[l - 1]._t == '') - text = text.trimLeft(); - pars[l - 1]._t += text; - - //посчитаем отступы у текста, чтобы выделить потом параграфы - const lines = text.split('\n'); - for (let line of lines) { - line = repSpaces2(line).replace(/\t/g, ' '); - - let l = 0; - while (l < line.length && line[l] == ' ') { - l++; - } - if (!spaceCounter[l]) - spaceCounter[l] = 0; - spaceCounter[l]++; - } - }; - - const newPara = new Set(['tr', 'br', 'br/', 'dd', 'p', 'title', '/title', 'h1', 'h2', 'h3', '/h1', '/h2', '/h3']); - - const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars - if (!cutCounter) { - growParagraph(text); - } - - if (inTitle && !title) - title = text; - }; - - const onStartNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars - if (!cutCounter) { - if (newPara.has(tag)) - newParagraph(); - } - - if (tag == 'title') - inTitle = true; - }; - - const onEndNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars - if (tag == 'title') - inTitle = false; - }; - - let buf = this.decode(data).toString(); - - sax.parseSync(buf, { - onStartNode, onEndNode, onTextNode, - innerCut: new Set(['head', 'script', 'style']) - }); - - titleInfo['book-title'] = title; - - //подозрение на чистый текст, надо разбить на параграфы - if (isText || pars.length < buf.length/2000) { - let total = 0; - for (let i = 0; i < spaceCounter.length; i++) { - total += (spaceCounter[i] ? spaceCounter[i] : 0); - } - total /= 10; - let i = spaceCounter.length - 1; - while (i > 0 && (!spaceCounter[i] || spaceCounter[i] < total)) i--; - - const parIndent = (i > 0 ? i : 0); - - let newPars = []; - const newPar = () => { - newPars.push({_n: 'p', _t: ''}); - }; - - const growPar = (text) => { - if (!newPars.length) - newPar(); - - const l = newPars.length; - newPars[l - 1]._t += text; - } - - i = 0; - for (const par of pars) { - if (i > 0) - newPar(); - i++; - - const lines = par._t.split('\n'); - for (let line of lines) { - line = repSpaces2(line).replace(/\t/g, ' '); - - let l = 0; - while (l < line.length && line[l] == ' ') { - l++; - } - - if (l >= parIndent) - newPar(); - growPar(line.trim() + ' '); - } - } - - body.section._a[0] = newPars; - } else { - body.section._a[0] = pars; - } - - //убираем лишнее - for (let i = 0; i < pars.length; i++) - pars[i]._t = repSpaces(pars[i]._t).trim(); - - return this.formatFb2(fb2); - } - - convertSamlib(data, hostname) { - let titleInfo = {}; - let desc = {_n: 'description', 'title-info': titleInfo}; - let pars = []; - let body = {_n: 'body', section: {_a: pars}}; - let fb2 = [desc, body]; - - let inSubtitle = false; - let inJustify = true; - let inImage = false; - let isFirstPara = false; - let path = ''; - let tag = '';// eslint-disable-line no-unused-vars - - let inText = false; - let textFound = false; - let node = {_a: pars}; - - let inPara = false; - let italic = false; - let bold = false; - - const openTag = (name, attrs) => { - if (name == 'p') - inPara = true; - let n = {_n: name, _attrs: attrs, _a: [], _p: node}; - node._a.push(n); - node = n; - }; - - const closeTag = (name) => { - if (name == 'p') - inPara = false; - if (node._p) { - const exact = (node._n == name); - node = node._p; - if (!exact) - closeTag(name); - } - }; - - const growParagraph = (text) => { - if (!node._p) { - if (text.trim() != '') - openTag('p'); - else - return; - } - if (node._n == 'p' && node._a.length == 0) - text = text.trimLeft(); - node._a.push({_t: text}); - }; - - const onStartNode = (elemName, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars - if (elemName == '') - return; - if (!inText) { - path += '/' + elemName; - tag = elemName; - } else { - switch (elemName) { - case 'li': - case 'p': - case 'dd': - case 'br': - if (!(inSubtitle && isFirstPara)) { - if (inPara) - closeTag('p'); - openTag('p'); - } - isFirstPara = false; - break; - case 'h1': - case 'h2': - case 'h3': - if (inPara) - closeTag('p'); - openTag('p'); - bold = true; - break; - case 'i': - case 'em': - italic = true; - break; - case 'b': - case 'strong': - bold = true; - break; - case 'div': - if (inPara) - closeTag('p'); - if (tail.indexOf('align="center"') >= 0) { - openTag('subtitle'); - inSubtitle = true; - isFirstPara = true; - } - - if (tail.indexOf('align="justify"') >= 0) { - openTag('p'); - inJustify = true; - } - - break; - case 'img': { - if (inPara) - closeTag('p'); - const attrs = sax.getAttrsSync(tail); - if (attrs.src && attrs.src.value) { - let href = attrs.src.value; - if (href[0] == '/') - href = `http://${hostname}${href}`; - openTag('image', {href}); - inImage = true; - } - break; - } - } - } - }; - - const onEndNode = (elemName, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars - if (!inText) { - const oldPath = path; - let t = ''; - do { - let i = path.lastIndexOf('/'); - t = path.substr(i + 1); - path = path.substr(0, i); - } while (t != elemName && path); - - if (t != elemName) { - path = oldPath; - } - - let i = path.lastIndexOf('/'); - tag = path.substr(i + 1); - } else { - switch (elemName) { - case 'li': - case 'p': - case 'dd': - closeTag('p'); - break; - case 'h1': - case 'h2': - case 'h3': - closeTag('p'); - bold = false; - break; - case 'i': - case 'em': - italic = false; - break; - case 'b': - case 'strong': - bold = false; - break; - case 'div': - if (inSubtitle) { - closeTag('subtitle'); - inSubtitle = false; - isFirstPara = false; - } - - if (inJustify) { - closeTag('p'); - inJustify = false; - } - break; - case 'img': - if (inImage) - closeTag('image'); - inImage = false; - break; - } - } - }; - - const onComment = (text) => {// eslint-disable-line no-unused-vars - if (text == '--------- Собственно произведение -------------') { - inText = true; - textFound = true; - } - if (text == '-----------------------------------------------') - inText = false; - }; - - const onTextNode = (text) => {// eslint-disable-line no-unused-vars - if (text && text.trim() == '') - text = (text.indexOf(' ') >= 0 ? ' ' : ''); - - if (!text) - return; - - switch (path) { - case '/html/body/center/h2': - titleInfo['book-title'] = text; - return; - case '/html/body/div/h3': - if (!titleInfo.author) - titleInfo.author = {}; - text = text.replace(':', '').trim().split(' '); - if (text[0]) - titleInfo.author['last-name'] = text[0]; - if (text[1]) - titleInfo.author['first-name'] = text[1]; - if (text[2]) - titleInfo.author['middle-name'] = text[2]; - return; - } - - let tOpen = (bold ? '' : ''); - tOpen += (italic ? '' : ''); - let tClose = (italic ? '' : ''); - tClose += (bold ? '' : ''); - - if (inText) - growParagraph(`${tOpen}${text}${tClose}`); - }; - - sax.parseSync(repSpaces3(this.decode(data).toString()), { - onStartNode, onEndNode, onTextNode, onComment, - innerCut: new Set(['head', 'script', 'style']) - }); - - //текст не найден на странице, обрабатываем как html - if (!textFound) - return this.convertHtml(data); - - const title = (titleInfo['book-title'] ? titleInfo['book-title'] : ''); - let author = ''; - if (titleInfo.author) { - author = _.compact([ - (titleInfo.author['last-name'] ? titleInfo.author['last-name'] : ''), - (titleInfo.author['first-name'] ? titleInfo.author['first-name'] : ''), - (titleInfo.author['middle-name'] ? titleInfo.author['middle-name'] : ''), - ]).join(' '); - } - - pars.unshift({_n: 'title', _a: [ - {_n: 'p', _t: author}, {_n: 'p', _t: ''}, - {_n: 'p', _t: title}, {_n: 'p', _t: ''}, - ]}) - - return this.formatFb2(fb2); - } - - formatFb2(fb2) { - let out = ''; - out += ''; - out += this.formatFb2Node(fb2); - out += ''; - return out; - } - - formatFb2Node(node, name) { - let out = ''; - - if (Array.isArray(node)) { - for (const n of node) { - out += this.formatFb2Node(n); - } - } else if (typeof node == 'string') { - if (name) - out += `<${name}>${repSpaces(node)}`; - else - out += repSpaces(node); - } else { - if (node._n) - name = node._n; - - let attrs = ''; - if (node._attrs) { - for (let attrName in node._attrs) { - attrs += ` ${attrName}="${node._attrs[attrName]}"`; - } - } - - let tOpen = ''; - let tBody = ''; - let tClose = ''; - if (name) - tOpen += `<${name}${attrs}>`; - if (node.hasOwnProperty('_t')) - tBody += repSpaces(node._t); - - for (let nodeName in node) { - if (nodeName && nodeName[0] == '_' && nodeName != '_a') - continue; - - const n = node[nodeName]; - tBody += this.formatFb2Node(n, nodeName); - } - - if (name) - tClose += ``; - - if (attrs == '' && name == 'p' && tBody.trim() == '') - out += '' - else - out += `${tOpen}${tBody}${tClose}`; - } - return out; + callback(100); } } diff --git a/server/core/FileDetector.js b/server/core/FileDetector.js index 806389c3..692465aa 100644 --- a/server/core/FileDetector.js +++ b/server/core/FileDetector.js @@ -9,18 +9,18 @@ detect.addSignature( "rules": [ { "type": "or", "rules": [ - { "type": "contains", "bytes": "3c68746d6c" }, - { "type": "contains", "bytes": "3c00680074006d006c00" }, + { "type": "equal", "end": 5, "bytes": "3c68746d6c" }, + { "type": "equal", "end": 10, "bytes": "3c00680074006d006c00" }, - { "type": "contains", "bytes": "3c21646f6374797065" }, - { "type": "contains", "bytes": "3c626f6479" }, - { "type": "contains", "bytes": "3c68656164" }, - { "type": "contains", "bytes": "3c696672616d65" }, - { "type": "contains", "bytes": "3c696d67" }, - { "type": "contains", "bytes": "3c6f626a656374" }, - { "type": "contains", "bytes": "3c736372697074" }, - { "type": "contains", "bytes": "3c7461626c65" }, - { "type": "contains", "bytes": "3c7469746c65" }, + { "type": "equal", "end": 9, "bytes": "3c21646f6374797065" }, + { "type": "equal", "end": 5, "bytes": "3c626f6479" }, + { "type": "equal", "end": 5, "bytes": "3c68656164" }, + { "type": "equal", "end": 7, "bytes": "3c696672616d65" }, + { "type": "equal", "end": 4, "bytes": "3c696d67" }, + { "type": "equal", "end": 7, "bytes": "3c6f626a656374" }, + { "type": "equal", "end": 7, "bytes": "3c736372697074" }, + { "type": "equal", "end": 6, "bytes": "3c7461626c65" }, + { "type": "equal", "end": 6, "bytes": "3c7469746c65" }, ] } ] @@ -36,7 +36,7 @@ detect.addSignature( "rules": [ { "type": "or", "rules": [ - { "type": "contains", "bytes": "3c3f786d6c2076657273696f6e3d22312e3022" }, + { "type": "equal", "end": 19, "bytes": "3c3f786d6c2076657273696f6e3d22312e3022" }, ] } ] diff --git a/server/core/ReaderWorker.js b/server/core/ReaderWorker.js index a087da8c..5692f999 100644 --- a/server/core/ReaderWorker.js +++ b/server/core/ReaderWorker.js @@ -22,7 +22,7 @@ class ReaderWorker { this.down = new FileDownloader(); this.decomp = new FileDecompressor(); - this.bookConverter = new BookConverter(); + this.bookConverter = new BookConverter(this.config); if (!singleCleanExecute) { this.periodicCleanDir(this.config.tempPublicDir, this.config.maxTempPublicDirSize, 60*60*1000);//1 раз в час @@ -66,14 +66,14 @@ class ReaderWorker { const decompFilename = await this.decomp.decompressFile(downloadedFilename, decompDir); wState.set({progress: 100}); - //parse book + //конвертирование в fb2 wState.set({state: 'convert', step: 3, progress: 0}); convertFilename = `${this.config.tempDownloadDir}/${tempFilename2}`; await this.bookConverter.convertToFb2(decompFilename, convertFilename, url, progress => { wState.set({progress}); }); - //compress file to tmp dir, if not exists with the same hashname + //сжимаем файл в tmp, если там уже нет с тем же именем-sha256 const compFilename = await this.decomp.gzipFileIfNotExists(convertFilename, `${this.config.tempPublicDir}`); wState.set({progress: 100}); diff --git a/server/index.js b/server/index.js index 47b6620a..04da3b09 100644 --- a/server/index.js +++ b/server/index.js @@ -1,7 +1,7 @@ const config = require('./config'); -const {initLogger, getLog} = require('./core/getLogger'); -initLogger(config); -const log = getLog(); +const logger = require('./core/getLogger'); +logger.initLogger(config); +const log = logger.getLog(); const configSaver = require('./config/configSaver'); const argv = require('minimist')(process.argv.slice(2));