Merge branch 'release/0.9.12'

Добавлены соответствующие настройки в читалку.

client/api/reader.js

@@ -66,7 +66,7 @@ class Reader {
                 await utils.sleep(refreshPause);
 
             i++;
-            if (i > 120*1000/refreshPause) {//2 мин ждем телодвижений воркера
+            if (i > 180*1000/refreshPause) {//3 мин ждем телодвижений воркера
                 throw new Error('Слишком долгое время ожидания');
             }
             //проверка воркера
@@ -181,9 +181,8 @@ class Reader {
             maxUploadFileSize = 10*1024*1024;
         if (file.size > maxUploadFileSize)
             throw new Error(`Размер файла превышает ${maxUploadFileSize} байт`);
-
         let formData = new FormData();
-        formData.append('file', file);
+        formData.append('file', file, file.name);
 
         const options = {
             headers: {

client/components/App.vue

@@ -2,7 +2,7 @@
     <div class="fit row">
         <Notify ref="notify"/>
         <StdDialog ref="stdDialog"/>
-        <keep-alive>
+        <keep-alive v-if="showPage">
             <router-view class="col"></router-view>
         </keep-alive>
     </div>
@@ -12,8 +12,11 @@
 //-----------------------------------------------------------------------------
 import Vue from 'vue';
 import Component from 'vue-class-component';
+
 import Notify from './share/Notify.vue';
 import StdDialog from './share/StdDialog.vue';
+
+import miscApi from '../api/misc';
 import * as utils from '../share/utils';
 
 export default @Component({
@@ -30,6 +33,8 @@ export default @Component({
 
 })
 class App extends Vue {
+    showPage = false;
+
     itemRuText = {
         '/cardindex': 'Картотека',
         '/reader': 'Читалка',
@@ -42,7 +47,6 @@ class App extends Vue {
 
     created() {
         this.commit = this.$store.commit;
-        this.dispatch = this.$store.dispatch;
         this.state = this.$store.state;
         this.uistate = this.$store.state.uistate;
         this.config = this.$store.state.config;
@@ -116,18 +120,28 @@ class App extends Vue {
         this.$root.notify = this.$refs.notify;
         this.$root.stdDialog = this.$refs.stdDialog;
 
-        this.dispatch('config/loadConfig');
-        this.$watch('apiError', function(newError) {
-            if (newError) {
-                let mes = newError.message;
-                if (newError.response && newError.response.config)
-                    mes = newError.response.config.url + '<br>' + newError.response.statusText;
-                this.$root.notify.error(mes, 'Ошибка API');
-            }
-        });
-
         this.setAppTitle();
         (async() => {
+            //загрузим конфиг сревера
+            try {
+                const config = await miscApi.loadConfig();
+                this.commit('config/setConfig', config);
+                this.showPage = true;
+            } catch(e) {
+                //проверим, не получен ли конфиг ранее
+                if (!this.mode) {
+                    this.$root.notify.error(e.message, 'Ошибка API');
+                } else {
+                    //вероятно, работаем в оффлайне
+                    this.showPage = true;
+                }
+                console.error(e);
+            }
+
+            //запросим persistent storage
+            if (navigator.storage && navigator.storage.persist) {
+                navigator.storage.persist();
+            }
             await this.routerReady();
             this.redirectIfNeeded();
         })();

client/components/Reader/ContentsPage/ContentsPage.vue

@@ -16,6 +16,7 @@
             class="no-mp bg-grey-4 text-grey-7"
         >
             <q-tab name="contents" icon="la la-list" label="Оглавление" />
+            <q-tab name="images" icon="la la-image" label="Изображения" />
             <q-tab name="bookmarks"  icon="la la-bookmark" label="Закладки" />
         </q-tabs>
     </div>
@@ -25,7 +26,7 @@
     <div class="tab-panel" v-show="selectedTab == 'contents'">
         <div>
             <div v-for="item in contents" :key="item.key" class="column" style="width: 540px">
-                <div class="row item q-px-sm no-wrap">
+                <div class="row q-px-sm no-wrap" :class="{'item': !item.isBookPos, 'item-book-pos': item.isBookPos}">
                     <div v-if="item.list.length" class="row justify-center items-center expand-button clickable" @click="expandClick(item.key)">
                         <q-icon name="la la-caret-right" class="icon" :class="{'expanded-icon': item.expanded}" color="green-8" size="20px"/>
                     </div>
@@ -40,7 +41,7 @@
                 </div>
                 
                 <div v-if="item.expanded" :ref="`subitem${item.key}`" class="subitems-transition">
-                    <div v-for="subitem in item.list" :key="subitem.key" class="row subitem q-px-sm no-wrap">
+                    <div v-for="subitem in item.list" :key="subitem.key" class="row q-px-sm no-wrap" :class="{'subitem': !subitem.isBookPos, 'subitem-book-pos': subitem.isBookPos}">
                         <div class="col row clickable" @click="setBookPos(subitem.offset)">
                             <div class="no-expand-button"></div>
                             <div :style="subitem.indentStyle"></div>
@@ -56,6 +57,33 @@
         </div>
     </div>
 
+    <div class="tab-panel" v-show="selectedTab == 'images'">
+        <div>
+            <div v-for="item in images" :key="item.key" class="column" style="width: 540px">
+                <div class="row q-px-sm no-wrap" :class="{'item': !item.isBookPos, 'item-book-pos': item.isBookPos}">
+                    <div class="col row clickable" @click="setBookPos(item.offset)">
+                        <div class="image-thumb-box row justify-center items-center">
+                            <div v-show="!imageLoaded[item.id]" class="image-thumb column justify-center"><i class="loading-img-icon la la-images"></i></div>
+                            <img v-show="imageLoaded[item.id]" class="image-thumb" :src="imageSrc[item.id]"/>
+                        </div>
+                        <div class="no-expand-button column justify-center items-center">
+                            <div class="image-num">{{ item.num }}</div>
+                            <div v-show="item.type == 'image/jpeg'" class="image-type it-jpg-color row justify-center">JPG</div>
+                            <div v-show="item.type == 'image/png'" class="image-type it-png-color row justify-center">PNG</div>
+                            <div v-show="!item.local" class="image-type it-net-color row justify-center">INET</div>
+                        </div>
+                        <div :style="item.indentStyle"></div>
+                        <div class="q-mr-sm col overflow-hidden column justify-center" :style="item.labelStyle" v-html="item.label"></div>
+                        <div class="column justify-center">{{ item.perc }}%</div>
+                    </div>
+                </div>
+            </div>
+            <div v-if="!images.length" class="column justify-center items-center" style="height: 100px">
+                Изображения отсутствуют
+            </div>
+        </div>
+    </div>
+
     <div class="tab-panel" v-show="selectedTab == 'bookmarks'">
         <div class="column justify-center items-center" style="height: 100px">
             Раздел находится в разработке
@@ -74,16 +102,29 @@ import Component from 'vue-class-component';
 import Window from '../../share/Window.vue';
 import * as utils from '../../../share/utils';
 
+const ContentsPageProps = Vue.extend({
+    props: {
+        bookPos: Number,
+        isVisible: Boolean,
+    }
+});
+
 export default @Component({
     components: {
         Window,
     },
     watch: {
+        bookPos: function() {
+            this.updateBookPosSelection();
+        }
     },
 })
-class ContentsPage extends Vue {
+class ContentsPage extends ContentsPageProps {
     selectedTab = 'contents';
     contents = [];
+    images = [];
+    imageSrc = [];
+    imageLoaded = [];
 
     created() {
     }
@@ -93,34 +134,48 @@ class ContentsPage extends Vue {
 
         //закладки
 
-        //далее формаирование оглавления
+        //проверим, надо ли обновлять списки
-        if (this.parsed == parsed)
+        if (this.parsed == parsed) {
+            this.updateBookPosSelection();
             return;
+        }
 
+        //далее формирование оглавления
         this.parsed = parsed;
         this.contents = [];
         await this.$nextTick();
 
         const pc = parsed.contents;
-        const newpc = [];
+        const ims = parsed.images;
-        //преобразуем все, кроме первого, разделы body в title-subtitle
+        const newpc = [];        
-        let curSubtitles = [];
+        if (pc.length) {//если есть оглавление
-        let prevBodyIndex = -1;
+            //преобразуем все, кроме первого, разделы body в title-subtitle
-        for (let i = 0; i < pc.length; i++) {
+            let curSubtitles = [];
-            const cont = pc[i];
+            let prevBodyIndex = -1;
-            if (prevBodyIndex != cont.bodyIndex)
+            for (let i = 0; i < pc.length; i++) {
-                curSubtitles = [];
+                const cont = pc[i];
+                if (prevBodyIndex != cont.bodyIndex)
+                    curSubtitles = [];
 
-            prevBodyIndex = cont.bodyIndex;
+                prevBodyIndex = cont.bodyIndex;
 
-            if (cont.bodyIndex > 1) {
+                if (cont.bodyIndex > 1) {
-                if (cont.inset < 1) {
+                    if (cont.inset < 1) {
-                    newpc.push(Object.assign({}, cont, {subtitles: curSubtitles}));
+                        newpc.push(Object.assign({}, cont, {subtitles: curSubtitles}));
+                    } else {
+                        curSubtitles.push(Object.assign({}, cont, {inset: cont.inset - 1}));
+                    }
                 } else {
-                    curSubtitles.push(Object.assign({}, cont, {inset: cont.inset - 1}));
+                    newpc.push(cont);
+                }
+            }
+        } else {//попробуем вытащить из images
+            for (let i = 0; i < ims.length; i++) {
+                const image = ims[i];
+
+                if (image.alt) {
+                    newpc.push({paraIndex: image.paraIndex, title: image.alt, inset: 1, bodyIndex: 0, subtitles: []});
                 }
-            } else {
-                newpc.push(cont);
             }
         }
 
@@ -166,6 +221,90 @@ class ContentsPage extends Vue {
         });
 
         this.contents = newContents;
+
+        //формируем newImages
+        const newImages = [];
+        for (i = 0; i < ims.length; i++) {
+            const image = ims[i];
+            const bin = parsed.binary[image.id];
+            const type = (bin ? bin.type : '');
+            
+            const label = (image.alt ? image.alt : '<span style="font-size: 90%; color: #dddddd"><i>Без названия</i></span>');
+            const indentStyle = getIndentStyle(1);
+            const labelStyle = getLabelStyle(1);
+
+            const p = parsed.para[image.paraIndex];
+            newImages.push({perc: (p.offset/parsed.textLength*100).toFixed(0), label, key: i, offset: p.offset,
+                indentStyle, labelStyle, type, num: image.num, id: image.id, local: image.local});
+        }
+
+        this.images = newImages;
+
+        if (this.selectedTab == 'contents' && !this.contents.length && this.images.length)
+            this.selectedTab = 'images';
+
+        //выделим на bookPos
+        this.updateBookPosSelection();
+
+        //асинхронная загрузка изображений
+        this.imageSrc = [];
+        this.imageLoaded = [];
+        await utils.sleep(50);
+        (async() => {
+            for (i = 0; i < ims.length; i++) {
+                const {id, local} = ims[i];
+                const bin = this.parsed.binary[id];
+                if (local)
+                    this.$set(this.imageSrc, id, (bin ? `data:${bin.type};base64,${bin.data}` : ''));
+                else
+                    this.$set(this.imageSrc, id, id);
+                this.imageLoaded[id] = true;
+                await utils.sleep(5);
+            }
+        })();
+    }
+
+    async updateBookPosSelection() {
+        if (!this.isVisible)
+            return;
+
+        await utils.sleep(50);
+        const bp = this.bookPos;
+        
+        for (let i = 0; i < this.contents.length; i++) {
+            const item = this.contents[i];
+            const nextOffset = (i < this.contents.length - 1 ? this.contents[i + 1].offset : this.parsed.textLength);
+
+            for (let j = 0; j < item.list.length; j++) {
+                const subitem = item.list[j];
+                const nextSubOffset = (j < item.list.length - 1 ? item.list[j + 1].offset : nextOffset);
+
+                if (bp >= subitem.offset && bp < nextSubOffset) {
+                    subitem.isBookPos = true;
+                    this.$set(this.contents, i, Object.assign(item, {list: item.list}));
+                } else if (subitem.isBookPos) {
+                    subitem.isBookPos = false;
+                    this.$set(this.contents, i, Object.assign(item, {list: item.list}));
+                }
+            }
+
+            if (bp >= item.offset && bp < nextOffset) {
+                this.$set(this.contents, i, Object.assign(item, {isBookPos: true}));
+            } else if (item.isBookPos) {
+                this.$set(this.contents, i, Object.assign(item, {isBookPos: false}));
+            }
+        }
+
+        for (let i = 0; i < this.images.length; i++) {
+            const img = this.images[i];
+            const nextOffset = (i < this.images.length - 1 ? this.images[i + 1].offset : this.parsed.textLength);
+
+            if (bp >= img.offset && bp < nextOffset) {
+                this.$set(this.images, i, Object.assign(img, {isBookPos: true}));
+            } else if (img.isBookPos) {
+                this.$set(this.images, i, Object.assign(img, {isBookPos: false}));
+            }
+        }
     }
 
     async expandClick(key) {
@@ -219,7 +358,7 @@ class ContentsPage extends Vue {
     padding: 10px 0 10px 0;
 }
 
-.item, .subitem {
+.item, .subitem, .item-book-pos, .subitem-book-pos {
     border-bottom: 1px solid #e0e0e0;
 }
 
@@ -227,6 +366,22 @@ class ContentsPage extends Vue {
     background-color: #f0f0f0;
 }
 
+.item-book-pos {
+    background-color: #b0f0b0;
+}
+
+.subitem-book-pos {
+    background-color: #d0f5d0;
+}
+
+.item-book-pos:hover {
+    background-color: #b0e0b0;
+}
+
+.subitem-book-pos:hover {
+    background-color: #d0f0d0;
+}
+
 .expand-button, .no-expand-button {
     width: 40px;
 }
@@ -244,4 +399,38 @@ class ContentsPage extends Vue {
 .expanded-icon {
     transform: rotate(90deg);
 }
+
+.image-num {
+    font-size: 120%;
+    padding-bottom: 3px;
+}
+.image-type {
+    border: 1px solid black;
+    border-radius: 6px;
+    font-size: 80%;
+    padding: 2px 0 2px 0;
+    width: 34px;
+}
+.it-jpg-color {
+    background: linear-gradient(to right, #fabc3d, #ffec6d);
+}
+.it-png-color {
+    background: linear-gradient(to right, #4bc4e5, #6bf4ff);
+}
+.it-net-color {
+    background: linear-gradient(to right, #00c400, #00f400);
+}
+
+.image-thumb-box {
+    width: 120px;
+    overflow: hidden;
+}
+
+.image-thumb {
+    height: 50px;
+}
+
+.loading-img-icon {
+    font-size: 250%;
+}
 </style>

client/components/Reader/LoaderPage/LoaderPage.vue

@@ -7,8 +7,8 @@
             <span class="greeting"><b>{{ title }}</b></span>
             <div class="q-my-sm"></div>
             <span class="greeting">Добро пожаловать!</span>
-            <span class="greeting">Поддерживаются форматы: <b>fb2, html, txt</b> и сжатие: <b>zip, bz2, gz</b></span>
+            <span class="greeting">Поддерживаются форматы: <b>fb2, html, txt</b> и сжатие: <b>zip, bz2, gz<span v-if="isExternalConverter">, rar</span></b></span>
-            <span v-if="isExternalConverter" class="greeting">...а также форматы: <b>rtf, doc, docx, pdf, epub, mobi</b></span>
+            <span v-if="isExternalConverter" class="greeting">...а также частично форматы: <b>epub, mobi, rtf, doc, docx, pdf, djvu</b></span>
         </div>
 
         <div class="col-auto column justify-start items-center no-wrap overflow-hidden">

client/components/Reader/Reader.vue

@@ -39,9 +39,9 @@
                         <q-icon name="la la-copy" size="32px"/>
                         <q-tooltip :delay="1500" anchor="bottom middle" content-style="font-size: 80%">{{ rstore.readerActions['copyText'] }}</q-tooltip>
                     </button>
-                    <button ref="splitToPara" v-show="showToolButton['splitToPara']" class="tool-button" :class="buttonActiveClass('splitToPara')" @click="buttonClick('splitToPara')" v-ripple>
+                    <button ref="convOptions" v-show="showToolButton['convOptions']" class="tool-button" :class="buttonActiveClass('convOptions')" @click="buttonClick('convOptions')" v-ripple>
-                        <q-icon name="la la-retweet" size="32px"/>
+                        <q-icon name="la la-magic" size="32px"/>
-                        <q-tooltip :delay="1500" anchor="bottom middle" content-style="font-size: 80%">{{ rstore.readerActions['splitToPara'] }}</q-tooltip>
+                        <q-tooltip :delay="1500" anchor="bottom middle" content-style="font-size: 80%">{{ rstore.readerActions['convOptions'] }}</q-tooltip>
                     </button>
                     <button ref="refresh" v-show="showToolButton['refresh']" class="tool-button" :class="buttonActiveClass('refresh')" @click="buttonClick('refresh')" v-ripple>
                         <q-icon name="la la-sync" size="32px" :class="{clear: !showRefreshIcon}"/>
@@ -99,7 +99,7 @@
             <HelpPage v-if="helpActive" ref="helpPage" @do-action="doAction"></HelpPage>
             <ClickMapPage v-show="clickMapActive" ref="clickMapPage"></ClickMapPage>
             <ServerStorage v-show="hidden" ref="serverStorage"></ServerStorage>
-            <ContentsPage v-show="contentsActive" ref="contentsPage" @do-action="doAction" @book-pos-changed="bookPosChanged"></ContentsPage>
+            <ContentsPage v-show="contentsActive" ref="contentsPage" :book-pos="bookPos" :is-visible="contentsActive" @do-action="doAction" @book-pos-changed="bookPosChanged"></ContentsPage>
 
             <ReaderDialogs ref="dialogs" @donate-toggle="donateToggle" @version-history-toggle="versionHistoryToggle"></ReaderDialogs>
         </div>
@@ -133,6 +133,9 @@ import ReaderDialogs from './ReaderDialogs/ReaderDialogs.vue';
 import bookManager from './share/bookManager';
 import rstore from '../../store/modules/reader';
 import readerApi from '../../api/reader';
+import miscApi from '../../api/misc';
+
+import {versionHistory} from './versionHistory';
 import * as utils from '../../share/utils';
 
 export default @Component({
@@ -229,7 +232,6 @@ class Reader extends Vue {
         this.rstore = rstore;
         this.loading = true;
         this.commit = this.$store.commit;
-        this.dispatch = this.$store.dispatch;
         this.reader = this.$store.state.reader;
         this.config = this.$store.state.config;
 
@@ -293,6 +295,16 @@ class Reader extends Vue {
 
             await this.$refs.dialogs.init();
         })();
+
+        (async() => {
+            this.isFirstNeedUpdateNotify = true;
+            //вечный цикл, запрашиваем периодически конфиг для проверки выхода новой версии читалки
+            while (true) {// eslint-disable-line no-constant-condition
+                await this.checkNewVersionAvailable();
+                await utils.sleep(3600*1000); //каждый час
+            }
+            //дальше кода нет
+        })();
     }
 
     loadSettings() {
@@ -304,6 +316,11 @@ class Reader extends Vue {
         this.blinkCachedLoad = settings.blinkCachedLoad;
         this.showToolButton = settings.showToolButton;
         this.enableSitesFilter = settings.enableSitesFilter;
+        this.showNeedUpdateNotify = settings.showNeedUpdateNotify;
+        this.splitToPara = settings.splitToPara;
+        this.djvuQuality = settings.djvuQuality;
+        this.pdfAsText = settings.pdfAsText;
+        this.pdfQuality = settings.pdfQuality;
 
         this.readerActionByKeyCode = utils.userHotKeysObjectSwap(settings.userHotKeys);
         this.$root.readerActionByKeyEvent = (event) => {
@@ -313,6 +330,30 @@ class Reader extends Vue {
         this.updateHeaderMinWidth();
     }
 
+    async checkNewVersionAvailable() {
+        if (!this.checkingNewVersion && this.showNeedUpdateNotify) {
+            this.checkingNewVersion = true;
+            try {
+                await utils.sleep(15*1000); //подождем 15 секунд, чтобы прогрузился ServiceWorker при выходе новой версии
+                const config = await miscApi.loadConfig();
+                this.commit('config/setConfig', config);
+
+                let againMes = '';
+                if (this.isFirstNeedUpdateNotify) {
+                    againMes = ' еще один раз';
+                }
+
+                if (this.version != this.clientVersion)
+                    this.$root.notify.info(`Вышла новая версия (v${this.version}) читалки.<br>Пожалуйста, обновите страницу${againMes}.`, 'Обновление');
+            } catch(e) {
+                console.error(e);
+            } finally {
+                this.checkingNewVersion = false;
+            }        
+            this.isFirstNeedUpdateNotify = false;
+        }
+    }
+
     updateHeaderMinWidth() {
         const showButtonCount = Object.values(this.showToolButton).reduce((a, b) => a + (b ? 1 : 0), 0);
         if (this.$refs.buttons)
@@ -394,6 +435,16 @@ class Reader extends Vue {
         return this.$store.state.config.mode;
     }
 
+    get version() {
+        return this.$store.state.config.version;
+    }
+
+    get clientVersion() {
+        let v = versionHistory[0].header;
+        v = v.split(' ')[0];
+        return v;
+    }
+
     get routeParamUrl() {
         let result = '';
         const path = this.$route.fullPath;
@@ -593,12 +644,6 @@ class Reader extends Vue {
         }
     }
 
-    refreshBookSplitToPara() {
-        if (this.mostRecentBook()) {
-            this.loadBook({url: this.mostRecentBook().url, skipCheck: true, isText: true, force: true});
-        }
-    }
-
     recentBooksClose() {
         this.recentBooksActive = false;
     }
@@ -662,6 +707,12 @@ class Reader extends Vue {
         }
     }
 
+    convOptionsToggle() {
+        this.settingsToggle();
+        if (this.settingsActive)
+            this.$refs.settingsPage.selectedTab = 'convert';
+    }
+
     helpToggle() {
         this.helpActive = !this.helpActive;
         if (this.helpActive) {
@@ -689,9 +740,8 @@ class Reader extends Vue {
     }
 
     refreshBook() {
-        if (this.mostRecentBook()) {
+        const mrb = this.mostRecentBook();
-            this.loadBook({url: this.mostRecentBook().url, force: true});
+        this.loadBook({url: mrb.url, uploadFileName: mrb.uploadFileName, force: true});
-        }
     }
 
     undoAction() {
@@ -731,7 +781,7 @@ class Reader extends Vue {
             case 'scrolling':
             case 'search':
             case 'copyText':
-            case 'splitToPara':
+            case 'convOptions':
             case 'refresh':
             case 'contents':
             case 'libs':
@@ -765,7 +815,6 @@ class Reader extends Vue {
                 case 'contents':
                     classResult = classDisabled;
                     break;
-                case 'splitToPara':
                 case 'refresh':
                 case 'recentBooks':
                     if (!this.mostRecentBookReactive)
@@ -846,8 +895,12 @@ class Reader extends Vue {
 
         let url = encodeURI(decodeURI(opts.url));
 
+        //TODO: убрать конвертирование 'file://' после 06.2021
+        if (url.length == 71 && url.indexOf('file://') == 0)
+            url = url.replace(/^file/, 'disk');
+
         if ((url.indexOf('http://') != 0) && (url.indexOf('https://') != 0) &&
-            (url.indexOf('file://') != 0))
+            (url.indexOf('disk://') != 0))
             url = 'http://' + url;
 
         // уже просматривается сейчас
@@ -878,6 +931,7 @@ class Reader extends Vue {
             wasOpened = (wasOpened ? wasOpened : {});
             const bookPos = (opts.bookPos !== undefined ? opts.bookPos : wasOpened.bookPos);
             const bookPosSeen = (opts.bookPos !== undefined ? opts.bookPos : wasOpened.bookPosSeen);
+            const uploadFileName = (opts.uploadFileName ? opts.uploadFileName : '');
 
             let book = null;
 
@@ -922,9 +976,13 @@ class Reader extends Vue {
             if (!book) {
                 book = await readerApi.loadBook({
                         url,
-                        skipCheck: (opts.skipCheck ? true : false),
+                        uploadFileName,
-                        isText: (opts.isText ? true : false),
+                        enableSitesFilter: this.enableSitesFilter,
-                        enableSitesFilter: this.enableSitesFilter
+                        skipHtmlCheck: (this.splitToPara ? true : false),
+                        isText: (this.splitToPara ? true : false),
+                        djvuQuality: this.djvuQuality,
+                        pdfAsText: this.pdfAsText,
+                        pdfQuality: this.pdfQuality,
                     },
                     (state) => {
                         progress.setState(state);
@@ -940,7 +998,7 @@ class Reader extends Vue {
             });
 
             // добавляем в историю
-            await bookManager.setRecentBook(Object.assign({bookPos, bookPosSeen}, addedBook));
+            await bookManager.setRecentBook(Object.assign({bookPos, bookPosSeen, uploadFileName}, addedBook));
             this.mostRecentBook();
             this.addAction(bookPos);
             this.updateRoute(true);
@@ -958,6 +1016,8 @@ class Reader extends Vue {
             progress.hide(); this.progressActive = false;
             this.loaderActive = true;
             this.$root.stdDialog.alert(e.message, 'Ошибка', {color: 'negative'});
+        } finally {
+            this.checkNewVersionAvailable();
         }
     }
 
@@ -977,7 +1037,7 @@ class Reader extends Vue {
 
             progress.hide(); this.progressActive = false;
 
-            await this.loadBook({url});
+            await this.loadBook({url, uploadFileName: opts.file.name, force: true});
         } catch (e) {
             progress.hide(); this.progressActive = false;
             this.loaderActive = true;
@@ -1048,8 +1108,8 @@ class Reader extends Vue {
             case 'copyText':
                 this.copyTextToggle();
                 break;
-            case 'splitToPara':
+            case 'convOptions':
-                this.refreshBookSplitToPara();
+                this.convOptionsToggle();
                 break;
             case 'refresh':
                 this.refreshBook();

client/components/Reader/ReaderDialogs/ReaderDialogs.vue

@@ -57,37 +57,6 @@
                 <q-btn class="q-px-sm" dense no-caps @click="donationDialogRemind">Напомнить позже</q-btn>
             </span>
         </Dialog>
-
-        <Dialog ref="dialog3" v-model="liberamaTopVisible">
-            <template slot="header">
-                Здравствуйте, уважаемые читатели!
-            </template>
-
-            <div style="word-break: normal">
-                Создан новый ресурс:<br><br>
-
-                <a href="https://liberama.top" target="_blank">https://liberama.top</a>
-                <br><br>
-                Это клон читалки Omni Reader, но с некоторыми дополнениями, ориентированными в сторону более свободного обмена книгами:
-
-                <ul>
-                    <li>добавлено новое окно "Библиотека" для свободного доступа к Флибусте и другим ресурсам по желанию читателя</li>
-                    <li>планируется добавить возможность создания подборок книг и обмена ими между пользователями</li>
-                </ul>
-
-                Легко мигрировать на новый сайт можно с помощью синхронизации с сервером.
-                О багах и предложениях просьба сообщать на почту <a href="mailto:bookpauk@gmail.com">bookpauk@gmail.com</a><br><br>
-                Спасибо, что вы с нами!
-                <br><br>
-                <div class="row justify-center">
-                    <q-btn class="q-px-sm" color="primary" dense no-caps rounded @click="openDonate">Помочь проекту</q-btn>
-                </div>
-            </div>
-
-            <span slot="footer">
-                <q-btn class="q-px-sm" dense no-caps @click="liberamaTopDialogDisable">Больше не показывать</q-btn>
-            </span>
-        </Dialog>
     </div>
 </template>
 
@@ -114,7 +83,6 @@ class ReaderDialogs extends Vue {
     whatsNewVisible = false;
     whatsNewContent = '';
     donationVisible = false;
-    liberamaTopVisible = false;
 
     created() {
         this.commit = this.$store.commit;
@@ -127,14 +95,12 @@ class ReaderDialogs extends Vue {
     async init() {
         await this.showWhatsNew();
         await this.showDonation();
-        await this.showLiberamaTop();
     }
 
     loadSettings() {
         const settings = this.settings;
         this.showWhatsNewDialog = settings.showWhatsNewDialog;
         this.showDonationDialog2020 = settings.showDonationDialog2020;
-        this.showLiberamaTopDialog2020 = settings.showLiberamaTopDialog2020;
     }
 
     async showWhatsNew() {
@@ -171,7 +137,6 @@ class ReaderDialogs extends Vue {
 
     openDonate() {
         this.donationVisible = false;
-        this.liberamaTopVisible = false;
         this.$emit('donate-toggle');
     }
 
@@ -210,24 +175,8 @@ class ReaderDialogs extends Vue {
         return this.$store.state.reader.donationRemindDate;
     }
 
-    async showLiberamaTop() {
-        const today = utils.formatDate(new Date(), 'coDate');
-
-        if (this.mode == 'omnireader' && today < '2020-12-01' && this.showLiberamaTopDialog2020) {
-            await utils.sleep(3000);
-            this.liberamaTopVisible = true;
-        }
-    }
-
-    liberamaTopDialogDisable()  {
-        this.liberamaTopVisible = false;
-        if (this.showLiberamaTopDialog2020) {
-            this.commit('reader/setSettings', { showLiberamaTopDialog2020: false });
-        }
-    }
-
     keyHook() {
-        if (this.$refs.dialog1.active || this.$refs.dialog2.active || this.$refs.dialog3.active)
+        if (this.$refs.dialog1.active || this.$refs.dialog2.active)
             return true;
         return false;
     }

client/components/Reader/RecentBooksPage/RecentBooksPage.vue

@@ -112,6 +112,7 @@ class RecentBooksPage extends Vue {
     pagination = {};
 
     created() {
+        this.firstInit = true;
         this.pagination = {rowsPerPage: 0};
 
         this.columns = [
@@ -167,26 +168,11 @@ class RecentBooksPage extends Vue {
             this.initing = true;
 
 
-            if (!bookManager.loaded) {
+            if (this.firstInit) {//для отзывчивости
-                await this.updateTableData(10);
+                await this.updateTableData(20);
-                //для отзывчивости
+                this.firstInit = false;
-                await utils.sleep(100);
-                let i = 0;
-                let j = 5;
-                while (i < 500 && !bookManager.loaded) {
-                    if (i % j == 0) {
-                        bookManager.sortedRecentCached = null;
-                        await this.updateTableData(20);
-                        j *= 2;
-                    }
-
-                    await utils.sleep(100);
-                    i++;
-                }
-            } else {
-                //для отзывчивости
-                await utils.sleep(100);
             }
+            await utils.sleep(50);
             await this.updateTableData();
             this.initing = false;
         })();
@@ -310,7 +296,7 @@ class RecentBooksPage extends Vue {
 
     isUrl(url) {
         if (url)
-            return (url.indexOf('file://') != 0);
+            return (url.indexOf('disk://') != 0);
         else
             return false;
     }

client/components/Reader/ServerStorage/ServerStorage.vue

@@ -216,8 +216,15 @@ class ServerStorage extends Vue {
     }
 
     error(message) {
-        if (this.showServerStorageMessages && !this.offlineModeActive)
+        if (this.showServerStorageMessages && !this.offlineModeActive) {
-            this.$root.notify.error(message);
+            this.errorMessageCounter = (this.errorMessageCounter ? this.errorMessageCounter + 1 : 1);
+            const hint = (this.errorMessageCounter < 2 ? '' :
+                '<div><br>Надоело это сообщение? Добавьте в настройках кнопку "Автономный режим" ' +
+                '<i class="la la-unlink" style="font-size: 20px; color: white"></i> на панель инструментов и активируйте ее.</div>'
+            );
+
+            this.$root.notify.error(message + hint);
+        }
     }
 
     async loadSettings(force = false, doNotifySuccess = true) {
@@ -580,8 +587,8 @@ class ServerStorage extends Vue {
             let newRecent = {};
             if (!itemKey || (needSaveRecentPatch && Object.keys(newRecentPatch.data).length > 10)) {
                 //ждем весь bm.recent
-                while (!bookManager.loaded)
+                /*while (!bookManager.loaded)
-                    await utils.sleep(100);
+                    await utils.sleep(100);*/
 
                 newRecent = {rev: this.cachedRecent.rev + 1, data: _.cloneDeep(bm.recent)};
                 newRecentPatch = {rev: this.cachedRecentPatch.rev + 1, data: {}};

client/components/Reader/SettingsPage/SettingsPage.vue

@@ -26,6 +26,7 @@
                     <q-tab class="tab" name="buttons" icon="la la-grip-horizontal" label="Кнопки" />
                     <q-tab class="tab" name="keys" icon="la la-gamepad" label="Управление" />
                     <q-tab class="tab" name="pagemove" icon="la la-school" label="Листание" />
+                    <q-tab class="tab" name="convert" icon="la la-magic" label="Конвертир." />
                     <q-tab class="tab" name="others" icon="la la-list-ul" label="Прочее" />
                     <q-tab class="tab" name="reset" icon="la la-broom" label="Сброс" />
                     <div v-show="tabsScrollable" class="q-pt-lg"/>
@@ -53,6 +54,10 @@
                 <div v-if="selectedTab == 'pagemove'" class="fit tab-panel">
                     @@include('./include/PageMoveTab.inc');
                 </div>
+                <!-- Конвертирование ------------------------------------------------------------->
+                <div v-if="selectedTab == 'convert'" class="fit tab-panel">
+                    @@include('./include/ConvertTab.inc');
+                </div>
                 <!-- Прочее ---------------------------------------------------------------------->
                 <div v-if="selectedTab == 'others'" class="fit tab-panel">
                     @@include('./include/OthersTab.inc');
@@ -218,6 +223,10 @@ class SettingsPage extends Vue {
         return this.$store.state.config.mode;
     }
 
+    get isExternalConverter() {
+        return this.$store.state.config.useExternalBookConverter;
+    }
+
     get settings() {
         return this.$store.state.reader.settings;
     }
@@ -544,7 +553,7 @@ class SettingsPage extends Vue {
     margin-bottom: 5px;
 }
 
-.label-1 {
+.label-1, .label-7 {
     width: 75px;
 }
 
@@ -556,7 +565,7 @@ class SettingsPage extends Vue {
     width: 100px;
 }
 
-.label-1, .label-2, .label-3, .label-4, .label-5, .label-6 {
+.label-1, .label-2, .label-3, .label-4, .label-5, .label-6, .label-7 {
     display: flex;
     flex-direction: column;
     justify-content: center;

client/components/Reader/SettingsPage/include/ConvertTab.inc

@@ -0,0 +1,87 @@
+<!---------------------------------------------->
+<div class="q-mt-sm column items-center">
+    <span>Настройки конвертирования применяются ко всем</span>
+    <span>вновь загружаемым или обновляемым файлам</span>
+</div>
+
+<!---------------------------------------------->
+<div class="part-header">HTML, XML, TXT</div>
+
+<div class="item row">
+    <div class="label-7">Текст</div>
+    <div class="col row">
+        <q-checkbox v-model="splitToPara" size="xs" label="Попытаться разбить текст на параграфы">
+            <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+                Опция принудительно включает эвристику разбиения текста на<br>
+                параграфы в случае, если формат файла определен как html,<br>
+                xml или txt. Возможна нечитабельная разметка текста.
+            </q-tooltip>
+        </q-checkbox>
+    </div>
+</div>
+
+<div class="item row">
+    <div class="label-7">Сайты</div>
+    <div class="col row">
+        <q-checkbox v-model="enableSitesFilter" size="xs" label="Включить html-фильтр для сайтов">
+            <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+                Html-фильтр вырезает лишние элементы со<br>
+                страницы для определенных сайтов, таких как:<br>
+                samlib.ru<br>
+                www.fanfiction.net<br>
+                archiveofourown.org<br>
+                и других
+            </q-tooltip>
+        </q-checkbox>
+    </div>
+</div>
+
+<!---------------------------------------------->
+<div v-if="isExternalConverter">
+    <div class="part-header">PDF</div>
+
+    <div class="item row">
+        <div class="label-7">Формат</div>
+        <div class="col row">
+            <q-checkbox v-model="pdfAsText" size="xs" label="Извлекать текст из PDF">
+                <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+                    Пытается извлечь текст из pdf-файла и переразбить на параграфы.<br>
+                    Размер получаемого fb2-файла при этом относительно небольшой.<br>
+                    При отключении этой опции, pdf будет представлен как набор<br>
+                    изображений (аналогично ковертированию djvu).
+                </q-tooltip>
+            </q-checkbox>
+        </div>
+    </div>
+
+    <div class="item row">
+        <div class="label-7">Качество</div>
+        <div class="col row">
+            <NumInput class="col-5" v-model="pdfQuality" :min="10" :max="100" :disable="pdfAsText" >
+                <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+                    Качество конвертирования Pdf в Fb2. Чем значение выше, тем больше<br>
+                    размер итогового файла. Если сервер отказывается конвертировать<br>
+                    слишком большой файл, то попробуйте понизить качество.
+                </q-tooltip>
+            </NumInput>
+        </div>
+    </div>
+</div>
+
+<!---------------------------------------------->
+<div v-if="isExternalConverter">
+    <div class="part-header">DJVU</div>
+
+    <div class="item row">
+        <div class="label-7">Качество</div>
+        <div class="col row">
+            <NumInput class="col-5" v-model="djvuQuality" :min="10" :max="100">
+                <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+                    Качество конвертирования Djvu в Fb2. Чем значение выше, тем больше<br>
+                    размер итогового файла. Если сервер отказывается конвертировать<br>
+                    слишком большой файл, то попробуйте понизить качество.
+                </q-tooltip>
+            </NumInput>
+        </div>
+    </div>
+</div>

client/components/Reader/SettingsPage/include/OthersTab.inc

@@ -36,7 +36,18 @@
         Показывать уведомление "Что нового"
         <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
             Показывать уведомления "Что нового"<br>
-            при каждом выходе новой версии читалки
+            при появлении новой версии читалки
+        </q-tooltip>
+    </q-checkbox>
+</div>
+
+<div class="item row">
+    <div class="label-6">Уведомление</div>
+    <q-checkbox size="xs" v-model="showNeedUpdateNotify">
+        Показывать уведомление о новой версии
+        <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
+            Напоминать о необходимости обновления страницы<br>
+            при появлении новой версии читалки
         </q-tooltip>
     </q-checkbox>
 </div>
@@ -54,22 +65,6 @@
 <!---------------------------------------------->
 <div class="part-header">Другое</div>
 
-<div class="item row">
-    <div class="label-6">Обработка</div>
-    <div class="col row">
-        <q-checkbox v-model="enableSitesFilter" @input="needTextReload" size="xs" label="Включить html-фильтр для сайтов">
-            <q-tooltip :delay="1000" anchor="top middle" self="bottom middle" content-style="font-size: 80%">
-                Html-фильтр вырезает лишние элементы со<br>
-                страницы для определенных сайтов, таких как:<br>
-                samlib.ru<br>
-                www.fanfiction.net<br>
-                archiveofourown.org<br>
-                и других
-            </q-tooltip>
-        </q-checkbox>
-    </div>
-</div>
-
 <div class="item row">
     <div class="label-6">Обработка</div>
     <q-checkbox size="xs" v-model="lazyParseEnabled" label="Предварительная подготовка текста">

client/components/Reader/TextPage/DrawHelper.js

@@ -77,9 +77,15 @@ export default class DrawHelper {
             let j = 0;
             //формируем строку
             for (const part of line.parts) {
-                let tOpen = (part.style.bold ? '<b>' : '');
+                let tOpen = '';
+                tOpen += (part.style.bold ? '<b>' : '');
                 tOpen += (part.style.italic ? '<i>' : '');
-                let tClose = (part.style.italic ? '</i>' : '');
+                tOpen += (part.style.sup ? '<span style="vertical-align: baseline; position: relative; line-height: 0; top: -0.3em">' : '');
+                tOpen += (part.style.sub ? '<span style="vertical-align: baseline; position: relative; line-height: 0; top: 0.3em">' : '');
+                let tClose = '';
+                tClose += (part.style.sub ? '</span>' : '');
+                tClose += (part.style.sup ? '</span>' : '');
+                tClose += (part.style.italic ? '</i>' : '');
                 tClose += (part.style.bold ? '</b>' : '');
 
                 let text = '';
@@ -154,12 +160,13 @@ export default class DrawHelper {
         return out;
     }
 
-    drawPercentBar(x, y, w, h, font, fontSize, bookPos, textLength) {
+    drawPercentBar(x, y, w, h, font, fontSize, bookPos, textLength, imageNum, imageLength) {
         const pad = 3;
         const fh = h - 2*pad;
         const fh2 = fh/2;
 
-        const t1 = `${Math.floor((bookPos + 1)/1000)}k/${Math.floor(textLength/1000)}k`;
+        const tImg = (imageNum > 0 ? ` (${imageNum}/${imageLength})` : '');
+        const t1 = `${Math.floor((bookPos + 1)/1000)}/${Math.floor(textLength/1000)}${tImg}`;
         const w1 = this.measureTextFont(t1, font) + fh2;
         const read = (bookPos + 1)/textLength;
         const t2 = `${(read*100).toFixed(2)}%`;
@@ -182,7 +189,7 @@ export default class DrawHelper {
         return out;
     }
 
-    drawStatusBar(statusBarTop, statusBarHeight, bookPos, textLength, title) {
+    drawStatusBar(statusBarTop, statusBarHeight, bookPos, textLength, title, imageNum, imageLength) {
 
         let out = `<div class="layout" style="` + 
             `width: ${this.realWidth}px; height: ${statusBarHeight}px; ` + 
@@ -200,7 +207,7 @@ export default class DrawHelper {
 
         out += this.fillTextShift(this.fittingString(title, this.realWidth/2 - fontSize - 3, font), fontSize, 2, font, fontSize);
 
-        out += this.drawPercentBar(this.realWidth/2, 2, this.realWidth/2 - timeW - 2*fontSize, statusBarHeight, font, fontSize, bookPos, textLength);
+        out += this.drawPercentBar(this.realWidth/2, 2, this.realWidth/2 - timeW - 2*fontSize, statusBarHeight, font, fontSize, bookPos, textLength, imageNum, imageLength);
         
         out += '</div>';
         return out;

client/components/Reader/TextPage/TextPage.vue

@@ -722,8 +722,24 @@ class TextPage extends Vue {
                     message = this.statusBarMessage;
                 if (!message)
                     message = this.title;
+
+                //check image num
+                let imageNum = 0;
+                const len = (lines.length > 2 ? 2 : lines.length);
+                loop:
+                for (let j = 0; j < len; j++) {
+                    const line = lines[j];
+                    for (const part of line.parts) {
+                        if (part.image) {
+                            imageNum = part.image.num;
+                            break loop;
+                        }
+                    }
+                }
+                //drawing
                 this.statusBar = this.drawHelper.drawStatusBar(this.statusBarTop, this.statusBarHeight, 
-                    lines[i].end, this.parsed.textLength, message);
+                    lines[i].end, this.parsed.textLength, message, imageNum, this.parsed.images.length);
+
                 this.bookPosSeen = lines[i].end;
             }
         } else {
@@ -1053,7 +1069,7 @@ class TextPage extends Vue {
 
     onStatusBarClick() {
         const url = this.meta.url;
-        if (url && url.indexOf('file://') != 0) {
+        if (url && url.indexOf('disk://') != 0) {
             window.open(url, '_blank');
         } else {
             this.$root.stdDialog.alert('Оригинал недоступен, т.к. файл книги был загружен с локального диска.', ' ', {color: 'info'});

client/components/Reader/share/BookParser.js

@@ -54,12 +54,14 @@ export default class BookParser {
 
         //оглавление
         this.contents = [];
+        this.images = [];
         let curTitle = {paraIndex: -1, title: '', subtitles: []};
         let curSubtitle = {paraIndex: -1, title: ''};
         let inTitle = false;
         let inSubtitle = false;
         let sectionLevel = 0;
         let bodyIndex = 0;
+        let imageNum = 0;
 
         let paraIndex = -1;
         let paraOffset = 0;
@@ -194,6 +196,7 @@ export default class BookParser {
             if (tag == 'binary') {
                 let attrs = sax.getAttrsSync(tail);
                 binaryType = (attrs['content-type'] && attrs['content-type'].value ? attrs['content-type'].value : '');
+                binaryType = (binaryType == 'image/jpg' ? 'image/jpeg' : binaryType);
                 if (binaryType == 'image/jpeg' || binaryType == 'image/png' || binaryType == 'application/octet-stream')
                     binaryId = (attrs.id.value ? attrs.id.value : '');
             }
@@ -202,16 +205,27 @@ export default class BookParser {
                 let attrs = sax.getAttrsSync(tail);
                 if (attrs.href && attrs.href.value) {
                     const href = attrs.href.value;
+                    const alt = (attrs.alt && attrs.alt.value ? attrs.alt.value : '');
+                    const {id, local} = this.imageHrefToId(href);
                     if (href[0] == '#') {//local
+                        imageNum++;
+
                         if (inPara && !this.showInlineImagesInCenter && !center)
-                            growParagraph(`<image-inline href="${href}"></image-inline>`, 0);
+                            growParagraph(`<image-inline href="${href}" num="${imageNum}"></image-inline>`, 0);
                         else
-                            newParagraph(`<image href="${href}">${' '.repeat(maxImageLineCount)}</image>`, maxImageLineCount);
+                            newParagraph(`<image href="${href}" num="${imageNum}">${' '.repeat(maxImageLineCount)}</image>`, maxImageLineCount);
+
+                        this.images.push({paraIndex, num: imageNum, id, local, alt});
+
                         if (inPara && this.showInlineImagesInCenter)
                             newParagraph(' ', 1);
                     } else {//external
+                        imageNum++;
+
                         dimPromises.push(getExternalImageDimensions(href));
-                        newParagraph(`<image href="${href}">${' '.repeat(maxImageLineCount)}</image>`, maxImageLineCount);
+                        newParagraph(`<image href="${href}" num="${imageNum}">${' '.repeat(maxImageLineCount)}</image>`, maxImageLineCount);
+
+                        this.images.push({paraIndex, num: imageNum, id, local, alt});
                     }
                 }
             }
@@ -285,7 +299,7 @@ export default class BookParser {
                     sectionLevel++;
                 }
 
-                if (tag == 'emphasis' || tag == 'strong') {
+                if (tag == 'emphasis' || tag == 'strong' || tag == 'sup' || tag == 'sub') {
                     growParagraph(`<${tag}>`, 0);
                 }
 
@@ -304,6 +318,11 @@ export default class BookParser {
                     bold = true;
                     center = true;
 
+                    if (curTitle.paraIndex < 0) {
+                        curTitle = {paraIndex, title: 'Оглавление', inset: sectionLevel, bodyIndex, subtitles: []};
+                        this.contents.push(curTitle);
+                    }
+
                     inSubtitle = true;
                     curSubtitle = {paraIndex, inset: sectionLevel, title: ''};
                     curTitle.subtitles.push(curSubtitle);
@@ -343,7 +362,7 @@ export default class BookParser {
                         sectionLevel--;
                     }
 
-                    if (tag == 'emphasis' || tag == 'strong') {
+                    if (tag == 'emphasis' || tag == 'strong' || tag == 'sup' || tag == 'sub') {
                         growParagraph(`</${tag}>`, 0);
                     }
 
@@ -483,6 +502,15 @@ export default class BookParser {
         return {fb2};
     }
 
+    imageHrefToId(id) {
+        let local = false;
+        if (id[0] == '#') {
+            id = id.substr(1);
+            local = true;
+        }
+        return {id, local};
+    }
+
     findParaIndex(bookPos) {
         let result = undefined;
         //дихотомия
@@ -507,7 +535,7 @@ export default class BookParser {
 
     splitToStyle(s) {
         let result = [];/*array of {
-            style: {bold: Boolean, italic: Boolean, center: Boolean, space: Number},
+            style: {bold: Boolean, italic: Boolean, sup: Boolean, sub: Boolean, center: Boolean, space: Number},
             image: {local: Boolean, inline: Boolean, id: String},
             text: String,
         }*/
@@ -530,6 +558,12 @@ export default class BookParser {
                 case 'emphasis':
                     style.italic = true;
                     break;
+                case 'sup': 
+                    style.sup = true;
+                    break;
+                case 'sub':
+                    style.sub = true;
+                    break;
                 case 'center':
                     style.center = true;
                     break;
@@ -542,28 +576,21 @@ export default class BookParser {
                 case 'image': {
                     let attrs = sax.getAttrsSync(tail);
                     if (attrs.href && attrs.href.value) {
-                        let id = attrs.href.value;
+                        image = this.imageHrefToId(attrs.href.value);
-                        let local = false;
+                        image.inline = false;
-                        if (id[0] == '#') {
+                        image.num = (attrs.num && attrs.num.value ? attrs.num.value : 0);
-                            id = id.substr(1);
-                            local = true;
-                        }
-                        image = {local, inline: false, id};
                     }
                     break;
                 }
                 case 'image-inline': {
                     let attrs = sax.getAttrsSync(tail);
                     if (attrs.href && attrs.href.value) {
-                        let id = attrs.href.value;
+                        const img = this.imageHrefToId(attrs.href.value);
-                        let local = false;
+                        img.inline = true;
-                        if (id[0] == '#') {
+                        img.num = (attrs.num && attrs.num.value ? attrs.num.value : 0);
-                            id = id.substr(1);
-                            local = true;
-                        }
                         result.push({
                             style: Object.assign({}, style),
-                            image: {local, inline: true, id},
+                            image: img,
                             text: ''
                         });
                     }
@@ -580,6 +607,12 @@ export default class BookParser {
                 case 'emphasis':
                     style.italic = false;
                     break;
+                case 'sup': 
+                    style.sup = false;
+                    break;
+                case 'sub':
+                    style.sub = false;
+                    break;
                 case 'center':
                     style.center = false;
                     break;
@@ -784,6 +817,7 @@ export default class BookParser {
                         paraIndex,
                         w: imageWidth,
                         h: imageHeight,
+                        num: part.image.num
                     }});
                     lines.push(line);
                     line = {begin: line.end + 1, parts: []};
@@ -794,7 +828,7 @@ export default class BookParser {
                 line.last = true;
                 line.parts.push({style, text: ' ',
                     image: {local: part.image.local, inline: false, id: part.image.id,
-                        imageLine: i, lineCount, paraIndex, w: imageWidth, h: imageHeight}
+                        imageLine: i, lineCount, paraIndex, w: imageWidth, h: imageHeight, num: part.image.num}
                 });
                 
                 continue;
@@ -806,7 +840,7 @@ export default class BookParser {
                     let imgH = (bin.h > this.fontSize ? this.fontSize : bin.h);
                     imgW += bin.w*imgH/bin.h;
                     line.parts.push({style, text: '',
-                        image: {local: part.image.local, inline: true, id: part.image.id}});
+                        image: {local: part.image.local, inline: true, id: part.image.id, num: part.image.num}});
                 }
             }
 

client/components/Reader/share/bookManager.js

@@ -4,7 +4,7 @@ import _ from 'lodash';
 import * as utils from '../../../share/utils';
 import BookParser from './BookParser';
 
-const maxDataSize = 300*1024*1024;//compressed bytes
+const maxDataSize = 500*1024*1024;//compressed bytes
 
 //локальный кэш метаданных книг, ограничение maxDataSize
 const bmMetaStore = localForage.createInstance({
@@ -39,7 +39,7 @@ class BookManager {
 
         this.saveRecentItem = _.debounce(() => {
             bmRecentStoreNew.setItem('recent-item', this.recentItem);
-            this.recentRev++;
+            this.recentRev = (this.recentRev < 1000 ? this.recentRev + 1 : 1);
             bmRecentStoreNew.setItem('rev', this.recentRev);
         }, 200, {maxWait: 300});
 
@@ -64,6 +64,12 @@ class BookManager {
 
             await this.cleanRecentBooks();
 
+            //TODO: убрать после 06.2021, когда bmRecentStoreOld устареет
+            {
+                await this.convertFileToDiskPrefix();
+                if (this.recentRev > 10)
+                    await bmRecentStoreOld.clear();
+            }
         } else {//TODO: убрать после 06.2021, когда bmRecentStoreOld устареет
             this.recentLast = await bmRecentStoreOld.getItem('recent-last');
             if (this.recentLast) {
@@ -163,7 +169,7 @@ class BookManager {
     }
 
     async deflateWithProgress(data, callback) {
-        const chunkSize = 128*1024;
+        const chunkSize = 512*1024;
         const deflator = new utils.pako.Deflate({level: 5});
 
         let chunkTotal = 1 + Math.floor(data.length/chunkSize);
@@ -197,7 +203,7 @@ class BookManager {
     }
 
     async inflateWithProgress(data, callback) {
-        const chunkSize = 64*1024;
+        const chunkSize = 512*1024;
         const inflator = new utils.pako.Inflate({to: 'string'});
 
         let chunkTotal = 1 + Math.floor(data.length/chunkSize);
@@ -404,16 +410,12 @@ class BookManager {
     }
 
     async setRecentBook(value) {
-        const result = this.metaOnly(value);
+        let result = this.metaOnly(value);
         result.touchTime = Date.now();
         result.deleted = 0;
 
-        if (this.recent[result.key] && this.recent[result.key].deleted) {
+        if (this.recent[result.key]) {
-            //восстановим из небытия пользовательские данные
+            result = Object.assign({}, this.recent[result.key], result);
-            if (!result.bookPos)
-                result.bookPos = this.recent[result.key].bookPos;
-            if (!result.bookPosSeen)
-                result.bookPosSeen = this.recent[result.key].bookPosSeen;
         }
 
         await this.recentSetLastKey(result.key);
@@ -453,6 +455,33 @@ class BookManager {
         return isDel;
     }
 
+    async convertFileToDiskPrefix() {
+        let isConverted = false;
+
+        const newRecent = {};
+        for (let key of Object.keys(this.recent)) {
+            let newKey = key;
+            let newUrl = this.recent[key].url;
+
+            if (newKey.indexOf('66696c65') == 0) {
+                newKey = newKey.replace(/^66696c65/, '6469736b');
+                if (newUrl)
+                    newUrl = newUrl.replace(/^file/, 'disk');
+                isConverted = true;
+            }
+
+            newRecent[newKey] = this.recent[key];
+            newRecent[newKey].key = newKey;
+            if (newUrl)
+                newRecent[newKey].url = newUrl;
+        }
+        if (isConverted) {
+            this.recent = newRecent;
+            await this.recentSetItem(null, true);
+        }
+        return isConverted;
+    }
+
     mostRecentBook() {
         if (this.recentLastKey) {
             return this.recent[this.recentLastKey];

client/components/Reader/versionHistory.js

@@ -1,4 +1,42 @@
 export const versionHistory = [
+{
+    showUntil: '2020-12-17',
+    header: '0.9.12 (2020-12-18)',
+    content:
+`
+<ul>
+    <li>добавлена вкладка "Изображения" в окно оглавления</li>
+    <li>настройки конвертирования вынесены в отдельную вкладку</li>
+    <li>добавлена кнопка для быстрого доступа к настройкам конвертирования</li>
+    <li>улучшения работы конвертеров</li>
+</ul>
+`
+},
+
+{
+    showUntil: '2020-12-08',
+    header: '0.9.11 (2020-12-09)',
+    content:
+`
+<ul>
+    <li>оптимизации, улучшения работы конвертеров</li>
+</ul>
+`
+},
+
+{
+    showUntil: '2020-12-10',
+    header: '0.9.10 (2020-12-03)',
+    content:
+`
+<ul>
+    <li>добавлена частичная поддержка формата Djvu</li>
+    <li>добавлена поддержка Rar-архивов</li>
+    <li>исправления багов</li>
+</ul>
+`
+},
+
 {
     showUntil: '2020-11-20',
     header: '0.9.9 (2020-11-21)',

client/components/share/Dialog.vue

@@ -1,5 +1,5 @@
 <template>
-    <q-dialog v-model="active">
+    <q-dialog v-model="active" no-route-dismiss>
         <div class="column bg-white no-wrap">
             <div class="header row">
                 <div class="caption col row items-center q-ml-md">

client/components/share/StdDialog.vue

@@ -1,5 +1,5 @@
 <template>
-    <q-dialog ref="dialog" v-model="active" @show="onShow" @hide="onHide">
+    <q-dialog ref="dialog" v-model="active" @show="onShow" @hide="onHide" no-route-dismiss>
         <slot></slot>
 
         <!--------------------------------------------------->

client/store/modules/config.js

@@ -10,18 +10,7 @@ const state = {
 const getters = {};
 
 // actions
-const actions = {
+const actions = {};
-    async loadConfig({ commit, state }) {
-        commit('setApiError', null, { root: true });
-        commit('setConfig', {});
-        try {
-            const config = await miscApi.loadConfig();
-            commit('setConfig', config);
-        } catch (e) {
-            commit('setApiError', e, { root: true });
-        }
-    },
-};
 
 // mutations
 const mutations = {

client/store/modules/reader.js

@@ -12,7 +12,7 @@ const readerActions = {
     'setPosition': 'Установить позицию',
     'search': 'Найти в тексте',
     'copyText': 'Скопировать текст со страницы',
-    'splitToPara': 'Обновить с разбиением на параграфы',
+    'convOptions': 'Настроить конвертирование',
     'refresh': 'Принудительно обновить книгу',
     'offlineMode': 'Автономный режим (без интернета)',
     'contents': 'Оглавление/закладки',
@@ -41,7 +41,7 @@ const toolButtons = [
     {name: 'setPosition', show: true},
     {name: 'search',      show: true},
     {name: 'copyText',    show: false},
-    {name: 'splitToPara', show: false},
+    {name: 'convOptions', show: true},
     {name: 'refresh',     show: true},
     {name: 'contents',    show: true},
     {name: 'libs',        show: true},
@@ -60,8 +60,8 @@ const hotKeys = [
     {name: 'scrolling', codes: ['Z']},
     {name: 'setPosition', codes: ['P']},
     {name: 'search', codes: ['Ctrl+F']},
-    {name: 'copyText', codes: ['Ctrl+C']},
+    {name: 'copyText', codes: ['Ctrl+C']},    
-    {name: 'splitToPara', codes: ['Shift+R']},
+    {name: 'convOptions', codes: ['Ctrl+M']},
     {name: 'refresh', codes: ['R']},
     {name: 'contents', codes: ['C']},
     {name: 'libs', codes: ['L']},
@@ -251,11 +251,16 @@ const settingDefaults = {
     compactTextPerc: 0,
     imageHeightLines: 100,
     imageFitWidth: true,
+    enableSitesFilter: true,
+    splitToPara: false,
+    djvuQuality: 20,
+    pdfAsText: true,
+    pdfQuality: 20,
+
     showServerStorageMessages: true,
     showWhatsNewDialog: true,
     showDonationDialog2020: true,
-    showLiberamaTopDialog2020: true,
+    showNeedUpdateNotify: true,
-    enableSitesFilter: true,
 
     fontShifts: {},
     showToolButton: {},

docs/beta/run_server.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-sudo -H -u www-data /home/beta.liberama/liberama
+sudo -H -u www-data bash -c "cd /var/www; /home/beta.liberama/liberama"

docs/omnireader.ru/README.md

@@ -27,15 +27,19 @@ sudo chown www-data.www-data /home/liberama
 
 ### external converter `calibre`, download from https://download.calibre-ebook.com/
 ```
-wget "https://download.calibre-ebook.com/3.39.1/calibre-3.39.1-x86_64.txz"
+wget "https://download.calibre-ebook.com/5.5.0/calibre-5.5.0-x86_64.txz"
 sudo -u www-data mkdir -p /home/liberama/data/calibre
-sudo -u www-data tar xvf calibre-3.39.1-x86_64.txz -C /home/liberama/data/calibre
+sudo -u www-data tar xvf calibre-5.5.0-x86_64.txz -C /home/liberama/data/calibre
 ```
 
 ### external converters
 ```
+sudo apt install rar
 sudo apt install libreoffice
 sudo apt install poppler-utils
+sudo apt install djvulibre-bin
+sudo apt install libtiff-tools
+sudo apt install graphicsmagick-imagemagick-compat
 ```
 
 ### nginx, server config

docs/omnireader.ru/cron_server.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if ! pgrep -x "liberama" > /dev/null ; then
-    sudo -H -u www-data /home/liberama/liberama
+    sudo -H -u www-data bash -c "cd /var/www; /home/liberama/liberama"
 else
     echo "Process 'liberama' already running"
 fi

docs/omnireader.ru/start_server.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-sudo -H -u www-data /home/liberama/liberama &
+sudo -H -u www-data bash -c "cd /var/www; /home/liberama/liberama" & disown
 sudo service cron start

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "Liberama",
-  "version": "0.9.9",
+  "version": "0.9.11",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -10124,6 +10124,21 @@
       "integrity": "sha512-ISBaA8xQNmwELC7eOjqFKMESB2VIqt4PPDD0nsS95b/9dZXvVKOlz9keMSnoGGKcOHXfTvDD6WMaRoSc9UuhRA==",
       "dev": true
     },
+    "pidusage": {
+      "version": "2.0.21",
+      "resolved": "https://registry.npmjs.org/pidusage/-/pidusage-2.0.21.tgz",
+      "integrity": "sha512-cv3xAQos+pugVX+BfXpHsbyz/dLzX+lr44zNMsYiGxUw+kV5sgQCIcLd1z+0vq+KyC7dJ+/ts2PsfgWfSC3WXA==",
+      "requires": {
+        "safe-buffer": "^5.2.1"
+      },
+      "dependencies": {
+        "safe-buffer": {
+          "version": "5.2.1",
+          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+          "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
+        }
+      }
+    },
     "pify": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "Liberama",
-  "version": "0.9.9",
+  "version": "0.9.12",
   "author": "Book Pauk <bookpauk@gmail.com>",
   "license": "CC0-1.0",
   "repository": "bookpauk/liberama",
@@ -8,7 +8,7 @@
     "node": ">=10.0.0"
   },
   "scripts": {
-    "dev": "nodemon --inspect --exec 'node server'",
+    "dev": "nodemon --inspect --ignore server/public --ignore server/data --ignore client --exec 'node server'",
     "build:client": "webpack --config build/webpack.prod.config.js",
     "build:linux": "npm run build:client && node build/linux && pkg -t latest-linux-x64 -o dist/linux/liberama .",
     "build:win": "npm run build:client && node build/win && pkg -t latest-win-x64 -o dist/win/liberama .",
@@ -72,6 +72,7 @@
     "multer": "^1.4.2",
     "pako": "^1.0.11",
     "path-browserify": "^1.0.0",
+    "pidusage": "^2.0.21",
     "quasar": "^1.14.3",
     "safe-buffer": "^5.2.0",
     "sjcl": "^1.0.8",

server/controllers/ReaderController.js

@@ -20,8 +20,12 @@ class ReaderController extends BaseController {
             const workerId = this.readerWorker.loadBookUrl({
                 url: request.url, 
                 enableSitesFilter: (request.hasOwnProperty('enableSitesFilter') ? request.enableSitesFilter : true),
-                skipCheck: (request.hasOwnProperty('skipCheck') ? request.skipCheck : false),
+                skipHtmlCheck: (request.hasOwnProperty('skipHtmlCheck') ? request.skipHtmlCheck : false),
                 isText: (request.hasOwnProperty('isText') ? request.isText : false),
+                uploadFileName: (request.hasOwnProperty('uploadFileName') ? request.uploadFileName : false),
+                djvuQuality: (request.hasOwnProperty('djvuQuality') ? request.djvuQuality : false),
+                pdfAsText: (request.hasOwnProperty('pdfAsText') ? request.pdfAsText : false),
+                pdfQuality: (request.hasOwnProperty('pdfQuality') ? request.pdfQuality : false),
             });
             const state = this.workerState.getState(workerId);
             return (state ? state : {});

server/controllers/WebSocketController.js

@@ -136,8 +136,9 @@ class WebSocketController {
                 break;
 
             i++;
-            if (i > 2*60*1000/refreshPause) {//2 мин ждем телодвижений воркера
+            if (i > 3*60*1000/refreshPause) {//3 мин ждем телодвижений воркера
                 this.send({state: 'error', error: 'Время ожидания процесса истекло'}, req, ws);
+                break;
             }
             i = (prevProgress != state.progress || prevState != state.state ? 1 : i);
         }        

server/core/FileDecompressor.js

@@ -15,6 +15,13 @@ class FileDecompressor {
     constructor(limitFileSize = 0) {
         this.detector = new FileDetector();
         this.limitFileSize = limitFileSize;
+
+        this.rarPath = '/usr/bin/rar';
+        this.rarExists = false;
+        (async() => {
+            if (await fs.pathExists(this.rarPath))
+                this.rarExists = true;
+        })();
     }
 
     async decompressNested(filename, outputDir) {
@@ -30,7 +37,11 @@ class FileDecompressor {
             files: []
         };
 
-        if (!fileType || !(fileType.ext == 'zip' || fileType.ext == 'bz2' || fileType.ext == 'gz' || fileType.ext == 'tar')) {
+        if (!fileType || !(
+                    fileType.ext == 'zip' || fileType.ext == 'bz2' || fileType.ext == 'gz'
+                    || fileType.ext == 'tar' || (this.rarExists && fileType.ext == 'rar')
+                )
+            ) {
             return result;
         }
 
@@ -94,6 +105,11 @@ class FileDecompressor {
     async decompress(fileExt, filename, outputDir) {
         let files = [];
 
+        if (fileExt == 'rar' && this.rarExists) {
+            files = await this.unRar(filename, outputDir);
+            return files;
+        }
+
         switch (fileExt) {
             case 'zip':
                 files = await this.unZip(filename, outputDir);
@@ -119,17 +135,16 @@ class FileDecompressor {
         try {
             return await zip.unpack(filename, outputDir, {
                 limitFileSize: this.limitFileSize, 
-                limitFileCount: 1000,
+                limitFileCount: 10000,
                 decodeEntryNameCallback: (nameRaw) => {
                     return utils.bufferRemoveZeroes(nameRaw);
                 }
-            }
+            });
-);
         } catch (e) {
             fs.emptyDir(outputDir);
             return await zip.unpack(filename, outputDir, {
                 limitFileSize: this.limitFileSize, 
-                limitFileCount: 1000,
+                limitFileCount: 10000,
                 decodeEntryNameCallback: (nameRaw) => {
                     nameRaw = utils.bufferRemoveZeroes(nameRaw);
                     const enc = textUtils.getEncodingLite(nameRaw);
@@ -156,7 +171,7 @@ class FileDecompressor {
 
             if (this.limitFileSize) {
                 if ((await fs.stat(filename)).size > this.limitFileSize) {
-                    reject('Файл слишком большой');
+                    reject(new Error('Файл слишком большой'));
                     return;
                 }
             }
@@ -222,7 +237,39 @@ class FileDecompressor {
         
             inputStream.pipe(stream).pipe(outputStream);
         })().catch(reject); });
-   }
+    }
+
+    async unRar(filename, outputDir) {
+        try {
+            const args = ['x', '-p-', '-y', filename, `${outputDir}`];
+            const result = await utils.spawnProcess(this.rarPath, {
+                killAfter: 60,
+                args
+            });
+
+            if (result.code == 0) {
+                const files = [];
+                await utils.findFiles(async(file) => {
+                    const stat = await fs.stat(file);
+                    files.push({path: path.relative(outputDir, file), size: stat.size});
+                }, outputDir);
+
+                return files;
+
+            } else {
+                const error = `${result.code}|FORLOG|, exec: ${this.rarPath}, args: ${args.join(' ')}, stdout: ${result.stdout}, stderr: ${result.stderr}`;
+                throw new Error(`Архиватор Rar завершился с ошибкой: ${error}`);
+            }
+        } catch(e) {
+            if (e.status == 'killed') {
+                throw new Error('Слишком долгое ожидание архиватора Rar');
+            } else if (e.status == 'error') {
+                throw new Error(e.error);
+            } else {
+                throw new Error(e);
+            }
+        }
+    }
 
     async gzipBuffer(buf) {
         return new Promise((resolve, reject) => {

server/core/FileDetector/index.js

@@ -54,7 +54,7 @@ class FileDetector {
 
     fromBuffer(buffer, callback) {
         let result = null;
-
+        //console.log(buffer);
         const invalidSignaturesList = this.validateSigantures();
         if (invalidSignaturesList.length) {
             return callback(invalidSignaturesList);

server/core/FileDetector/signatures.json

@@ -676,7 +676,9 @@
         { "type": "equal", "end": 19, "bytes": "3c3f786d6c2076657273696f6e3d22312e3022" },
         { "type": "equal", "end": 22, "bytes": "efbbbf3c3f786d6c2076657273696f6e3d22312e3022" },
         { "type": "equal", "end": 19, "bytes": "3c3f786d6c2076657273696f6e3d27312e3027" },
-        { "type": "equal", "end": 22, "bytes": "efbbbf3c3f786d6c2076657273696f6e3d27312e3027" }
+        { "type": "equal", "end": 22, "bytes": "efbbbf3c3f786d6c2076657273696f6e3d27312e3027" },
+        { "type": "equal", "end": 40, "bytes": "fffe3c003f0078006d006c002000760065007200730069006f006e003d00220031002e0030002200" },
+        { "type": "equal", "end": 40, "bytes": "fffe3c003f0078006d006c002000760065007200730069006f006e003d00270031002e0030002700" }
       ]
       }
     ]

server/core/LimitedQueue.js

@@ -28,7 +28,7 @@ class LimitedQueue {
     get(onPlaceChange) {
         return new Promise((resolve, reject) => {
             if (this.destroyed)
-                reject('destroyed');
+                reject(new Error('destroyed'));
 
             const take = () => {
                 if (this.freed <= 0)
@@ -73,7 +73,7 @@ class LimitedQueue {
                     if (onPlaceChange)
                         onPlaceChange(this.listeners.length);
                 } else {
-                    reject('Превышен размер очереди ожидания');
+                    reject(new Error('Превышен размер очереди ожидания'));
                 }
             }
         });

server/core/Reader/BookConverter/.gitignore

@@ -1 +0,0 @@
-test

server/core/Reader/BookConverter/ConvertBase.js

@@ -5,8 +5,9 @@ const he = require('he');
 const LimitedQueue = require('../../LimitedQueue');
 const textUtils = require('./textUtils');
 const utils = require('../../utils');
+const xmlParser = require('../../xmlParser');
 
-const queue = new LimitedQueue(2, 20, 3*60*1000);//3 минуты ожидание подвижек
+const queue = new LimitedQueue(3, 20, 2*60*1000);//2 минуты ожидание подвижек
 
 class ConvertBase {
     constructor(config) {
@@ -14,7 +15,6 @@ class ConvertBase {
 
         this.calibrePath = `${config.dataDir}/calibre/ebook-convert`;
         this.sofficePath = '/usr/bin/soffice';
-        this.pdfToHtmlPath = '/usr/bin/pdftohtml';
     }
 
     async run(data, opts) {// eslint-disable-line no-unused-vars
@@ -27,9 +27,6 @@ class ConvertBase {
 
         if (!await fs.pathExists(this.sofficePath))
             throw new Error('Внешний конвертер LibreOffice не найден');
-
-        if (!await fs.pathExists(this.pdfToHtmlPath))
-            throw new Error('Внешний конвертер pdftohtml не найден');
     }
 
     async execConverter(path, args, onData, abort) {
@@ -42,22 +39,38 @@ class ConvertBase {
             throw new Error('Слишком большая очередь конвертирования. Пожалуйста, попробуйте позже.');
         }
 
+        abort = (abort ? abort : () => false);
+        const myAbort = () => {
+            return q.abort() || abort();
+        }
+
         try {
+            if (myAbort())
+                throw new Error('abort');
+
             const result = await utils.spawnProcess(path, {
-                killAfter: 600,
+                killAfter: 3600,//1 час
                 args, 
                 onData: (data) => {
-                    q.resetTimeout();
+                    if (queue.freed > 0)
+                        q.resetTimeout();
                     onData(data);
                 },
-                abort
+                //будем периодически проверять работу конвертера и если очереди нет, то разрешаем работу пинком onData
+                onUsage: (stats) => {
+                    if (queue.freed > 0 && stats.cpu >= 10) {
+                        q.resetTimeout();
+                        onData('.');
+                    }
+                },
+                onUsageInterval: 10,
+                abort: myAbort
             });
             if (result.code != 0) {
-                let error = result.code;
+                const error = `${result.code}|FORLOG|, exec: ${path}, args: ${args.join(' ')}, stdout: ${result.stdout}, stderr: ${result.stderr}`;
-                if (this.config.branch == 'development')
-                    error = `exec: ${path}, stdout: ${result.stdout}, stderr: ${result.stderr}`;
                 throw new Error(`Внешний конвертер завершился с ошибкой: ${error}`);
             }
+            return result;
         } catch(e) {
             if (e.status == 'killed') {
                 throw new Error('Слишком долгое ожидание конвертера');
@@ -90,62 +103,20 @@ class ConvertBase {
         return he.escape(he.decode(text.replace(/ /g, ' ')));
     }
 
-    formatFb2(fb2) {
+    isDataXml(data) {
-        let out = '<?xml version="1.0" encoding="utf-8"?>';
+        const str = data.slice(0, 100).toString().trim();
-        out += '<FictionBook xmlns="http://www.gribuser.ru/xml/fictionbook/2.0" xmlns:l="http://www.w3.org/1999/xlink">';
+        return (str.indexOf('<?xml version="1.0"') == 0 || str.indexOf('<?xml version=\'1.0\'') == 0 );
-        out += this.formatFb2Node(fb2);
-        out += '</FictionBook>';
-        return out;
     }
 
-    formatFb2Node(node, name) {
+    formatFb2(fb2) {
-        let out = '';
+        const out = xmlParser.formatXml({
-
+            FictionBook: {
-        if (Array.isArray(node)) {
+                _attrs: {xmlns: 'http://www.gribuser.ru/xml/fictionbook/2.0', 'xmlns:l': 'http://www.w3.org/1999/xlink'},
-            for (const n of node) {
+                _a: [fb2],
-                out += this.formatFb2Node(n);
             }
-        } else if (typeof node == 'string') {
+        }, 'utf-8', this.repSpaces);
-            if (name)
-                out += `<${name}>${this.repSpaces(node)}</${name}>`;
-            else
-                out += this.repSpaces(node);
-        } else {
-            if (node._n)
-                name = node._n;
 
-            let attrs = '';
+        return out.replace(/<p>\s*?<\/p>/g, '<empty-line/>');
-            if (node._attrs) {
-                for (let attrName in node._attrs) {
-                    attrs += ` ${attrName}="${node._attrs[attrName]}"`;
-                }
-            }
-
-            let tOpen = '';
-            let tBody = '';
-            let tClose = '';
-            if (name)
-                tOpen += `<${name}${attrs}>`;
-            if (node.hasOwnProperty('_t'))
-                tBody += this.repSpaces(node._t);
-
-            for (let nodeName in node) {
-                if (nodeName && nodeName[0] == '_' && nodeName != '_a')
-                    continue;
-
-                const n = node[nodeName];
-                tBody += this.formatFb2Node(n, nodeName);
-            }
-            
-            if (name)
-                tClose += `</${name}>`;
-
-            if (attrs == '' && name == 'p' && tBody.trim() == '')
-                out += '<empty-line/>'
-            else
-                out += `${tOpen}${tBody}${tClose}`;
-        }
-        return out;
     }
 }
 

server/core/Reader/BookConverter/ConvertDjvu.js

@@ -0,0 +1,119 @@
+const fs = require('fs-extra');
+const path = require('path');
+const utils = require('../../utils');
+
+const ConvertJpegPng = require('./ConvertJpegPng');
+
+class ConvertDjvu extends ConvertJpegPng {
+    check(data, opts) {
+        const {inputFiles} = opts;
+
+        return this.config.useExternalBookConverter && 
+            inputFiles.sourceFileType && inputFiles.sourceFileType.ext == 'djvu';
+    }
+
+    async run(data, opts) {
+        if (!this.check(data, opts))
+            return false;
+
+        let {inputFiles, callback, abort, djvuQuality} = opts;
+
+        djvuQuality = (djvuQuality && djvuQuality <= 100 && djvuQuality >= 10 ? djvuQuality : 20);
+        let jpegQuality = djvuQuality;
+        let tiffQuality = djvuQuality + 30;
+        tiffQuality = (tiffQuality < 85 ? tiffQuality : 85);
+
+        const ddjvuPath = '/usr/bin/ddjvu';
+        if (!await fs.pathExists(ddjvuPath))
+            throw new Error('Внешний конвертер ddjvu не найден');
+
+        const djvusedPath = '/usr/bin/djvused';
+        if (!await fs.pathExists(djvusedPath))
+            throw new Error('Внешний конвертер djvused не найден');
+
+        const tiffsplitPath = '/usr/bin/tiffsplit';
+        if (!await fs.pathExists(tiffsplitPath))
+            throw new Error('Внешний конвертер tiffsplitPath не найден');
+
+        const mogrifyPath = '/usr/bin/mogrify';
+        if (!await fs.pathExists(mogrifyPath))
+            throw new Error('Внешний конвертер mogrifyPath не найден');
+
+        const dir = `${inputFiles.filesDir}/`;
+        const baseFile = `${dir}${path.basename(inputFiles.sourceFile)}`;
+        const tifFile = `${baseFile}.tif`;
+
+        //конвертируем в tiff
+        let perc = 0;
+        await this.execConverter(ddjvuPath, ['-format=tiff', `-quality=${tiffQuality}`, '-verbose', inputFiles.sourceFile, tifFile], () => {
+            perc = (perc < 100 ? perc + 1 : 40);
+            callback(perc);
+        }, abort);
+
+        const tifFileSize = (await fs.stat(tifFile)).size;
+        let limitSize = 4*this.config.maxUploadFileSize;
+        if (tifFileSize > limitSize) {
+            throw new Error(`Файл для конвертирования слишком большой|FORLOG| tifFileSize: ${tifFileSize} > ${limitSize}`);
+        }
+
+        //разбиваем на файлы
+        await this.execConverter(tiffsplitPath, [tifFile, dir], null, abort);
+
+        await fs.remove(tifFile);
+
+        //конвертируем в jpg
+        await this.execConverter(mogrifyPath, ['-quality', jpegQuality, '-scale', '2048>', '-verbose', '-format', 'jpg', `${dir}*.tif`], () => {
+            perc = (perc < 100 ? perc + 1 : 40);
+            callback(perc);
+        }, abort);
+
+        limitSize = 2*this.config.maxUploadFileSize;
+        let jpgFilesSize = 0;
+        //ищем изображения
+        let files = [];
+        await utils.findFiles(async(file) => {
+            if (path.extname(file) == '.jpg') {
+                jpgFilesSize += (await fs.stat(file)).size;
+                if (jpgFilesSize > limitSize) {
+                    throw new Error(`Файл для конвертирования слишком большой|FORLOG| jpgFilesSize: ${jpgFilesSize} > ${limitSize}`);
+                }
+
+                files.push({name: file, base: path.basename(file)});
+            }
+        }, dir);
+
+        files.sort((a, b) => a.base.localeCompare(b.base));
+
+        //схема документа (outline)
+        const djvusedResult = await this.execConverter(djvusedPath, ['-u', '-e', 'print-outline', inputFiles.sourceFile], null, abort);
+
+        const outline = [];
+        const lines = djvusedResult.stdout.match(/\(\s*".*"\s*?"#\d+"/g);
+        if (lines) {
+            lines.forEach(l => {
+                const m = l.match(/"(.*)"\s*?"#(\d+)"/);
+                if (m) {
+                    const pageNum = m[2];
+                    let s = outline[pageNum];
+                    if (!s)
+                        s = m[1].trim();
+                    else
+                        s += `${(s[s.length - 1] != '.' ? '.' : '')} ${m[1].trim()}`;
+
+                    outline[pageNum] = s;
+                }
+            });
+        }
+
+        await utils.sleep(100);
+        let i = 0;
+        const imageFiles = files.map(f => {
+            i++;
+            let alt = (outline[i] ? outline[i] : '');
+            return {src: f.name, alt};
+        });
+        return await super.run(data, Object.assign({}, opts, {imageFiles}));
+    }
+}
+
+module.exports = ConvertDjvu;

server/core/Reader/BookConverter/ConvertFb2.js

@@ -1,34 +1,71 @@
 const ConvertBase = require('./ConvertBase');
 const iconv = require('iconv-lite');
+const textUtils = require('./textUtils');
 
 class ConvertFb2 extends ConvertBase {
     check(data, opts) {
         const {dataType} = opts;
 
-        return (dataType && dataType.ext == 'xml' && data.toString().indexOf('<FictionBook') >= 0);
+        return (
+            ( (dataType && dataType.ext == 'xml') || this.isDataXml(data) ) && 
+            data.toString().indexOf('<FictionBook') >= 0
+        );
     }
 
     async run(data, opts) {
-        if (!this.check(data, opts))
+        let newData = data.slice(0, 1024);
+
+        //Корректируем кодировку для проверки, 16-битные кодировки должны стать utf-8
+        const encoding = textUtils.getEncoding(newData);
+        if (encoding.indexOf('UTF-16') == 0) {
+            newData = Buffer.from(iconv.decode(newData, encoding));
+        }
+
+        //Проверяем
+        if (!this.check(newData, opts))
             return false;
 
-        return this.checkEncoding(data);
+        //Корректируем кодировку всего объема
+        newData = data;
+        if (encoding.indexOf('UTF-16') == 0) {
+            newData = Buffer.from(iconv.decode(newData, encoding));
+        }
+
+        //Корректируем пробелы, всякие файлы попадаются :(
+        if (newData[0] == 32) {
+            newData = Buffer.from(newData.toString().trim());
+        }
+
+        //Окончательно корректируем кодировку
+        return this.checkEncoding(newData);
     }
 
     checkEncoding(data) {
         let result = data;
 
-        const left = data.indexOf('<?xml version="1.0"');
+        let q = '"';
+        let left = data.indexOf('<?xml version="1.0"');
+        if (left < 0) {
+            left = data.indexOf('<?xml version=\'1.0\'');
+            q = '\'';
+        }
+
         if (left >= 0) {
             const right = data.indexOf('?>', left);
             if (right >= 0) {
                 const head = data.slice(left, right + 2).toString();
-                const m = head.match(/encoding="(.*?)"/);
+                const m = head.match(/encoding=['"](.*?)['"]/);
                 if (m) {
                     let encoding = m[1].toLowerCase();
                     if (encoding != 'utf-8') {
-                        result = iconv.decode(data, encoding);
+                        //encoding может не соответсвовать реальной кодировке файла, поэтому:
-                        result = Buffer.from(result.toString().replace(m[0], 'encoding="utf-8"'));
+                        let calcEncoding = textUtils.getEncoding(data);
+                        if (calcEncoding.indexOf('ISO-8859') >= 0) {
+                            calcEncoding = encoding;
+                        }
+
+                        result = iconv.decode(data, calcEncoding);
+                        result = Buffer.from(result.toString().replace(m[0], `encoding=${q}utf-8${q}`));
                     }
                 }
             }

server/core/Reader/BookConverter/ConvertFb3.js

@@ -2,7 +2,7 @@ const fs = require('fs-extra');
 
 const ConvertHtml = require('./ConvertHtml');
 
-class ConvertDocX extends ConvertHtml {
+class ConvertFb3 extends ConvertHtml {
     async check(data, opts) {
         const {inputFiles} = opts;
         if (this.config.useExternalBookConverter && 
@@ -39,13 +39,14 @@ class ConvertDocX extends ConvertHtml {
         const title = this.getTitle(text)
             .replace(/<\/?p>/g, '')
         ;
-        text = `<title>${title}</title>` + text
+        text = `<fb2-title>${title}</fb2-title>` + text
             .replace(/<title>/g, '<br><b>')
             .replace(/<\/title>/g, '</b><br>')
-            .replace(/<subtitle>/g, '<br><br><subtitle>')
+            .replace(/<subtitle>/g, '<br><br><fb2-subtitle>')
+            .replace(/<\/subtitle>/g, '</fb2-subtitle>')
         ;
-        return await super.run(Buffer.from(text), {skipCheck: true, cutTitle: true});
+        return await super.run(Buffer.from(text), {skipHtmlCheck: true});
     }
 }
 
-module.exports = ConvertDocX;
+module.exports = ConvertFb3;

server/core/Reader/BookConverter/ConvertHtml.js

@@ -7,7 +7,7 @@ class ConvertHtml extends ConvertBase {
         const {dataType} = opts;
 
         //html?
-        if (dataType && (dataType.ext == 'html' || dataType.ext == 'xml')) 
+        if ( ( (dataType && (dataType.ext == 'html' || dataType.ext == 'xml')) ) || this.isDataXml(data) )
             return {isText: false};
 
         //может это чистый текст?
@@ -16,7 +16,7 @@ class ConvertHtml extends ConvertBase {
         }
 
         //из буфера обмена?
-        if (data.toString().indexOf('<buffer>') == 0) {
+        if (data.slice(0, 50).toString().indexOf('<buffer>') == 0) {
             return {isText: false};
         }
 
@@ -24,17 +24,14 @@ class ConvertHtml extends ConvertBase {
     }
 
     async run(data, opts) {
-        let isText = false;
+        let {isText = false, uploadFileName = ''} = opts;
-        if (!opts.skipCheck) {
+        if (!opts.skipHtmlCheck) {
             const checkResult = this.check(data, opts);
             if (!checkResult)
                 return false;
 
             isText = checkResult.isText;
-        } else {
-            isText = opts.isText;
         }
-        let {cutTitle} = opts;
 
         let titleInfo = {};
         let desc = {_n: 'description', 'title-info': titleInfo};
@@ -44,12 +41,17 @@ class ConvertHtml extends ConvertBase {
         let fb2 = [desc, body, binary];
 
         let title = '';
+        let author = '';
         let inTitle = false;
+        let inSectionTitle = false;
+        let inAuthor = false;
         let inSubTitle = false;
         let inImage = false;
         let image = {};
         let bold = false;
         let italic = false;
+        let superscript = false;
+        let subscript = false;
         let begining = true;
 
         let spaceCounter = [];
@@ -62,7 +64,7 @@ class ConvertHtml extends ConvertBase {
         };
 
         const growParagraph = (text) => {
-            if (!pars.length)
+            if (!pars.length || pars[pars.length - 1]._n != 'p')
                 newParagraph();
 
             const l = pars.length;
@@ -94,12 +96,16 @@ class ConvertHtml extends ConvertBase {
         const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
             text = this.escapeEntities(text);
 
-            if (!cutCounter && !(cutTitle && inTitle)) {
+            if (!(cutCounter || inTitle || inSectionTitle || inSubTitle)) {
                 let tOpen = '';
                 tOpen += (inSubTitle ? '<subtitle>' : '');
                 tOpen += (bold ? '<strong>' : '');
                 tOpen += (italic ? '<emphasis>' : '');
+                tOpen += (superscript ? '<sup>' : '');
+                tOpen += (subscript ? '<sub>' : '');
                 let tClose = ''
+                tClose += (subscript ? '</sub>' : '');
+                tClose += (superscript ? '</sup>' : '');
                 tClose +=  (italic ? '</emphasis>' : '');
                 tClose += (bold ? '</strong>' : '');
                 tClose += (inSubTitle ? '</subtitle>' : '');
@@ -110,12 +116,22 @@ class ConvertHtml extends ConvertBase {
             if (inTitle && !title)
                 title = text;
 
+            if (inAuthor && !author)
+                author = text;
+
+            if (inSectionTitle) {
+                pars.unshift({_n: 'title', _t: text});
+            }
+
+            if (inSubTitle) {
+                pars.push({_n: 'subtitle', _t: text});
+            }
+
             if (inImage) {
                 image._t = text;
                 binary.push(image);
 
                 pars.push({_n: 'image', _attrs: {'l:href': '#' + image._attrs.id}, _t: ''});
-                newParagraph();
             }
 
         };
@@ -140,15 +156,27 @@ class ConvertHtml extends ConvertBase {
                         bold = true;
                         break;
                 }
+
+                if (tag == 'sup')
+                    superscript = true;
+        
+                if (tag == 'sub')
+                    subscript = true;
             }
 
-            if (tag == 'title' || tag == 'cut-title') {
+            if (tag == 'title' || tag == 'fb2-title') {
                 inTitle = true;
-                if (tag == 'cut-title')
-                    cutTitle = true;
             }
 
-            if (tag == 'subtitle') {
+            if (tag == 'fb2-author') {
+                inAuthor = true;
+            }
+
+            if (tag == 'fb2-section-title') {
+                inSectionTitle = true;
+            }
+
+            if (tag == 'fb2-subtitle') {
                 inSubTitle = true;
             }
 
@@ -156,7 +184,7 @@ class ConvertHtml extends ConvertBase {
                 inImage = true;
                 const attrs = sax.getAttrsSync(tail);
                 image = {_n: 'binary', _attrs: {id: attrs.name.value, 'content-type': attrs.type.value}, _t: ''};
-            }
+            }            
         };
 
         const onEndNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
@@ -179,12 +207,26 @@ class ConvertHtml extends ConvertBase {
                         bold = false;
                         break;
                 }
+
+                if (tag == 'sup')
+                    superscript = false;
+        
+                if (tag == 'sub')
+                    subscript = false;
             }
 
-            if (tag == 'title' || tag == 'cut-title')
+            if (tag == 'title' || tag == 'fb2-title')
                 inTitle = false;
 
-            if (tag == 'subtitle')
+            if (tag == 'fb2-author') {
+                inAuthor = false;
+            }
+
+            if (tag == 'fb2-section-title') {
+                inSectionTitle = false;
+            }
+
+            if (tag == 'fb2-subtitle')
                 inSubTitle = false;
 
             if (tag == 'fb2-image')
@@ -195,10 +237,20 @@ class ConvertHtml extends ConvertBase {
 
         sax.parseSync(buf, {
             onStartNode, onEndNode, onTextNode,
-            innerCut: new Set(['head', 'script', 'style', 'binary', 'fb2-image'])
+            innerCut: new Set(['head', 'script', 'style', 'binary', 'fb2-image', 'fb2-title', 'fb2-author'])
         });
 
+        if (!title)
+            title = uploadFileName;
+        
         titleInfo['book-title'] = title;
+        if (author)
+            titleInfo.author = {'last-name': author};
+
+        body.section._a[0] = pars;
+
+        //console.log(JSON.stringify(fb2, null, 2));
+
         //подозрение на чистый текст, надо разбить на параграфы
         if (isText || (buf.length > 30*1024 && pars.length < buf.length/2000)) {
             let total = 0;
@@ -218,7 +270,7 @@ class ConvertHtml extends ConvertBase {
 
             let i = 0;
             //если разброс не слишком большой, выделяем параграфы
-            if (d < 10 && spaceCounter.length) {
+            if (d < 20 && spaceCounter.length) {
                 total /= 20;
                 i = spaceCounter.length - 1;
                 while (i > 0 && (!spaceCounter[i] || spaceCounter[i] < total)) i--;
@@ -228,56 +280,49 @@ class ConvertHtml extends ConvertBase {
             if (parIndent > 2) parIndent--;
 
             let newPars = [];
+            let curPar = {};
             const newPar = () => {
-                newPars.push({_n: 'p', _t: ''});
+                curPar = {_n: 'p', _t: ''};
+                newPars.push(curPar);
             };
 
-            const growPar = (text) => {
-                if (!newPars.length)
-                    newPar();
-
-                const l = newPars.length;
-                newPars[l - 1]._t += text;
-            }
-
-            i = 0;
             for (const par of pars) {
                 if (par._n != 'p') {
                     newPars.push(par);
                     continue;
                 }
 
-                if (i > 0)
+                newPar();
-                    newPar();
+                
-                i++;
-
-                let j = 0;
                 const lines = par._t.split('\n');
-                for (let line of lines) {
+                for (let j = 0; j < lines.length; j++) {
-                    line = repCrLfTab(line);
+                    const line = repCrLfTab(lines[j]);
 
                     let l = 0;
                     while (l < line.length && line[l] == ' ') {
                         l++;
                     }
 
-                    if (l >= parIndent || line == '') {
+                    if (j > 0 &&
-                        if (j > 0)
+                        (l >= parIndent ||
-                            newPar();
+                            (j < lines.length - 1 && line == '')
-                        j++;
+                        )
+                    ) {
+                        newPar();
                     }
-                    growPar(line.trim() + ' ');
+
+                    curPar._t += line.trim() + ' ';
                 }
             }
 
             body.section._a[0] = newPars;
-        } else {
-            body.section._a[0] = pars;
         }
 
         //убираем лишнее, делаем валидный fb2, т.к. в рез-те разбиения на параграфы бьются теги
         bold = false;
         italic = false;
+        superscript = false;
+        subscript = false;
         inSubTitle = false;
         pars = body.section._a[0];
         for (let i = 0; i < pars.length; i++) {
@@ -297,7 +342,11 @@ class ConvertHtml extends ConvertBase {
                     tOpen += (inSubTitle ? '<subtitle>' : '');
                     tOpen += (bold ? '<strong>' : '');
                     tOpen += (italic ? '<emphasis>' : '');
+                    tOpen += (superscript ? '<sup>' : '');
+                    tOpen += (subscript ? '<sub>' : '');
                     let tClose = ''
+                    tClose += (subscript ? '</sub>' : '');
+                    tClose += (superscript ? '</sup>' : '');
                     tClose +=  (italic ? '</emphasis>' : '');
                     tClose += (bold ? '</strong>' : '');
                     tClose += (inSubTitle ? '</subtitle>' : '');
@@ -313,6 +362,10 @@ class ConvertHtml extends ConvertBase {
                         bold = true;
                     if (tag == 'emphasis')
                         italic = true;
+                    if (tag == 'sup')
+                        superscript = true;
+                    if (tag == 'sub')
+                        subscript = true;
                     if (tag == 'subtitle')
                         inSubTitle = true;
                 }
@@ -322,6 +375,10 @@ class ConvertHtml extends ConvertBase {
                         bold = false;
                     if (tag == 'emphasis')
                         italic = false;
+                    if (tag == 'sup')
+                        superscript = false;
+                    if (tag == 'sub')
+                        subscript = false;
                     if (tag == 'subtitle')
                         inSubTitle = false;
                 }

server/core/Reader/BookConverter/ConvertJpegPng.js

@@ -0,0 +1,100 @@
+const fs = require('fs-extra');
+const path = require('path');
+//const utils = require('../../utils');
+
+const ConvertBase = require('./ConvertBase');
+
+class ConvertJpegPng extends ConvertBase {
+    check(data, opts) {
+        const {inputFiles} = opts;
+
+        return this.config.useExternalBookConverter && 
+            inputFiles.sourceFileType && 
+            (inputFiles.sourceFileType.ext == 'jpg' || inputFiles.sourceFileType.ext == 'png' );
+    }
+
+    async run(data, opts) {
+        const {inputFiles, uploadFileName, imageFiles} = opts;
+
+        if (!imageFiles) {
+            if (!this.check(data, opts))
+                return false;
+        }
+
+        let files = [];
+        if (imageFiles) {
+            files = imageFiles;
+        } else {
+            const imageFile = `${inputFiles.filesDir}/${path.basename(inputFiles.sourceFile)}.${inputFiles.sourceFileType.ext}`;
+            await fs.copy(inputFiles.sourceFile, imageFile);
+            files.push({src: imageFile});
+        }
+
+        //читаем изображения
+        const limitSize = 2*this.config.maxUploadFileSize;
+        let imagesSize = 0;
+
+        const loadImage = async(image) => {
+            const src = path.parse(image.src);
+            let type = 'unknown';
+            switch (src.ext) {
+                case '.jpg': type = 'image/jpeg'; break;
+                case '.png': type = 'image/png'; break;
+            }
+            if (type != 'unknown') {
+                image.data = (await fs.readFile(image.src)).toString('base64');
+                image.type = type;
+                image.name = src.base;
+
+                imagesSize += image.data.length;
+                if (imagesSize > limitSize) {
+                    throw new Error(`Файл для конвертирования слишком большой|FORLOG| imagesSize: ${imagesSize} > ${limitSize}`);
+                }
+            }
+        }
+
+        let images = [];
+        let loading = [];
+        files.forEach(img => {
+            images.push(img);
+            loading.push(loadImage(img));
+        });
+
+        await Promise.all(loading);
+
+        //формируем fb2
+        let titleInfo = {};
+        let desc = {_n: 'description', 'title-info': titleInfo};
+        let pars = [];
+        let body = {_n: 'body', section: {_a: [pars]}};
+        let binary = [];
+        let fb2 = [desc, body, binary];
+
+        let title = '';
+        if (uploadFileName)
+            title = uploadFileName;
+
+        titleInfo['book-title'] = title;
+
+        for (const image of images) {
+            if (image.type) {
+                const img = {_n: 'binary', _attrs: {id: image.name, 'content-type': image.type}, _t: image.data};
+                binary.push(img);
+
+                const attrs = {'l:href': `#${image.name}`};
+                if (image.alt) {
+                    image.alt = (image.alt.length > 256 ? image.alt.substring(0, 256) : image.alt);
+                    attrs.alt = image.alt;
+                }
+
+                pars.push({_n: 'p', _t: ''});
+                pars.push({_n: 'image', _attrs: attrs});
+            }
+        }
+        pars.push({_n: 'p', _t: ''});
+        
+        return this.formatFb2(fb2);
+    }
+}
+
+module.exports = ConvertJpegPng;

server/core/Reader/BookConverter/ConvertPdf.js

@@ -1,3 +1,4 @@
+//const _ = require('lodash');
 const fs = require('fs-extra');
 const path = require('path');
 
@@ -14,17 +15,24 @@ class ConvertPdf extends ConvertHtml {
     }
 
     async run(notUsed, opts) {
-        if (!this.check(notUsed, opts))
+        if (!opts.pdfAsText || !this.check(notUsed, opts))
             return false;
+
         await this.checkExternalConverterPresent();
 
-        const {inputFiles, callback, abort} = opts;
+        const {inputFiles, callback, abort, uploadFileName} = opts;
 
-        const outFile = `${inputFiles.filesDir}/${utils.randomHexString(10)}.xml`;
+        const inpFile = inputFiles.sourceFile;
+        const outBasename = `${inputFiles.filesDir}/${utils.randomHexString(10)}`;
+        const outFile = `${outBasename}.xml`;
+
+        const pdftohtmlPath = '/usr/bin/pdftohtml';
+        if (!await fs.pathExists(pdftohtmlPath))
+            throw new Error('Внешний конвертер pdftohtml не найден');
 
         //конвертируем в xml
         let perc = 0;
-        await this.execConverter(this.pdfToHtmlPath, ['-c', '-s', '-xml', inputFiles.sourceFile, outFile], () => {
+        await this.execConverter(pdftohtmlPath, ['-nodrm', '-c', '-s', '-xml', inpFile, outFile], () => {
             perc = (perc < 80 ? perc + 10 : 40);
             callback(perc);
         }, abort);
@@ -33,17 +41,24 @@ class ConvertPdf extends ConvertHtml {
         const data = await fs.readFile(outFile);
         callback(90);
 
+        await utils.sleep(100);
+
         //парсим xml
         let lines = [];
+        let pagelines = [];
+        let line = {text: ''};
+        let page = {};
+        let fonts = {};
+        let sectionTitleFound = false;
+
         let images = [];
         let loading = [];
+
         let inText = false;
         let bold = false;
         let italic = false;
-        let title = '';
+
-        let prevTop = 0;
         let i = -1;
-        let titleCount = 0;
 
         const loadImage = async(image) => {
             const src = path.parse(image.src);
@@ -57,7 +72,7 @@ class ConvertPdf extends ConvertHtml {
                 image.type = type;
                 image.name = src.base;
             }
-        }
+        };
 
         const putImage = (curTop) => {
             if (!isNaN(curTop) && images.length) {
@@ -67,7 +82,72 @@ class ConvertPdf extends ConvertHtml {
                     images.shift();
                 }
             }
-        }
+        };
+
+        const isTextBold = (text) => {
+            const m = text.trim().match(/^<b>(.*)<\/b>$/);
+            return m && !m[1].match(/<b>|<\/b>|<i>|<\/i>/g);
+        };
+
+        const isTextEmpty = (text) => {
+            return text.replace(/<b>|<\/b>|<i>|<\/i>/g, '').trim() == '';
+        };
+
+        const putPageLines = () => {
+            pagelines.sort((a, b) => (Math.abs(a.top - b.top) > 3 ? a.top - b.top : 0)*10000 + (a.left - b.left))
+            
+            //объединяем в одну строку равные по высоте
+            const pl = [];
+            let pt = 0;
+            let j = -1;
+            pagelines.forEach(line => {
+                if (isTextEmpty(line.text))
+                    return;
+
+                //проверим, возможно это заголовок
+                if (line.fontId && line.pageWidth) {
+                    const centerLeft = (line.pageWidth - line.width)/2;
+                    if (isTextBold(line.text) && Math.abs(centerLeft - line.left) < 10) {
+                        if (!sectionTitleFound) {
+                            line.isSectionTitle = true;
+                            sectionTitleFound = true;
+                        } else {
+                            line.isSubtitle = true;
+                        }
+                    }
+                }
+
+                //объединяем
+                if (pt == 0 || Math.abs(pt - line.top) > 3) {
+                    j++;
+                    pl[j] = line;
+                } else {
+                    pl[j].text += ` ${line.text}`;
+                }
+                pt = line.top;
+            });
+
+            //заполняем lines
+            const lastIndex = i;
+            pl.forEach(line => {
+                putImage(line.top);
+
+                //добавим пустую строку, если надо
+                const prevLine = (i > lastIndex ? lines[i] : {fonts: [], top: 0});
+                if (prevLine && !prevLine.isImage) {
+                    const f = (prevLine.fontId ? fonts[prevLine.fontId] : (line.fontId ? fonts[line.fontId] : null));
+                    if (f && f.fontSize && !line.isImage && line.top - prevLine.top > f.fontSize * 1.8) {
+                        i++;
+                        lines[i] = {text: '<br>'};
+                    }
+                }
+
+                i++;
+                lines[i] = line;
+            });
+            pagelines = [];
+            putImage(100000);
+        };
 
         const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
             if (!cutCounter && inText) {
@@ -76,67 +156,80 @@ class ConvertPdf extends ConvertHtml {
                 let tClose = (italic ? '</i>' : '');
                 tClose += (bold ? '</b>' : '');
 
-                lines[i].text += `${tOpen}${text}${tClose} `;
+                line.text += ` ${tOpen}${text}${tClose}`;
-                if (titleCount < 2 && text.trim() != '') {
-                    title += text + (titleCount ? '' : ' - ');
-                    titleCount++;
-                }
             }
         };
 
         const onStartNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
-            if (!cutCounter) {
+            if (inText) {
-                if (inText) {
+                switch (tag) {
-                    switch (tag) {
+                    case 'i':
-                        case 'i':
+                        italic = true;
-                            italic = true;
+                        break;
-                            break;
+                    case 'b':
-                        case 'b':
+                        bold = true;
-                            bold = true;
+                        break;
-                            break;
-                    }
                 }
+            }
 
-                if (tag == 'text' && !inText) {
+            if (tag == 'page') {
-                    let attrs = sax.getAttrsSync(tail);
+                const attrs = sax.getAttrsSync(tail);
-                    const line = {
+                page = {
-                        text: '',
+                    width: parseInt((attrs.width && attrs.width.value ? attrs.width.value : null), 10),
-                        top: parseInt((attrs.top && attrs.top.value ? attrs.top.value : null), 10),
+                };
-                        left: parseInt((attrs.left && attrs.left.value ? attrs.left.value : null), 10),
+
-                        width: parseInt((attrs.width && attrs.width.value ? attrs.width.value : null), 10),
+                putPageLines();
-                        height: parseInt((attrs.height && attrs.height.value ? attrs.height.value : null), 10),
+            }
+
+            if (tag == 'fontspec') {
+                const attrs = sax.getAttrsSync(tail);
+                const fontId = (attrs.id && attrs.id.value ? attrs.id.value : '');
+                const fontSize = (attrs.size && attrs.size.value ? attrs.size.value : '');
+
+                if (fontId) {
+                    fonts[fontId] = {fontSize};
+
+                }
+            }
+
+            if (tag == 'text' && !inText) {
+                const attrs = sax.getAttrsSync(tail);
+                line = {
+                    text: '',
+                    top: parseInt((attrs.top && attrs.top.value ? attrs.top.value : null), 10),
+                    left: parseInt((attrs.left && attrs.left.value ? attrs.left.value : null), 10),
+                    width: parseInt((attrs.width && attrs.width.value ? attrs.width.value : null), 10),
+                    height: parseInt((attrs.height && attrs.height.value ? attrs.height.value : null), 10),
+                    isSectionTitle: false,
+                    isSubtitle: false,
+                    pageWidth: page.width,
+                    fontId: (attrs.font && attrs.font.value ? attrs.font.value : ''),
+                };
+
+                if (line.width != 0 || line.height != 0) {
+                    inText = true;
+                    pagelines.push(line);
+                }
+            }
+
+            if (tag == 'image') {
+                const attrs = sax.getAttrsSync(tail);
+                let src = (attrs.src && attrs.src.value ? attrs.src.value : '');
+                if (src) {
+                    const image = {
+                        isImage: true,
+                        src,
+                        data: '',
+                        type: '',
+                        top: parseInt((attrs.top && attrs.top.value ? attrs.top.value : null), 10) || 0,
+                        left: parseInt((attrs.left && attrs.left.value ? attrs.left.value : null), 10) || 0,
+                        width: parseInt((attrs.width && attrs.width.value ? attrs.width.value : null), 10) || 0,
+                        height: parseInt((attrs.height && attrs.height.value ? attrs.height.value : null), 10) || 0,
                     };
 
-                    if (line.width != 0 || line.height != 0) {
+                    loading.push(loadImage(image));
-                        inText = true;
+                    images.push(image);
-                        if (isNaN(line.top) || isNaN(prevTop) || (Math.abs(prevTop - line.top) > 3)) {
+                    images.sort((a, b) => (a.top - b.top)*10000 + (a.left - b.left));
-                            putImage(line.top);
-                            i++;
-                            lines[i] = line;
-                        }
-                        prevTop = line.top;
-                    }
-                }
-
-                if (tag == 'image') {
-                    const attrs = sax.getAttrsSync(tail);
-                    const src = (attrs.src && attrs.src.value ? attrs.src.value : '');
-                    if (src) {
-                        const image = {
-                            isImage: true,
-                            src,
-                            data: '',
-                            type: '',
-                            top: parseInt((attrs.top && attrs.top.value ? attrs.top.value : null), 10) || 0,
-                        };
-                        loading.push(loadImage(image));
-                        images.push(image);
-                        images.sort((a, b) => a.top - b.top)
-                    }
-                }
-
-                if (tag == 'page') {
-                    putImage(100000);
                 }
             }
         };
@@ -162,9 +255,10 @@ class ConvertPdf extends ConvertHtml {
             onStartNode, onEndNode, onTextNode
         });
 
-        putImage(100000);
+        putPageLines();
 
         await Promise.all(loading);
+        await utils.sleep(100);
 
         //найдем параграфы и отступы
         const indents = [];
@@ -185,16 +279,49 @@ class ConvertPdf extends ConvertHtml {
         }
         indents[0] = 0;
 
+        //author & title
+        let {author, title} = await this.getPdfTitleAndAuthor(inpFile);
+
+        if (!title && uploadFileName)
+            title = uploadFileName;
+
+        //console.log(JSON.stringify(lines, null, 2));
         //формируем текст
-        let text = `<title>${title}</title>`;
+        const limitSize = 2*this.config.maxUploadFileSize;
+        let text = '';
+        if (title)
+            text += `<fb2-title>${title}</fb2-title>`;
+        if (author)
+            text += `<fb2-author>${author}</fb2-author>`;
+
         let concat = '';
         let sp = '';
+        let firstLine = true;
         for (const line of lines) {
+            if (text.length > limitSize) {
+                throw new Error(`Файл для конвертирования слишком большой|FORLOG| text.length: ${text.length} > ${limitSize}`);
+            }
+            
             if (line.isImage) {
                 text += `<fb2-image type="${line.type}" name="${line.name}">${line.data}</fb2-image>`;
                 continue;
             }
 
+            if (line.isSectionTitle) {
+                if (firstLine)
+                    text += `<fb2-section-title>${line.text.trim()}</fb2-section-title>`;
+                else
+                    text += `<fb2-subtitle>${line.text.trim()}</fb2-subtitle>`;
+                continue;
+            }
+
+            firstLine = false;
+
+            if (line.isSubtitle) {
+                text += `<br><fb2-subtitle>${line.text.trim()}</fb2-subtitle>`;
+                continue;
+            }
+
             if (concat == '') {
                 const left = line.left || 0;
                 sp = ' '.repeat(indents[left]);
@@ -212,8 +339,36 @@ class ConvertPdf extends ConvertHtml {
         if (concat)
             text += sp + concat + "\n";
 
-        return await super.run(Buffer.from(text), {skipCheck: true, isText: true, cutTitle: true});
+        //console.log(text);
+        await utils.sleep(100);
+        return await super.run(Buffer.from(text), {skipHtmlCheck: true, isText: true});
+    }
+
+    async getPdfTitleAndAuthor(pdfFile) {
+        const result = {author: '', title: ''};
+
+        const pdfinfoPath = '/usr/bin/pdfinfo';
+
+        if (!await fs.pathExists(pdfinfoPath))
+            throw new Error('Внешний конвертер pdfinfo не найден');
+
+        const execResult = await this.execConverter(pdfinfoPath, [pdfFile]);
+
+        const titlePrefix = 'Title:';
+        const authorPrefix = 'Author:';
+
+        const stdout = execResult.stdout.split("\n");
+        stdout.forEach(line => {
+            if (line.indexOf(titlePrefix) == 0) 
+                result.title = line.substring(titlePrefix.length).trim();
+
+            if (line.indexOf(authorPrefix) == 0)
+                result.author = line.substring(authorPrefix.length).trim();
+        });
+
+        return result;
     }
 }
 
+
 module.exports = ConvertPdf;

server/core/Reader/BookConverter/ConvertPdfImages.js

@@ -0,0 +1,115 @@
+const fs = require('fs-extra');
+const path = require('path');
+const utils = require('../../utils');
+
+const sax = require('../../sax');
+
+const ConvertJpegPng = require('./ConvertJpegPng');
+
+class ConvertPdfImages extends ConvertJpegPng {
+    check(data, opts) {
+        const {inputFiles} = opts;
+
+        return this.config.useExternalBookConverter && 
+            inputFiles.sourceFileType && inputFiles.sourceFileType.ext == 'pdf';
+    }
+
+    async run(data, opts) {
+        if (!this.check(data, opts))
+            return false;
+
+        let {inputFiles, callback, abort, pdfQuality} = opts;
+        
+        pdfQuality = (pdfQuality && pdfQuality <= 100 && pdfQuality >= 10 ? pdfQuality : 20);
+
+        const pdftoppmPath = '/usr/bin/pdftoppm';
+        if (!await fs.pathExists(pdftoppmPath))
+            throw new Error('Внешний конвертер pdftoppm не найден');
+
+        const pdftohtmlPath = '/usr/bin/pdftohtml';
+        if (!await fs.pathExists(pdftohtmlPath))
+            throw new Error('Внешний конвертер pdftohtml не найден');
+
+        const inpFile = inputFiles.sourceFile;
+        const dir = `${inputFiles.filesDir}/`;
+        const outBasename = `${dir}${utils.randomHexString(10)}`;
+        const outFile = `${outBasename}.tmp`;
+
+        //конвертируем в jpeg
+        let perc = 0;
+        await this.execConverter(pdftoppmPath, ['-jpeg', '-jpegopt', `quality=${pdfQuality},progressive=y`, inpFile, outFile], () => {
+            perc = (perc < 100 ? perc + 1 : 40);
+            callback(perc);
+        }, abort);
+
+        const limitSize = 2*this.config.maxUploadFileSize;
+        let jpgFilesSize = 0;
+
+        //ищем изображения
+        let files = [];
+        await utils.findFiles(async(file) => {
+            if (path.extname(file) == '.jpg') {
+                jpgFilesSize += (await fs.stat(file)).size;
+                if (jpgFilesSize > limitSize) {
+                    throw new Error(`Файл для конвертирования слишком большой|FORLOG| jpgFilesSize: ${jpgFilesSize} > ${limitSize}`);
+                }
+
+                files.push({name: file, base: path.basename(file)});
+            }
+        }, dir);
+
+        files.sort((a, b) => a.base.localeCompare(b.base));
+
+        //схема документа (outline)
+        const outXml = `${outBasename}.xml`;
+        await this.execConverter(pdftohtmlPath, ['-nodrm', '-i', '-c', '-s', '-xml', inpFile, outXml], null, abort);
+        const outline = [];
+
+        let inOutline = 0;
+        let inItem = false;
+        let pageNum = 0;
+
+        const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
+            if (inOutline > 0 && inItem && pageNum) {
+                outline[pageNum] = text;
+            }
+        };
+
+        const onStartNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
+            if (tag == 'outline')
+                inOutline++;
+
+            if (inOutline > 0 && tag == 'item') {
+                const attrs = sax.getAttrsSync(tail);
+                pageNum = (attrs.page && attrs.page.value ? attrs.page.value : 0);
+                inItem = true;
+            }
+        };
+
+        const onEndNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
+            if (tag == 'outline')
+                inOutline--;
+            if (tag == 'item')
+                inItem = false;
+        };
+
+        const dataXml = await fs.readFile(outXml);
+        const buf = this.decode(dataXml).toString();
+        sax.parseSync(buf, {
+            onStartNode, onEndNode, onTextNode
+        });
+
+
+        await utils.sleep(100);
+        //формируем список файлов
+        let i = 0;
+        const imageFiles = files.map(f => {
+            i++;
+            let alt = (outline[i] ? outline[i] : '');
+            return {src: f.name, alt};
+        });
+        return await super.run(data, Object.assign({}, opts, {imageFiles}));
+    }
+}
+
+module.exports = ConvertPdfImages;

server/core/Reader/BookConverter/ConvertSites.js

@@ -48,7 +48,7 @@ class ConvertSites extends ConvertHtml {
         if (text === false)
             return false;
 
-        return await super.run(Buffer.from(text), {skipCheck: true, cutTitle: true});
+        return await super.run(Buffer.from(text), {skipHtmlCheck: true});
     }
 
     getTitle(text) {
@@ -79,7 +79,7 @@ class ConvertSites extends ConvertHtml {
         let book = this.getTitle(text);
         book = book.replace(' (fb2) | Флибуста', '');
 
-        const title = `<title>${author}${(author ? ' - ' : '')}${book}</title>`;
+        const title = `<fb2-title>${author}${(author ? ' - ' : '')}${book}</fb2-title>`;
 
         let begin = '<h3 class="book">';
         if (text.indexOf(begin) <= 0)
@@ -95,12 +95,12 @@ class ConvertSites extends ConvertHtml {
         return text.substring(l, r)
             .replace(/blockquote class="?book"?/g, 'p')
             .replace(/<br\/?>\s*<\/h3>/g, '</h3>')
-            .replace(/<h3 class="?book"?>/g, '<br><br><subtitle>')
+            .replace(/<h3 class="?book"?>/g, '<br><br><fb2-subtitle>')
-            .replace(/<h5 class="?book"?>/g, '<br><br><subtitle>')
+            .replace(/<h5 class="?book"?>/g, '<br><br><fb2-subtitle>')
-            .replace(/<h3>/g, '<br><br><subtitle>')
+            .replace(/<h3>/g, '<br><br><fb2-subtitle>')
-            .replace(/<h5>/g, '<br><br><subtitle>')
+            .replace(/<h5>/g, '<br><br><fb2-subtitle>')
-            .replace(/<\/h3>/g, '</subtitle><br>')
+            .replace(/<\/h3>/g, '</fb2-subtitle><br>')
-            .replace(/<\/h5>/g, '</subtitle><br>')
+            .replace(/<\/h5>/g, '</fb2-subtitle><br>')
             .replace(/<div class="?stanza"?>/g, '<br>')
             .replace(/<div>/g, '<br>')
             + title;

server/core/Reader/BookConverter/index.js

@@ -3,8 +3,11 @@ const FileDetector = require('../../FileDetector');
 
 //порядок важен
 const convertClassFactory = [
+    require('./ConvertJpegPng'),
     require('./ConvertEpub'),
+    require('./ConvertDjvu'),
     require('./ConvertPdf'),
+    require('./ConvertPdfImages'),
     require('./ConvertRtf'),
     require('./ConvertDocX'),
     require('./ConvertFb3'),

server/core/Reader/ReaderWorker.js

@@ -12,7 +12,7 @@ const utils = require('../utils');
 const log = new (require('../AppLogger'))().log;//singleton
 
 const cleanDirPeriod = 60*60*1000;//1 раз в час
-const queue = new LimitedQueue(5, 100, 5*60*1000);//5 минут ожидание подвижек
+const queue = new LimitedQueue(5, 100, 2*60*1000 + 15000);//2 минуты ожидание подвижек
 
 let instance = null;
 
@@ -30,7 +30,7 @@ class ReaderWorker {
 
             this.workerState = new WorkerState();
             this.down = new FileDownloader(config.maxUploadFileSize);
-            this.decomp = new FileDecompressor(2*config.maxUploadFileSize);
+            this.decomp = new FileDecompressor(3*config.maxUploadFileSize);
             this.bookConverter = new BookConverter(this.config);
 
             this.remoteWebDavStorage = false;
@@ -81,7 +81,7 @@ class ReaderWorker {
             const decompDirname = utils.randomHexString(30);
 
             //download or use uploaded
-            if (url.indexOf('file://') != 0) {//download
+            if (url.indexOf('disk://') != 0) {//download
                 const downdata = await this.down.load(url, (progress) => {
                     wState.set({progress});
                 }, q.abort);
@@ -130,6 +130,8 @@ class ReaderWorker {
             convertFilename = `${this.config.tempDownloadDir}/${tempFilename2}`;
             await this.bookConverter.convertToFb2(decompFiles, convertFilename, opts, progress => {
                 wState.set({progress});
+                if (queue.freed > 0)
+                    q.resetTimeout();
             }, q.abort);
 
             //сжимаем файл в tmp, если там уже нет с тем же именем-sha256
@@ -171,10 +173,15 @@ class ReaderWorker {
 
         } catch (e) {
             log(LM_ERR, e.stack);
+            let mes = e.message.split('|FORLOG|');
+            if (mes[1])
+                log(LM_ERR, mes[0] + mes[1]);
             log(LM_ERR, `downloadedFilename: ${downloadedFilename}`);
-            if (e.message == 'abort')
+
-                e.message = overLoadMes;
+            mes = mes[0];
-            wState.set({state: 'error', error: e.message});
+            if (mes == 'abort')
+                mes = overLoadMes;
+            wState.set({state: 'error', error: mes});
         } finally {
             //clean
             if (q)
@@ -209,7 +216,7 @@ class ReaderWorker {
             await fs.remove(file.path);
         }
 
-        return `file://${hash}`;
+        return `disk://${hash}`;
     }
 
     async restoreRemoteFile(filename) {

server/core/Zip/ZipStreamer.js

@@ -76,13 +76,13 @@ class ZipStreamer {
                 if (limitFileCount || limitFileSize || decodeEntryNameCallback) {
                     const entries = Object.values(unzip.entries());
                     if (limitFileCount && entries.length > limitFileCount) {
-                        reject('Слишком много файлов');
+                        reject(new Error('Слишком много файлов'));
                         return;
                     }
 
                     for (const entry of entries) {
                         if (limitFileSize && !entry.isDirectory && entry.size > limitFileSize) {
-                            reject('Файл слишком большой');
+                            reject(new Error('Файл слишком большой'));
                             return;
                         }
 

server/core/sax.js

@@ -6,7 +6,8 @@ function parseSync(xstr, options) {
         onCdata: _onCdata = dummy,
         onComment: _onComment = dummy,
         onProgress: _onProgress = dummy,
-        innerCut = new Set()
+        innerCut = new Set(),
+        lowerCase = true,
     } = options;
 
     let i = 0;
@@ -91,7 +92,8 @@ function parseSync(xstr, options) {
             } else {
                 tag = tagData;
             }
-            tag = tag.toLowerCase();
+            if (lowerCase)
+                tag = tag.toLowerCase();
 
             if (innerCut.has(tag) && (!cutCounter || cutTag === tag)) {
                 if (!cutCounter)
@@ -146,7 +148,8 @@ async function parse(xstr, options) {
         onCdata: _onCdata = dummy,
         onComment: _onComment = dummy,
         onProgress: _onProgress = dummy,
-        innerCut = new Set()
+        innerCut = new Set(),
+        lowerCase = true,
     } = options;
 
     let i = 0;
@@ -231,7 +234,8 @@ async function parse(xstr, options) {
             } else {
                 tag = tagData;
             }
-            tag = tag.toLowerCase();
+            if (lowerCase)
+                tag = tag.toLowerCase();
 
             if (innerCut.has(tag) && (!cutCounter || cutTag === tag)) {
                 if (!cutCounter)
@@ -276,7 +280,7 @@ async function parse(xstr, options) {
     await _onProgress(100);
 }
 
-function getAttrsSync(tail) {
+function getAttrsSync(tail, lowerCase = true) {
     let result = {};
     let name = '';    
     let value = '';
@@ -287,13 +291,16 @@ function getAttrsSync(tail) {
     let waitEq = false;
 
     const pushResult = () => {
+        if (lowerCase)
+            name = name.toLowerCase();
         if (name != '') {
+            const fn = name;
             let ns = '';
-            if (name.indexOf(':') >= 0) {
+            if (fn.indexOf(':') >= 0) {
-                [ns, name] = name.split(':');
+                [ns, name] = fn.split(':');
             }
 
-            result[name] = {value, ns};
+            result[name] = {value, ns, fn};
         }
         name = '';
         value = '';

server/core/utils.js

@@ -1,7 +1,9 @@
 const { spawn } = require('child_process');
 const fs = require('fs-extra');
+const path = require('path');
 const crypto = require('crypto');
 const baseX = require('base-x');
+const pidusage = require('pidusage');
 
 const BASE36 = '0123456789abcdefghijklmnopqrstuvwxyz';
 const bs36 = baseX(BASE36);
@@ -45,10 +47,11 @@ async function touchFile(filename) {
 }
 
 function spawnProcess(cmd, opts) {
-    let {args, killAfter, onData, abort} = opts;
+    let {args, killAfter, onData, onUsage, onUsageInterval, abort} = opts;
     killAfter = (killAfter ? killAfter : 120);//seconds
     onData = (onData ? onData : () => {});
     args = (args ? args : []);
+    onUsageInterval = (onUsageInterval ? onUsageInterval : 30);//seconds
 
     return new Promise((resolve, reject) => { (async() => {
         let resolved = false;
@@ -75,9 +78,19 @@ function spawnProcess(cmd, opts) {
             reject({status: 'error', error, stdout, stderr});
         });
 
+        //ждем процесс, контролируем его работу раз в секунду
+        let onUsageCounter = onUsageInterval;
         while (!resolved) {
             await sleep(1000);
-            killAfter -= 1;
+
+            onUsageCounter--;
+            if (onUsage && onUsageCounter <= 0) {
+                const stats = await pidusage(proc.pid);
+                onUsage(stats);
+                onUsageCounter = onUsageInterval;
+            }
+
+            killAfter--;
             if (killAfter <= 0 || (abort && abort())) {
                 process.kill(proc.pid);
                 if (killAfter <= 0) {
@@ -91,6 +104,22 @@ function spawnProcess(cmd, opts) {
     })().catch(reject); });
 }
 
+async function findFiles(callback, dir) {
+    if (!(callback && dir))
+        return;
+    let result = true;
+    const files = await fs.readdir(dir, { withFileTypes: true });
+
+    for (const file of files) {
+        const found = path.resolve(dir, file.name);
+        if (file.isDirectory())
+            result = await findFiles(callback, found);
+        else
+            await callback(found);
+    }
+    return result;
+}
+
 module.exports = {
     toBase36,
     fromBase36,
@@ -99,5 +128,6 @@ module.exports = {
     sleep,
     randomHexString,
     touchFile,
-    spawnProcess
+    spawnProcess,
+    findFiles
 };

server/core/xmlParser.js

@@ -0,0 +1,143 @@
+const sax = require('./sax');
+
+function formatXml(xmlParsed, encoding = 'utf-8', textFilterFunc) {
+    let out = `<?xml version="1.0" encoding="${encoding}"?>`;
+    out += formatXmlNode(xmlParsed, textFilterFunc);
+    return out;
+}
+
+function formatXmlNode(node, textFilterFunc) {
+    textFilterFunc = (textFilterFunc ? textFilterFunc : text => text);
+
+    const formatNode = (node, name) => {
+        let out = '';
+
+        if (Array.isArray(node)) {
+            for (const n of node) {
+                out += formatNode(n);
+            }
+        } else if (typeof node == 'string') {
+            if (name)
+                out += `<${name}>${textFilterFunc(node)}</${name}>`;
+            else
+                out += textFilterFunc(node);
+        } else {
+            if (node._n)
+                name = node._n;
+
+            let attrs = '';
+            if (node._attrs) {
+                for (let attrName in node._attrs) {
+                    attrs += ` ${attrName}="${node._attrs[attrName]}"`;
+                }
+            }
+
+            let tOpen = '';
+            let tBody = '';
+            let tClose = '';
+            if (name)
+                tOpen += `<${name}${attrs}>`;
+            if (node.hasOwnProperty('_t'))
+                tBody += textFilterFunc(node._t);
+
+            for (let nodeName in node) {
+                if (nodeName && nodeName[0] == '_' && nodeName != '_a')
+                    continue;
+
+                const n = node[nodeName];
+                tBody += formatNode(n, nodeName);
+            }
+            
+            if (name)
+                tClose += `</${name}>`;
+
+            out += `${tOpen}${tBody}${tClose}`;
+        }
+        return out;
+    }
+
+    return formatNode(node);
+}
+
+function parseXml(xmlString, lowerCase = true) {
+    let result = {};
+    let node = result;
+
+    const onTextNode = (text, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
+        node._t = text;
+    };
+
+    const onStartNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
+        if (tag == '?xml')
+            return;
+        
+        const newNode = {_n: tag, _p: node};
+
+        if (tail) {
+            const parsedAttrs = sax.getAttrsSync(tail, lowerCase);
+            const atKeys = Object.keys(parsedAttrs);
+            if (atKeys.length) {
+                const attrs = {};
+                for (let i = 0; i < atKeys.length; i++) {
+                    const attrName = atKeys[i];
+                    attrs[parsedAttrs[attrName].fn] = parsedAttrs[attrName].value;
+                }
+
+                newNode._attrs = attrs;
+            }
+        }
+
+        if (!node._a)
+            node._a = [];
+        node._a.push(newNode);
+        node = newNode;
+    };
+
+    const onEndNode = (tag, tail, singleTag, cutCounter, cutTag) => {// eslint-disable-line no-unused-vars
+        if (node._p && node._n == tag)
+            node = node._p;
+    };
+
+    sax.parseSync(xmlString, {
+        onStartNode, onEndNode, onTextNode, lowerCase
+    });
+
+    if (result._a)
+        result = result._a[0];
+    
+    return result;
+}
+
+function simplifyXmlParsed(node) {
+    
+    const simplifyNodeArray = (a) => {
+        const result = {};
+
+        for (let i = 0; i < a.length; i++) {
+            const child = a[i];
+            if (child._n && !result[child._n]) {
+                result[child._n] = {};
+                if (child._a) {
+                    result[child._n] = simplifyNodeArray(child._a);
+                }
+                if (child._t) {
+                    result[child._n]._t = child._t;
+                }
+                if (child._attrs) {
+                    result[child._n]._attrs = child._attrs;
+                }
+            }
+        }
+
+        return result;
+    };
+
+    return simplifyNodeArray([node]);
+}
+
+module.exports = {
+    formatXml,
+    formatXmlNode,
+    parseXml,
+    simplifyXmlParsed
+}