Merge branch 'Develop' into master

cps/__init__.py

@@ -73,7 +73,6 @@ ub.init_db(cli.settingspath)
 # pylint: disable=no-member
 config = config_sql.load_configuration(ub.session)
 
-searched_ids = {}
 web_server = WebServer()
 
 babel = Babel()
@@ -83,6 +82,8 @@ log = logger.create()
 
 from . import services
 
+db.CalibreDB.setup_db(config, cli.settingspath)
+
 calibre_db = db.CalibreDB()
 
 def create_app():
@@ -91,7 +92,7 @@ def create_app():
     if sys.version_info < (3, 0):
         app.static_folder = app.static_folder.decode('utf-8')
         app.root_path = app.root_path.decode('utf-8')
-        app.instance_path = app.instance_path .decode('utf-8')
+        app.instance_path = app.instance_path.decode('utf-8')
 
     cache_buster.init_cache_busting(app)
 
@@ -101,8 +102,6 @@ def create_app():
     app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))
 
     web_server.init_app(app, config)
-    calibre_db.setup_db(config, cli.settingspath)
-    calibre_db.start()
 
     babel.init_app(app)
     _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations())

cps/config_sql.py

@@ -287,7 +287,7 @@ class _ConfigSQL(object):
                 db_file = os.path.join(self.config_calibre_dir, 'metadata.db')
                 have_metadata_db = os.path.isfile(db_file)
         self.db_configured = have_metadata_db
-        constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip() for x in self.config_upload_formats.split(',')]
+        constants.EXTENSIONS_UPLOAD = [x.lstrip().rstrip().lower() for x in self.config_upload_formats.split(',')]
         logfile = logger.setup(self.config_logfile, self.config_log_level)
         if logfile != self.config_logfile:
             log.warning("Log path %s not valid, falling back to default", self.config_logfile)

cps/constants.py

@@ -81,10 +81,11 @@ SIDEBAR_PUBLISHER       = 1 << 12
 SIDEBAR_RATING          = 1 << 13
 SIDEBAR_FORMAT          = 1 << 14
 SIDEBAR_ARCHIVED        = 1 << 15
-# SIDEBAR_LIST            = 1 << 16
+SIDEBAR_DOWNLOAD        = 1 << 16
+SIDEBAR_LIST            = 1 << 17
 
 ADMIN_USER_ROLES        = sum(r for r in ALL_ROLES.values()) & ~ROLE_ANONYMOUS
-ADMIN_USER_SIDEBAR      = (SIDEBAR_ARCHIVED << 1) - 1
+ADMIN_USER_SIDEBAR      = (SIDEBAR_LIST << 1) - 1
 
 UPDATE_STABLE       = 0 << 0
 AUTO_UPDATE_STABLE  = 1 << 0

cps/db.py

@@ -24,14 +24,13 @@ import re
 import ast
 import json
 from datetime import datetime
-import threading
 
 from sqlalchemy import create_engine
 from sqlalchemy import Table, Column, ForeignKey, CheckConstraint
 from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float
 from sqlalchemy.orm import relationship, sessionmaker, scoped_session
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm.collections import InstrumentedList
+from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
 from sqlalchemy.pool import StaticPool
 from flask_login import current_user
 from sqlalchemy.sql.expression import and_, true, false, text, func, or_
@@ -43,47 +42,48 @@ from flask_babel import gettext as _
 from . import logger, ub, isoLanguages
 from .pagination import Pagination
 
+from weakref import WeakSet
+
 try:
     import unidecode
     use_unidecode = True
 except ImportError:
     use_unidecode = False
 
-
 cc_exceptions = ['datetime', 'comments', 'composite', 'series']
 cc_classes = {}
 
 Base = declarative_base()
 
 books_authors_link = Table('books_authors_link', Base.metadata,
-    Column('book', Integer, ForeignKey('books.id'), primary_key=True),
-    Column('author', Integer, ForeignKey('authors.id'), primary_key=True)
-    )
+                           Column('book', Integer, ForeignKey('books.id'), primary_key=True),
+                           Column('author', Integer, ForeignKey('authors.id'), primary_key=True)
+                           )
 
 books_tags_link = Table('books_tags_link', Base.metadata,
-    Column('book', Integer, ForeignKey('books.id'), primary_key=True),
-    Column('tag', Integer, ForeignKey('tags.id'), primary_key=True)
-    )
+                        Column('book', Integer, ForeignKey('books.id'), primary_key=True),
+                        Column('tag', Integer, ForeignKey('tags.id'), primary_key=True)
+                        )
 
 books_series_link = Table('books_series_link', Base.metadata,
-    Column('book', Integer, ForeignKey('books.id'), primary_key=True),
-    Column('series', Integer, ForeignKey('series.id'), primary_key=True)
-    )
+                          Column('book', Integer, ForeignKey('books.id'), primary_key=True),
+                          Column('series', Integer, ForeignKey('series.id'), primary_key=True)
+                          )
 
 books_ratings_link = Table('books_ratings_link', Base.metadata,
-    Column('book', Integer, ForeignKey('books.id'), primary_key=True),
-    Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True)
-    )
+                           Column('book', Integer, ForeignKey('books.id'), primary_key=True),
+                           Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True)
+                           )
 
 books_languages_link = Table('books_languages_link', Base.metadata,
-    Column('book', Integer, ForeignKey('books.id'), primary_key=True),
-    Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True)
-    )
+                             Column('book', Integer, ForeignKey('books.id'), primary_key=True),
+                             Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True)
+                             )
 
 books_publishers_link = Table('books_publishers_link', Base.metadata,
-    Column('book', Integer, ForeignKey('books.id'), primary_key=True),
-    Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True)
-    )
+                              Column('book', Integer, ForeignKey('books.id'), primary_key=True),
+                              Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True)
+                              )
 
 
 class Identifiers(Base):
@@ -171,6 +171,9 @@ class Comments(Base):
         self.text = text
         self.book = book
 
+    def get(self):
+        return self.text
+
     def __repr__(self):
         return u"<Comments({0})>".format(self.text)
 
@@ -184,6 +187,9 @@ class Tags(Base):
     def __init__(self, name):
         self.name = name
 
+    def get(self):
+        return self.name
+
     def __repr__(self):
         return u"<Tags('{0})>".format(self.name)
 
@@ -201,6 +207,9 @@ class Authors(Base):
         self.sort = sort
         self.link = link
 
+    def get(self):
+        return self.name
+
     def __repr__(self):
         return u"<Authors('{0},{1}{2}')>".format(self.name, self.sort, self.link)
 
@@ -216,6 +225,9 @@ class Series(Base):
         self.name = name
         self.sort = sort
 
+    def get(self):
+        return self.name
+
     def __repr__(self):
         return u"<Series('{0},{1}')>".format(self.name, self.sort)
 
@@ -229,6 +241,9 @@ class Ratings(Base):
     def __init__(self, rating):
         self.rating = rating
 
+    def get(self):
+        return self.rating
+
     def __repr__(self):
         return u"<Ratings('{0}')>".format(self.rating)
 
@@ -242,6 +257,12 @@ class Languages(Base):
     def __init__(self, lang_code):
         self.lang_code = lang_code
 
+    def get(self):
+        if self.language_name:
+            return self.language_name
+        else:
+            return self.lang_code
+
     def __repr__(self):
         return u"<Languages('{0}')>".format(self.lang_code)
 
@@ -257,13 +278,16 @@ class Publishers(Base):
         self.name = name
         self.sort = sort
 
+    def get(self):
+        return self.name
+
     def __repr__(self):
         return u"<Publishers('{0},{1}')>".format(self.name, self.sort)
 
 
 class Data(Base):
     __tablename__ = 'data'
-    __table_args__ = {'schema':'calibre'}
+    __table_args__ = {'schema': 'calibre'}
 
     id = Column(Integer, primary_key=True)
     book = Column(Integer, ForeignKey('books.id'), nullable=False)
@@ -277,6 +301,10 @@ class Data(Base):
         self.uncompressed_size = uncompressed_size
         self.name = name
 
+    # ToDo: Check
+    def get(self):
+        return self.name
+
     def __repr__(self):
         return u"<Data('{0},{1}{2}{3}')>".format(self.book, self.format, self.uncompressed_size, self.name)
 
@@ -284,14 +312,14 @@ class Data(Base):
 class Books(Base):
     __tablename__ = 'books'
 
-    DEFAULT_PUBDATE = "0101-01-01 00:00:00+00:00"
+    DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0)  # ("0101-01-01 00:00:00+00:00")
 
     id = Column(Integer, primary_key=True, autoincrement=True)
     title = Column(String(collation='NOCASE'), nullable=False, default='Unknown')
     sort = Column(String(collation='NOCASE'))
     author_sort = Column(String(collation='NOCASE'))
     timestamp = Column(TIMESTAMP, default=datetime.utcnow)
-    pubdate = Column(String) # , default=datetime.utcnow)
+    pubdate = Column(TIMESTAMP, default=DEFAULT_PUBDATE)
     series_index = Column(String, nullable=False, default="1.0")
     last_modified = Column(TIMESTAMP, default=datetime.utcnow)
     path = Column(String, default="", nullable=False)
@@ -321,7 +349,8 @@ class Books(Base):
         self.series_index = series_index
         self.last_modified = last_modified
         self.path = path
-        self.has_cover = has_cover
+        self.has_cover = (has_cover != None)
+
 
     def __repr__(self):
         return u"<Books('{0},{1}{2}{3}{4}{5}{6}{7}{8}')>".format(self.title, self.sort, self.author_sort,
@@ -332,6 +361,7 @@ class Books(Base):
     def atom_timestamp(self):
         return (self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '')
 
+
 class Custom_Columns(Base):
     __tablename__ = 'custom_columns'
 
@@ -352,46 +382,67 @@ class Custom_Columns(Base):
         return display_dict
 
 
-class CalibreDB(threading.Thread):
+class AlchemyEncoder(json.JSONEncoder):
+
+    def default(self, obj):
+        if isinstance(obj.__class__, DeclarativeMeta):
+            # an SQLAlchemy class
+            fields = {}
+            for field in [x for x in dir(obj) if not x.startswith('_') and x != 'metadata']:
+                if field == 'books':
+                    continue
+                data = obj.__getattribute__(field)
+                try:
+                    if isinstance(data, str):
+                        data = data.replace("'", "\'")
+                    elif isinstance(data, InstrumentedList):
+                        el = list()
+                        for ele in data:
+                            if ele.get:
+                                el.append(ele.get())
+                            else:
+                                el.append(json.dumps(ele, cls=AlchemyEncoder))
+                        data = ",".join(el)
+                        if data == '[]':
+                            data = ""
+                    else:
+                        json.dumps(data)
+                    fields[field] = data
+                except:
+                    fields[field] = ""
+            # a json-encodable dict
+            return fields
+
+        return json.JSONEncoder.default(self, obj)
+
+
+class CalibreDB():
+    _init = False
+    engine = None
+    config = None
+    session_factory = None
+    # This is a WeakSet so that references here don't keep other CalibreDB
+    # instances alive once they reach the end of their respective scopes
+    instances = WeakSet()
 
     def __init__(self):
-        threading.Thread.__init__(self)
-        self.engine = None
+        """ Initialize a new CalibreDB session
+        """
         self.session = None
-        self.queue = None
-        self.log = None
-        self.config = None
-
-    def add_queue(self,queue):
-        self.queue = queue
-        self.log = logger.create()
-
-    def run(self):
-        while True:
-            i = self.queue.get()
-            if i == 'dummy':
-                self.queue.task_done()
-                break
-            if i['task'] == 'add_format':
-                cur_book = self.session.query(Books).filter(Books.id == i['id']).first()
-                cur_book.data.append(i['format'])
-                try:
-                    # db.session.merge(cur_book)
-                    self.session.commit()
-                except OperationalError as e:
-                    self.session.rollback()
-                    self.log.error("Database error: %s", e)
-                    # self._handleError(_(u"Database error: %(error)s.", error=e))
-                    # return
-            self.queue.task_done()
+        if self._init:
+            self.initSession()
 
+        self.instances.add(self)
 
-    def stop(self):
-        self.queue.put('dummy')
 
-    def setup_db(self, config, app_db_path):
-        self.config = config
-        self.dispose()
+    def initSession(self):
+        self.session = self.session_factory()
+        self.update_title_sort(self.config)
+
+    @classmethod
+    def setup_db(cls, config, app_db_path):
+        cls.config = config
+        cls.dispose()
 
         if not config.config_calibre_dir:
             config.invalidate()
@@ -403,22 +454,21 @@ class CalibreDB(threading.Thread):
             return False
 
         try:
-            self.engine = create_engine('sqlite://',
-                                   echo=False,
-                                   isolation_level="SERIALIZABLE",
-                                   connect_args={'check_same_thread': False},
-                                   poolclass=StaticPool)
-            self.engine.execute("attach database '{}' as calibre;".format(dbpath))
-            self.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
-
-            conn = self.engine.connect()
+            cls.engine = create_engine('sqlite://',
+                                       echo=False,
+                                       isolation_level="SERIALIZABLE",
+                                       connect_args={'check_same_thread': False},
+                                       poolclass=StaticPool)
+            cls.engine.execute("attach database '{}' as calibre;".format(dbpath))
+            cls.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
+
+            conn = cls.engine.connect()
             # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
         except Exception as e:
             config.invalidate(e)
             return False
 
         config.db_configured = True
-        self.update_title_sort(config, conn.connection)
 
         if not cc_classes:
             cc = conn.execute("SELECT id, datatype FROM custom_columns")
@@ -433,12 +483,12 @@ class CalibreDB(threading.Thread):
                                      'book': Column(Integer, ForeignKey('books.id'),
                                                     primary_key=True),
                                      'map_value': Column('value', Integer,
-                                                     ForeignKey('custom_column_' +
-                                                                str(row.id) + '.id'),
-                                                     primary_key=True),
+                                                         ForeignKey('custom_column_' +
+                                                                    str(row.id) + '.id'),
+                                                         primary_key=True),
                                      'extra': Column(Float),
-                                     'asoc' : relationship('custom_column_' + str(row.id), uselist=False),
-                                     'value' : association_proxy('asoc', 'value')
+                                     'asoc': relationship('custom_column_' + str(row.id), uselist=False),
+                                     'value': association_proxy('asoc', 'value')
                                      }
                         books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'),
                                                                  (Base,), dicttable)
@@ -474,7 +524,7 @@ class CalibreDB(threading.Thread):
                             'custom_column_' + str(cc_id[0]),
                             relationship(cc_classes[cc_id[0]],
                                          primaryjoin=(
-                                         Books.id == cc_classes[cc_id[0]].book),
+                                             Books.id == cc_classes[cc_id[0]].book),
                                          backref='books'))
                 elif (cc_id[1] == 'series'):
                     setattr(Books,
@@ -488,17 +538,20 @@ class CalibreDB(threading.Thread):
                                          secondary=books_custom_column_links[cc_id[0]],
                                          backref='books'))
 
-        Session = scoped_session(sessionmaker(autocommit=False,
-                                              autoflush=False,
-                                              bind=self.engine))
-        self.session = Session()
+        cls.session_factory = scoped_session(sessionmaker(autocommit=False,
+                                                          autoflush=True,
+                                                          bind=cls.engine))
+        for inst in cls.instances:
+            inst.initSession()
+
+        cls._init = True
         return True
 
     def get_book(self, book_id):
         return self.session.query(Books).filter(Books.id == book_id).first()
 
     def get_filtered_book(self, book_id, allow_show_archived=False):
-        return self.session.query(Books).filter(Books.id == book_id).\
+        return self.session.query(Books).filter(Books.id == book_id). \
             filter(self.common_filters(allow_show_archived)).first()
 
     def get_book_by_uuid(self, book_uuid):
@@ -545,10 +598,12 @@ class CalibreDB(threading.Thread):
                     pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
 
     # Fill indexpage with all requested data from database
-    def fill_indexpage(self, page, database, db_filter, order, *join):
-        return self.fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join)
+    def fill_indexpage(self, page, pagesize, database, db_filter, order, *join):
+        return self.fill_indexpage_with_archived_books(page, pagesize, database, db_filter, order, False, *join)
 
-    def fill_indexpage_with_archived_books(self, page, database, db_filter, order, allow_show_archived, *join):
+    def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived,
+                                           *join):
+        pagesize = pagesize or self.config.config_books_per_page
         if current_user.show_detail_random():
             randm = self.session.query(Books) \
                 .filter(self.common_filters(allow_show_archived)) \
@@ -556,14 +611,14 @@ class CalibreDB(threading.Thread):
                 .limit(self.config.config_random_books)
         else:
             randm = false()
-        off = int(int(self.config.config_books_per_page) * (page - 1))
+        off = int(int(pagesize) * (page - 1))
         query = self.session.query(database) \
             .join(*join, isouter=True) \
             .filter(db_filter) \
             .filter(self.common_filters(allow_show_archived))
-        pagination = Pagination(page, self.config.config_books_per_page,
+        pagination = Pagination(page, pagesize,
                                 len(query.all()))
-        entries = query.order_by(*order).offset(off).limit(self.config.config_books_per_page).all()
+        entries = query.order_by(*order).offset(off).limit(pagesize).all()
         for book in entries:
             book = self.order_authors(book)
         return entries, randm, pagination
@@ -573,13 +628,16 @@ class CalibreDB(threading.Thread):
         sort_authors = entry.author_sort.split('&')
         authors_ordered = list()
         error = False
+        ids = [a.id for a in entry.authors]
         for auth in sort_authors:
+            results = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).all()
             # ToDo: How to handle not found authorname
-            result = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).first()
-            if not result:
+            if not len(results):
                 error = True
                 break
-            authors_ordered.append(result)
+            for r in results:
+                if r.id in ids:
+                    authors_ordered.append(r)
         if not error:
             entry.authors = authors_ordered
         return entry
@@ -599,24 +657,39 @@ class CalibreDB(threading.Thread):
         for authorterm in authorterms:
             q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
 
-        return self.session.query(Books)\
+        return self.session.query(Books) \
             .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first()
 
     # read search results from calibre-database and return it (function is used for feed and simple search
-    def get_search_results(self, term):
+    def get_search_results(self, term, offset=None, order=None, limit=None):
+        order = order or [Books.sort]
+        pagination = None
         term.strip().lower()
         self.session.connection().connection.connection.create_function("lower", 1, lcase)
         q = list()
         authorterms = re.split("[, ]+", term)
         for authorterm in authorterms:
             q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
-        return self.session.query(Books).filter(self.common_filters(True)).filter(
+        result = self.session.query(Books).filter(self.common_filters(True)).filter(
             or_(Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")),
                 Books.series.any(func.lower(Series.name).ilike("%" + term + "%")),
                 Books.authors.any(and_(*q)),
                 Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")),
                 func.lower(Books.title).ilike("%" + term + "%")
-                )).order_by(Books.sort).all()
+                )).order_by(*order).all()
+        result_count = len(result)
+        if offset != None and limit != None:
+            offset = int(offset)
+            limit_all = offset + int(limit)
+            pagination = Pagination((offset / (int(limit)) + 1), limit, result_count)
+        else:
+            offset = 0
+            limit_all = result_count
+
+        ub.store_ids(result)
+
+
+        return result[offset:limit_all], result_count, pagination,
 
     # Creates for all stored languages a translated speaking name in the array for the UI
     def speaking_language(self, languages=None):
@@ -650,17 +723,23 @@ class CalibreDB(threading.Thread):
         conn = conn or self.session.connection().connection.connection
         conn.create_function("title_sort", 1, _title_sort)
 
-    def dispose(self):
+    @classmethod
+    def dispose(cls):
         # global session
 
-        old_session = self.session
-        self.session = None
-        if old_session:
-            try: old_session.close()
-            except: pass
-            if old_session.bind:
-                try: old_session.bind.dispose()
-                except Exception: pass
+        for inst in cls.instances:
+            old_session = inst.session
+            inst.session = None
+            if old_session:
+                try:
+                    old_session.close()
+                except:
+                    pass
+                if old_session.bind:
+                    try:
+                        old_session.bind.dispose()
+                    except Exception:
+                        pass
 
         for attr in list(Books.__dict__.keys()):
             if attr.startswith("custom_column_"):
@@ -677,10 +756,11 @@ class CalibreDB(threading.Thread):
                     Base.metadata.remove(table)
 
     def reconnect_db(self, config, app_db_path):
-        self.session.close()
+        self.dispose()
         self.engine.dispose()
         self.setup_db(config, app_db_path)
 
+
 def lcase(s):
     try:
         return unidecode.unidecode(s.lower())

cps/editbooks.py

@@ -27,14 +27,17 @@ import json
 from shutil import copyfile
 from uuid import uuid4
 
+from babel import Locale as LC
 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
 from flask_babel import gettext as _
 from flask_login import current_user, login_required
 from sqlalchemy.exc import OperationalError
 
 from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
-from . import config, get_locale, ub, worker, db
+from . import config, get_locale, ub, db
 from . import calibre_db
+from .services.worker import WorkerThread
+from .tasks.upload import TaskUpload
 from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required
 
 
@@ -172,21 +175,42 @@ def modify_identifiers(input_identifiers, db_identifiers, db_session):
             changed = True
     return changed, error
 
+@editbook.route("/ajax/delete/<int:book_id>")
+@login_required
+def delete_book_from_details(book_id):
+    return Response(delete_book(book_id,"", True), mimetype='application/json')
+
 
-@editbook.route("/delete/<int:book_id>/", defaults={'book_format': ""})
-@editbook.route("/delete/<int:book_id>/<string:book_format>/")
+@editbook.route("/delete/<int:book_id>", defaults={'book_format': ""})
+@editbook.route("/delete/<int:book_id>/<string:book_format>")
 @login_required
-def delete_book(book_id, book_format):
+def delete_book_ajax(book_id, book_format):
+    return delete_book(book_id,book_format, False)
+
+def delete_book(book_id, book_format, jsonResponse):
+    warning = {}
     if current_user.role_delete_books():
         book = calibre_db.get_book(book_id)
         if book:
             try:
                 result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper())
                 if not result:
-                    flash(error, category="error")
-                    return redirect(url_for('editbook.edit_book', book_id=book_id))
+                    if jsonResponse:
+                        return json.dumps({"location": url_for("editbook.edit_book"),
+                                           "type": "alert",
+                                           "format": "",
+                                           "error": error}),
+                    else:
+                        flash(error, category="error")
+                        return redirect(url_for('editbook.edit_book', book_id=book_id))
                 if error:
-                    flash(error, category="warning")
+                    if jsonResponse:
+                        warning = {"location": url_for("editbook.edit_book"),
+                                                "type": "warning",
+                                                "format": "",
+                                                "error": error}
+                    else:
+                        flash(error, category="warning")
                 if not book_format:
                     # delete book from Shelfs, Downloads, Read list
                     ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete()
@@ -236,17 +260,29 @@ def delete_book(book_id, book_format):
                         filter(db.Data.format == book_format).delete()
                 calibre_db.session.commit()
             except Exception as e:
-                log.debug(e)
+                log.exception(e)
                 calibre_db.session.rollback()
         else:
             # book not found
             log.error('Book with id "%s" could not be deleted: not found', book_id)
     if book_format:
-        flash(_('Book Format Successfully Deleted'), category="success")
-        return redirect(url_for('editbook.edit_book', book_id=book_id))
+        if jsonResponse:
+            return json.dumps([warning, {"location": url_for("editbook.edit_book", book_id=book_id),
+                                         "type": "success",
+                                         "format": book_format,
+                                         "message": _('Book Format Successfully Deleted')}])
+        else:
+            flash(_('Book Format Successfully Deleted'), category="success")
+            return redirect(url_for('editbook.edit_book', book_id=book_id))
     else:
-        flash(_('Book Successfully Deleted'), category="success")
-        return redirect(url_for('web.index'))
+        if jsonResponse:
+            return json.dumps([warning, {"location": url_for('web.index'),
+                                         "type": "success",
+                                         "format": book_format,
+                                         "message": _('Book Successfully Deleted')}])
+        else:
+            flash(_('Book Successfully Deleted'), category="success")
+            return redirect(url_for('web.index'))
 
 
 def render_edit_book(book_id):
@@ -466,64 +502,64 @@ def edit_cc_data(book_id, book, to_save):
 def upload_single_file(request, book, book_id):
     # Check and handle Uploaded file
     if 'btn-upload-format' in request.files:
-            requested_file = request.files['btn-upload-format']
-            # check for empty request
-            if requested_file.filename != '':
-                if not current_user.role_upload():
-                    abort(403)
-                if '.' in requested_file.filename:
-                    file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
-                    if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD:
-                        flash(_("File extension '%(ext)s' is not allowed to be uploaded to this server", ext=file_ext),
-                              category="error")
-                        return redirect(url_for('web.show_book', book_id=book.id))
-                else:
-                    flash(_('File to be uploaded must have an extension'), category="error")
+        requested_file = request.files['btn-upload-format']
+        # check for empty request
+        if requested_file.filename != '':
+            if not current_user.role_upload():
+                abort(403)
+            if '.' in requested_file.filename:
+                file_ext = requested_file.filename.rsplit('.', 1)[-1].lower()
+                if file_ext not in constants.EXTENSIONS_UPLOAD and '' not in constants.EXTENSIONS_UPLOAD:
+                    flash(_("File extension '%(ext)s' is not allowed to be uploaded to this server", ext=file_ext),
+                          category="error")
                     return redirect(url_for('web.show_book', book_id=book.id))
+            else:
+                flash(_('File to be uploaded must have an extension'), category="error")
+                return redirect(url_for('web.show_book', book_id=book.id))
+
+            file_name = book.path.rsplit('/', 1)[-1]
+            filepath = os.path.normpath(os.path.join(config.config_calibre_dir, book.path))
+            saved_filename = os.path.join(filepath, file_name + '.' + file_ext)
 
-                file_name = book.path.rsplit('/', 1)[-1]
-                filepath = os.path.normpath(os.path.join(config.config_calibre_dir, book.path))
-                saved_filename = os.path.join(filepath, file_name + '.' + file_ext)
-
-                # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
-                if not os.path.exists(filepath):
-                    try:
-                        os.makedirs(filepath)
-                    except OSError:
-                        flash(_(u"Failed to create path %(path)s (Permission denied).", path=filepath), category="error")
-                        return redirect(url_for('web.show_book', book_id=book.id))
+            # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
+            if not os.path.exists(filepath):
                 try:
-                    requested_file.save(saved_filename)
+                    os.makedirs(filepath)
                 except OSError:
-                    flash(_(u"Failed to store file %(file)s.", file=saved_filename), category="error")
+                    flash(_(u"Failed to create path %(path)s (Permission denied).", path=filepath), category="error")
                     return redirect(url_for('web.show_book', book_id=book.id))
+            try:
+                requested_file.save(saved_filename)
+            except OSError:
+                flash(_(u"Failed to store file %(file)s.", file=saved_filename), category="error")
+                return redirect(url_for('web.show_book', book_id=book.id))
 
-                file_size = os.path.getsize(saved_filename)
-                is_format = calibre_db.get_book_format(book_id, file_ext.upper())
+            file_size = os.path.getsize(saved_filename)
+            is_format = calibre_db.get_book_format(book_id, file_ext.upper())
 
-                # Format entry already exists, no need to update the database
-                if is_format:
-                    log.warning('Book format %s already existing', file_ext.upper())
-                else:
-                    try:
-                        db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
-                        calibre_db.session.add(db_format)
-                        calibre_db.session.commit()
-                        calibre_db.update_title_sort(config)
-                    except OperationalError as e:
-                        calibre_db.session.rollback()
-                        log.error('Database error: %s', e)
-                        flash(_(u"Database error: %(error)s.", error=e), category="error")
-                        return redirect(url_for('web.show_book', book_id=book.id))
-
-                # Queue uploader info
-                uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
-                worker.add_upload(current_user.nickname,
-                    "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>")
-
-                return uploader.process(
-                    saved_filename, *os.path.splitext(requested_file.filename),
-                    rarExecutable=config.config_rarfile_location)
+            # Format entry already exists, no need to update the database
+            if is_format:
+                log.warning('Book format %s already existing', file_ext.upper())
+            else:
+                try:
+                    db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
+                    calibre_db.session.add(db_format)
+                    calibre_db.session.commit()
+                    calibre_db.update_title_sort(config)
+                except OperationalError as e:
+                    calibre_db.session.rollback()
+                    log.error('Database error: %s', e)
+                    flash(_(u"Database error: %(error)s.", error=e), category="error")
+                    return redirect(url_for('web.show_book', book_id=book.id))
+
+            # Queue uploader info
+            uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
+            WorkerThread.add(current_user.nickname, TaskUpload(
+                "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>"))
+
+            return uploader.process(
+                saved_filename, *os.path.splitext(requested_file.filename),
+                rarExecutable=config.config_rarfile_location)
 
 
 def upload_cover(request, book):
@@ -569,6 +605,7 @@ def edit_book(book_id):
         merge_metadata(to_save, meta)
         # Update book
         edited_books_id = None
+
         #handle book title
         if book.title != to_save["book_title"].rstrip().strip():
             if to_save["book_title"] == '':
@@ -779,42 +816,17 @@ def upload():
                         if not db_author:
                             db_author = stored_author
                         sort_author = stored_author.sort
-                    sort_authors_list.append(sort_author) # helper.get_sorted_author(sort_author))
+                    sort_authors_list.append(sort_author)
                 sort_authors = ' & '.join(sort_authors_list)
 
                 title_dir = helper.get_valid_filename(title)
                 author_dir = helper.get_valid_filename(db_author.name)
-                filepath = os.path.join(config.config_calibre_dir, author_dir, title_dir)
-                saved_filename = os.path.join(filepath, title_dir + meta.extension.lower())
-
-                # check if file path exists, otherwise create it, copy file to calibre path and delete temp file
-                if not os.path.exists(filepath):
-                    try:
-                        os.makedirs(filepath)
-                    except OSError:
-                        log.error("Failed to create path %s (Permission denied)", filepath)
-                        flash(_(u"Failed to create path %(path)s (Permission denied).", path=filepath), category="error")
-                        return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-                try:
-                    copyfile(meta.file_path, saved_filename)
-                    os.unlink(meta.file_path)
-                except OSError as e:
-                    log.error("Failed to move file %s: %s", saved_filename, e)
-                    flash(_(u"Failed to Move File %(file)s: %(error)s", file=saved_filename, error=e), category="error")
-                    return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
-
-                if meta.cover is None:
-                    has_cover = 0
-                    copyfile(os.path.join(constants.STATIC_DIR, 'generic_cover.jpg'),
-                             os.path.join(filepath, "cover.jpg"))
-                else:
-                    has_cover = 1
 
                 # combine path and normalize path from windows systems
                 path = os.path.join(author_dir, title_dir).replace('\\', '/')
                 # Calibre adds books with utc as timezone
                 db_book = db.Books(title, "", sort_authors, datetime.utcnow(), datetime(101, 1, 1),
-                                   '1', datetime.utcnow(), path, has_cover, db_author, [], "")
+                                   '1', datetime.utcnow(), path, meta.cover, db_author, [], "")
 
                 modif_date |= modify_database_object(input_authors, db_book.authors, db.Authors, calibre_db.session,
                                                      'author')
@@ -832,7 +844,7 @@ def upload():
                 modif_date |= edit_book_series(meta.series, db_book)
 
                 # Add file to book
-                file_size = os.path.getsize(saved_filename)
+                file_size = os.path.getsize(meta.file_path)
                 db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, title_dir)
                 db_book.data.append(db_data)
                 calibre_db.session.add(db_book)
@@ -840,39 +852,44 @@ def upload():
                 # flush content, get db_book.id available
                 calibre_db.session.flush()
 
-                # Comments needs book id therfore only possiblw after flush
+                # Comments needs book id therfore only possible after flush
                 modif_date |= edit_book_comments(Markup(meta.description).unescape(), db_book)
 
                 book_id = db_book.id
                 title = db_book.title
 
-                error = helper.update_dir_stucture(book_id, config.config_calibre_dir, input_authors[0])
+                error = helper.update_dir_structure_file(book_id,
+                                                   config.config_calibre_dir,
+                                                   input_authors[0],
+                                                   meta.file_path,
+                                                   title_dir + meta.extension)
 
                 # move cover to final directory, including book id
-                if has_cover:
-                    new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg")
-                    try:
-                        copyfile(meta.cover, new_coverpath)
+                if meta.cover:
+                    coverfile = meta.cover
+                else:
+                    coverfile = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg')
+                new_coverpath = os.path.join(config.config_calibre_dir, db_book.path, "cover.jpg")
+                try:
+                    copyfile(coverfile, new_coverpath)
+                    if meta.cover:
                         os.unlink(meta.cover)
-                    except OSError as e:
-                        log.error("Failed to move cover file %s: %s", new_coverpath, e)
-                        flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath,
-                                error=e),
-                              category="error")
+                except OSError as e:
+                    log.error("Failed to move cover file %s: %s", new_coverpath, e)
+                    flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath,
+                            error=e),
+                          category="error")
 
                 # save data to database, reread data
                 calibre_db.session.commit()
-                #calibre_db.setup_db(config, ub.app_DB_path)
-                # Reread book. It's important not to filter the result, as it could have language which hide it from
-                # current users view (tags are not stored/extracted from metadata and could also be limited)
-                #book = calibre_db.get_book(book_id)
+
                 if config.config_use_google_drive:
                     gdriveutils.updateGdriveCalibreFromLocal()
                 if error:
                     flash(error, category="error")
                 uploadText=_(u"File %(file)s uploaded", file=title)
-                worker.add_upload(current_user.nickname,
-                    "<a href=\"" + url_for('web.show_book', book_id=book_id) + "\">" + uploadText + "</a>")
+                WorkerThread.add(current_user.nickname, TaskUpload(
+                    "<a href=\"" + url_for('web.show_book', book_id=book_id) + "\">" + uploadText + "</a>"))
 
                 if len(request.files.getlist("btn-upload")) < 2:
                     if current_user.role_edit() or current_user.role_admin():
@@ -910,3 +927,112 @@ def convert_bookformat(book_id):
     else:
         flash(_(u"There was an error converting this book: %(res)s", res=rtn), category="error")
     return redirect(url_for('editbook.edit_book', book_id=book_id))
+
+@editbook.route("/ajax/editbooks/<param>", methods=['POST'])
+@login_required_if_no_ano
+def edit_list_book(param):
+    vals = request.form.to_dict()
+    # calibre_db.update_title_sort(config)
+    #calibre_db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
+    book = calibre_db.get_book(vals['pk'])
+    if param =='series_index':
+        edit_book_series_index(vals['value'], book)
+    elif param =='tags':
+        edit_book_tags(vals['value'], book)
+    elif param =='series':
+        edit_book_series(vals['value'], book)
+    elif param =='publishers':
+        vals['publisher'] = vals['value']
+        edit_book_publisher(vals, book)
+    elif param =='languages':
+        edit_book_languages(vals['value'], book)
+    elif param =='author_sort':
+        book.author_sort = vals['value']
+    elif param =='title':
+        book.title = vals['value']
+        helper.update_dir_stucture(book.id, config.config_calibre_dir)
+    elif param =='sort':
+        book.sort = vals['value']
+    # ToDo: edit books
+    elif param =='authors':
+        input_authors = vals['value'].split('&')
+        input_authors = list(map(lambda it: it.strip().replace(',', '|'), input_authors))
+        modify_database_object(input_authors, book.authors, db.Authors, calibre_db.session, 'author')
+        sort_authors_list = list()
+        for inp in input_authors:
+            stored_author = calibre_db.session.query(db.Authors).filter(db.Authors.name == inp).first()
+            if not stored_author:
+                stored_author = helper.get_sorted_author(inp)
+            else:
+                stored_author = stored_author.sort
+            sort_authors_list.append(helper.get_sorted_author(stored_author))
+        sort_authors = ' & '.join(sort_authors_list)
+        if book.author_sort != sort_authors:
+            book.author_sort = sort_authors
+        helper.update_dir_stucture(book.id, config.config_calibre_dir, input_authors[0])
+    book.last_modified = datetime.utcnow()
+    calibre_db.session.commit()
+    return ""
+
+@editbook.route("/ajax/sort_value/<field>/<int:bookid>")
+@login_required
+def get_sorted_entry(field, bookid):
+    if field == 'title' or field == 'authors':
+        book = calibre_db.get_filtered_book(bookid)
+        if book:
+            if field == 'title':
+                return json.dumps({'sort': book.sort})
+            elif field == 'authors':
+                return json.dumps({'author_sort': book.author_sort})
+    return ""
+
+
+@editbook.route("/ajax/simulatemerge", methods=['POST'])
+@login_required
+def simulate_merge_list_book():
+    vals = request.get_json().get('Merge_books')
+    if vals:
+        to_book = calibre_db.get_book(vals[0]).title
+        vals.pop(0)
+        if to_book:
+            for book_id in vals:
+                from_book = []
+                from_book.append(calibre_db.get_book(book_id).title)
+            return json.dumps({'to': to_book, 'from': from_book})
+    return ""
+
+
+@editbook.route("/ajax/mergebooks", methods=['POST'])
+@login_required
+def merge_list_book():
+    vals = request.get_json().get('Merge_books')
+    to_file = list()
+    if vals:
+        # load all formats from target book
+        to_book = calibre_db.get_book(vals[0])
+        vals.pop(0)
+        if to_book:
+            for file in to_book.data:
+                to_file.append(file.format)
+            to_name = helper.get_valid_filename(to_book.title) + ' - ' + \
+                      helper.get_valid_filename(to_book.authors[0].name)
+            for book_id in vals:
+                from_book = calibre_db.get_book(book_id)
+                if from_book:
+                    for element in from_book.data:
+                        if element.format not in to_file:
+                            # create new data entry with: book_id, book_format, uncompressed_size, name
+                            filepath_new = os.path.normpath(os.path.join(config.config_calibre_dir,
+                                                                         to_book.path,
+                                                                         to_name + "." + element.format.lower()))
+                            filepath_old = os.path.normpath(os.path.join(config.config_calibre_dir,
+                                                                         from_book.path,
+                                                                         element.name + "." + element.format.lower()))
+                            copyfile(filepath_old, filepath_new)
+                            to_book.data.append(db.Data(to_book.id,
+                                                        element.format,
+                                                        element.uncompressed_size,
+                                                        to_name))
+                    delete_book(from_book.id,"", True) # json_resp =
+                    return json.dumps({'success': True})
+    return ""

cps/epub.py

@@ -26,6 +26,7 @@ from .helper import split_authors
 from .constants import BookMeta
 
 
+
 def extractCover(zipFile, coverFile, coverpath, tmp_file_name):
     if coverFile is None:
         return None

cps/helper.py

@@ -32,13 +32,14 @@ from tempfile import gettempdir
 import requests
 from babel.dates import format_datetime
 from babel.units import format_unit
-from flask import send_from_directory, make_response, redirect, abort
+from flask import send_from_directory, make_response, redirect, abort, url_for
 from flask_babel import gettext as _
 from flask_login import current_user
 from sqlalchemy.sql.expression import true, false, and_, text, func
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash
 from . import calibre_db
+from .tasks.convert import TaskConvert
 
 try:
     from urllib.parse import quote
@@ -58,12 +59,12 @@ try:
 except ImportError:
     use_PIL = False
 
-from . import logger, config, get_locale, db, ub, worker
+from . import logger, config, get_locale, db, ub
 from . import gdriveutils as gd
 from .constants import STATIC_DIR as _STATIC_DIR
 from .subproc_wrapper import process_wait
-from .worker import STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS
-from .worker import TASK_EMAIL, TASK_CONVERT, TASK_UPLOAD, TASK_CONVERT_ANY
+from .services.worker import WorkerThread, STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS
+from .tasks.mail import TaskEmail
 
 
 log = logger.create()
@@ -73,46 +74,42 @@ log = logger.create()
 def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None):
     book = calibre_db.get_book(book_id)
     data = calibre_db.get_book_format(book.id, old_book_format)
+    file_path = os.path.join(calibrepath, book.path, data.name)
     if not data:
         error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
         log.error("convert_book_format: %s", error_message)
         return error_message
     if config.config_use_google_drive:
-        df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower())
-        if df:
-            datafile = os.path.join(calibrepath, book.path, data.name + u"." + old_book_format.lower())
-            if not os.path.exists(os.path.join(calibrepath, book.path)):
-                os.makedirs(os.path.join(calibrepath, book.path))
-            df.GetContentFile(datafile)
-        else:
+        if not gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()):
             error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
                               format=old_book_format, fn=data.name + "." + old_book_format.lower())
             return error_message
-    file_path = os.path.join(calibrepath, book.path, data.name)
-    if os.path.exists(file_path + "." + old_book_format.lower()):
-        # read settings and append converter task to queue
-        if kindle_mail:
-            settings = config.get_mail_settings()
-            settings['subject'] = _('Send to Kindle')  # pretranslate Subject for e-mail
-            settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
-            # text = _(u"%(format)s: %(book)s", format=new_book_format, book=book.title)
-        else:
-            settings = dict()
-        txt = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title))
-        settings['old_book_format'] = old_book_format
-        settings['new_book_format'] = new_book_format
-        worker.add_convert(file_path, book.id, user_id, txt, settings, kindle_mail)
-        return None
     else:
-        error_message = _(u"%(format)s not found: %(fn)s",
-                          format=old_book_format, fn=data.name + "." + old_book_format.lower())
-        return error_message
+        if not os.path.exists(file_path + "." + old_book_format.lower()):
+            error_message = _(u"%(format)s not found: %(fn)s",
+                              format=old_book_format, fn=data.name + "." + old_book_format.lower())
+            return error_message
+    # read settings and append converter task to queue
+    if kindle_mail:
+        settings = config.get_mail_settings()
+        settings['subject'] = _('Send to Kindle')  # pretranslate Subject for e-mail
+        settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
+    else:
+        settings = dict()
+    txt = (u"%s -> %s: %s" % (
+           old_book_format,
+           new_book_format,
+           "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + book.title + "</a>"))
+    settings['old_book_format'] = old_book_format
+    settings['new_book_format'] = new_book_format
+    WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, kindle_mail, user_id))
+    return None
 
 
 def send_test_mail(kindle_mail, user_name):
-    worker.add_email(_(u'Calibre-Web test e-mail'), None, None,
-                     config.get_mail_settings(), kindle_mail, user_name,
-                     _(u"Test e-mail"), _(u'This e-mail has been sent via Calibre-Web.'))
+    WorkerThread.add(user_name, TaskEmail(_(u'Calibre-Web test e-mail'), None, None,
+                     config.get_mail_settings(), kindle_mail, _(u"Test e-mail"),
+                               _(u'This e-mail has been sent via Calibre-Web.')))
     return
 
 
@@ -127,9 +124,16 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
     text += "Don't forget to change your password after first login.\r\n"
     text += "Sincerely\r\n\r\n"
     text += "Your Calibre-Web team"
-    worker.add_email(_(u'Get Started with Calibre-Web'), None, None,
-                     config.get_mail_settings(), e_mail, None,
-                     _(u"Registration e-mail for user: %(name)s", name=user_name), text)
+    WorkerThread.add(None, TaskEmail(
+        subject=_(u'Get Started with Calibre-Web'),
+        filepath=None,
+        attachment=None,
+        settings=config.get_mail_settings(),
+        recipient=e_mail,
+        taskMessage=_(u"Registration e-mail for user: %(name)s", name=user_name),
+        text=text
+    ))
+
     return
 
 
@@ -221,9 +225,9 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
     for entry in iter(book.data):
         if entry.format.upper() == book_format.upper():
             converted_file_name = entry.name + '.' + book_format.lower()
-            worker.add_email(_(u"Send to Kindle"), book.path, converted_file_name,
-                             config.get_mail_settings(), kindle_mail, user_id,
-                             _(u"E-mail: %(book)s", book=book.title), _(u'This e-mail has been sent via Calibre-Web.'))
+            WorkerThread.add(user_id, TaskEmail(_(u"Send to Kindle"), book.path, converted_file_name,
+                             config.get_mail_settings(), kindle_mail,
+                             _(u"E-mail: %(book)s", book=book.title), _(u'This e-mail has been sent via Calibre-Web.')))
             return
     return _(u"The requested file could not be read. Maybe wrong permissions?")
 
@@ -343,66 +347,69 @@ def delete_book_file(book, calibrepath, book_format=None):
                                path=book.path)
 
 
-def update_dir_structure_file(book_id, calibrepath, first_author):
+# Moves files in file storage during author/title rename, or from temp dir to file storage
+def update_dir_structure_file(book_id, calibrepath, first_author, orignal_filepath, db_filename):
+    # get book database entry from id, if original path overwrite source with original_filepath
     localbook = calibre_db.get_book(book_id)
-    path = os.path.join(calibrepath, localbook.path)
+    if orignal_filepath:
+        path = orignal_filepath
+    else:
+        path = os.path.join(calibrepath, localbook.path)
 
+    # Create (current) authordir and titledir from database
     authordir = localbook.path.split('/')[0]
+    titledir = localbook.path.split('/')[1]
+
+    # Create new_authordir from parameter or from database
+    # Create new titledir from database and add id
     if first_author:
         new_authordir = get_valid_filename(first_author)
     else:
         new_authordir = get_valid_filename(localbook.authors[0].name)
-
-    titledir = localbook.path.split('/')[1]
     new_titledir = get_valid_filename(localbook.title) + " (" + str(book_id) + ")"
 
-    if titledir != new_titledir:
-        new_title_path = os.path.join(os.path.dirname(path), new_titledir)
+    if titledir != new_titledir or authordir != new_authordir or orignal_filepath:
+        new_path = os.path.join(calibrepath, new_authordir, new_titledir)
+        new_name = get_valid_filename(localbook.title) + ' - ' + get_valid_filename(new_authordir)
         try:
-            if not os.path.exists(new_title_path):
-                os.renames(os.path.normcase(path), os.path.normcase(new_title_path))
-            else:
-                log.info("Copying title: %s into existing: %s", path, new_title_path)
+            if orignal_filepath:
+                os.renames(os.path.normcase(path),
+                           os.path.normcase(os.path.join(new_path, db_filename)))
+                log.debug("Moving title: %s to %s/%s", path, new_path, new_name)
+            # Check new path is not valid path
+            elif not os.path.exists(new_path):
+                # move original path to new path
+                os.renames(os.path.normcase(path), os.path.normcase(new_path))
+                log.debug("Moving title: %s to %s", path, new_path)
+            else: # path is valid copy only files to new location (merge)
+                log.info("Moving title: %s into existing: %s", path, new_path)
+                # Take all files and subfolder from old path (strange command)
                 for dir_name, __, file_list in os.walk(path):
                     for file in file_list:
                         os.renames(os.path.normcase(os.path.join(dir_name, file)),
-                                   os.path.normcase(os.path.join(new_title_path + dir_name[len(path):], file)))
-            path = new_title_path
-            localbook.path = localbook.path.split('/')[0] + '/' + new_titledir
+                                   os.path.normcase(os.path.join(new_path + dir_name[len(path):], file)))
+            # change location in database to new author/title path
+            localbook.path = os.path.join(new_authordir, new_titledir)
         except OSError as ex:
-            log.error("Rename title from: %s to %s: %s", path, new_title_path, ex)
+            log.error("Rename title from: %s to %s: %s", path, new_path, ex)
             log.debug(ex, exc_info=True)
             return _("Rename title from: '%(src)s' to '%(dest)s' failed with error: %(error)s",
-                     src=path, dest=new_title_path, error=str(ex))
-    if authordir != new_authordir:
-        new_author_path = os.path.join(calibrepath, new_authordir, os.path.basename(path))
-        try:
-            os.renames(os.path.normcase(path), os.path.normcase(new_author_path))
-            localbook.path = new_authordir + '/' + localbook.path.split('/')[1]
-        except OSError as ex:
-            log.error("Rename author from: %s to %s: %s", path, new_author_path, ex)
-            log.debug(ex, exc_info=True)
-            return _("Rename author from: '%(src)s' to '%(dest)s' failed with error: %(error)s",
-                     src=path, dest=new_author_path, error=str(ex))
-    # Rename all files from old names to new names
-    if authordir != new_authordir or titledir != new_titledir:
-        new_name = ""
+                     src=path, dest=new_path, error=str(ex))
+
+        # Rename all files from old names to new names
         try:
-            new_name = get_valid_filename(localbook.title) + ' - ' + get_valid_filename(new_authordir)
-            path_name = os.path.join(calibrepath, new_authordir, os.path.basename(path))
             for file_format in localbook.data:
                 os.renames(os.path.normcase(
-                    os.path.join(path_name, file_format.name + '.' + file_format.format.lower())),
-                           os.path.normcase(os.path.join(path_name, new_name + '.' + file_format.format.lower())))
+                    os.path.join(new_path, file_format.name + '.' + file_format.format.lower())),
+                           os.path.normcase(os.path.join(new_path, new_name + '.' + file_format.format.lower())))
                 file_format.name = new_name
         except OSError as ex:
-            log.error("Rename file in path %s to %s: %s", path, new_name, ex)
+            log.error("Rename file in path %s to %s: %s", new_path, new_name, ex)
             log.debug(ex, exc_info=True)
             return _("Rename file in path '%(src)s' to '%(dest)s' failed with error: %(error)s",
-                     src=path, dest=new_name, error=str(ex))
+                     src=new_path, dest=new_name, error=str(ex))
     return False
 
-
 def update_dir_structure_gdrive(book_id, first_author):
     error = False
     book = calibre_db.get_book(book_id)
@@ -505,11 +512,11 @@ def uniq(inpt):
 # ################################# External interface #################################
 
 
-def update_dir_stucture(book_id, calibrepath, first_author=None):
+def update_dir_stucture(book_id, calibrepath, first_author=None, orignal_filepath=None, db_filename=None):
     if config.config_use_google_drive:
         return update_dir_structure_gdrive(book_id, first_author)
     else:
-        return update_dir_structure_file(book_id, calibrepath, first_author)
+        return update_dir_structure_file(book_id, calibrepath, first_author, orignal_filepath, db_filename)
 
 
 def delete_book(book, calibrepath, book_format):
@@ -722,47 +729,30 @@ def format_runtime(runtime):
 # helper function to apply localize status information in tasklist entries
 def render_task_status(tasklist):
     renderedtasklist = list()
-    for task in tasklist:
-        if task['user'] == current_user.nickname or current_user.role_admin():
-            if task['formStarttime']:
-                task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale())
-            # task2['formStarttime'] = ""
-            else:
-                if 'starttime' not in task:
-                    task['starttime'] = ""
-
-            if 'formRuntime' not in task:
-                task['runtime'] = ""
-            else:
-                task['runtime'] = format_runtime(task['formRuntime'])
+    for num, user, added, task in tasklist:
+        if user == current_user.nickname or current_user.role_admin():
+            ret = {}
+            if task.start_time:
+                ret['starttime'] = format_datetime(task.start_time, format='short', locale=get_locale())
+                ret['runtime'] = format_runtime(task.runtime)
 
             # localize the task status
-            if isinstance(task['stat'], int):
-                if task['stat'] == STAT_WAITING:
-                    task['status'] = _(u'Waiting')
-                elif task['stat'] == STAT_FAIL:
-                    task['status'] = _(u'Failed')
-                elif task['stat'] == STAT_STARTED:
-                    task['status'] = _(u'Started')
-                elif task['stat'] == STAT_FINISH_SUCCESS:
-                    task['status'] = _(u'Finished')
-                else:
-                    task['status'] = _(u'Unknown Status')
-
-            # localize the task type
-            if isinstance(task['taskType'], int):
-                if task['taskType'] == TASK_EMAIL:
-                    task['taskMessage'] = _(u'E-mail: ') + task['taskMess']
-                elif task['taskType'] == TASK_CONVERT:
-                    task['taskMessage'] = _(u'Convert: ') + task['taskMess']
-                elif task['taskType'] == TASK_UPLOAD:
-                    task['taskMessage'] = _(u'Upload: ') + task['taskMess']
-                elif task['taskType'] == TASK_CONVERT_ANY:
-                    task['taskMessage'] = _(u'Convert: ') + task['taskMess']
+            if isinstance(task.stat, int):
+                if task.stat == STAT_WAITING:
+                    ret['status'] = _(u'Waiting')
+                elif task.stat == STAT_FAIL:
+                    ret['status'] = _(u'Failed')
+                elif task.stat == STAT_STARTED:
+                    ret['status'] = _(u'Started')
+                elif task.stat == STAT_FINISH_SUCCESS:
+                    ret['status'] = _(u'Finished')
                 else:
-                    task['taskMessage'] = _(u'Unknown Task: ') + task['taskMess']
+                    ret['status'] = _(u'Unknown Status')
 
-            renderedtasklist.append(task)
+            ret['taskMessage'] = "{}: {}".format(_(task.name), task.message)
+            ret['progress'] = "{} %".format(int(task.progress * 100))
+            ret['user'] = user
+            renderedtasklist.append(ret)
 
     return renderedtasklist
 

cps/jinjia.py

@@ -44,6 +44,8 @@ log = logger.create()
 def url_for_other_page(page):
     args = request.view_args.copy()
     args['page'] = page
+    for get, val in request.args.items():
+        args[get] = val
     return url_for(request.endpoint, **args)
 
 
@@ -76,22 +78,18 @@ def mimetype_filter(val):
 @jinjia.app_template_filter('formatdate')
 def formatdate_filter(val):
     try:
-        conformed_timestamp = re.sub(r"[:]|([-](?!((\d{2}[:]\d{2})|(\d{4}))$))", '', val)
-        formatdate = datetime.datetime.strptime(conformed_timestamp[:15], "%Y%m%d %H%M%S")
-        return format_date(formatdate, format='medium', locale=get_locale())
+        return format_date(val, format='medium', locale=get_locale())
     except AttributeError as e:
         log.error('Babel error: %s, Current user locale: %s, Current User: %s', e,
                   current_user.locale,
                   current_user.nickname
                   )
-        return formatdate
+        return val
 
 
 @jinjia.app_template_filter('formatdateinput')
 def format_date_input(val):
-    conformed_timestamp = re.sub(r"[:]|([-](?!((\d{2}[:]\d{2})|(\d{4}))$))", '', val)
-    date_obj = datetime.datetime.strptime(conformed_timestamp[:15], "%Y%m%d %H%M%S")
-    input_date = date_obj.isoformat().split('T', 1)[0]  # Hack to support dates <1900
+    input_date = val.isoformat().split('T', 1)[0]  # Hack to support dates <1900
     return '' if input_date == "0101-01-01" else input_date
 
 

cps/opds.py

@@ -100,7 +100,7 @@ def feed_normal_search():
 @requires_basic_auth_if_no_ano
 def feed_new():
     off = request.args.get("offset") or 0
-    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
                                                         db.Books, True, [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
@@ -118,7 +118,7 @@ def feed_discover():
 @requires_basic_auth_if_no_ano
 def feed_best_rated():
     off = request.args.get("offset") or 0
-    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
                                                         db.Books, db.Books.ratings.any(db.Ratings.rating > 9),
                                                         [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
@@ -164,7 +164,7 @@ def feed_authorindex():
 @requires_basic_auth_if_no_ano
 def feed_author(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
                                                         db.Books,
                                                         db.Books.authors.any(db.Authors.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -190,7 +190,7 @@ def feed_publisherindex():
 @requires_basic_auth_if_no_ano
 def feed_publisher(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
                                                         db.Books,
                                                         db.Books.publishers.any(db.Publishers.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -218,7 +218,7 @@ def feed_categoryindex():
 @requires_basic_auth_if_no_ano
 def feed_category(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
                                                         db.Books,
                                                         db.Books.tags.any(db.Tags.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -245,7 +245,7 @@ def feed_seriesindex():
 @requires_basic_auth_if_no_ano
 def feed_series(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
                                                         db.Books,
                                                         db.Books.series.any(db.Series.id == book_id),
                                                         [db.Books.series_index])
@@ -276,7 +276,7 @@ def feed_ratingindex():
 @requires_basic_auth_if_no_ano
 def feed_ratings(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
                                                         db.Books,
                                                         db.Books.ratings.any(db.Ratings.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -304,7 +304,7 @@ def feed_formatindex():
 @requires_basic_auth_if_no_ano
 def feed_format(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
                                                         db.Books,
                                                         db.Books.data.any(db.Data.format == book_id.upper()),
                                                         [db.Books.timestamp.desc()])
@@ -338,7 +338,7 @@ def feed_languagesindex():
 @requires_basic_auth_if_no_ano
 def feed_languages(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0,
                                                         db.Books,
                                                         db.Books.languages.any(db.Languages.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -408,7 +408,7 @@ def get_metadata_calibre_companion(uuid, library):
 
 def feed_search(term):
     if term:
-        entries = calibre_db.get_search_results(term)
+        entries, __ = calibre_db.get_search_results(term)
         entriescount = len(entries) if len(entries) > 0 else 1
         pagination = Pagination(1, entriescount, entriescount)
         return render_xml_template('feed.xml', searchterm=term, entries=entries, pagination=pagination)

cps/server.py

@@ -212,9 +212,6 @@ class WebServer(object):
     def stop(self, restart=False):
         from . import updater_thread
         updater_thread.stop()
-        from . import calibre_db
-        calibre_db.stop()
-
 
         log.info("webserver stop (restart=%s)", restart)
         self.restart = restart

cps/services/worker.py

@@ -0,0 +1,220 @@
+
+from __future__ import division, print_function, unicode_literals
+import threading
+import abc
+import uuid
+import time
+
+try:
+    import queue
+except ImportError:
+    import Queue as queue
+from datetime import datetime
+from collections import namedtuple
+
+from cps import logger
+
+log = logger.create()
+
+# task 'status' consts
+STAT_WAITING = 0
+STAT_FAIL = 1
+STAT_STARTED = 2
+STAT_FINISH_SUCCESS = 3
+
+# Only retain this many tasks in dequeued list
+TASK_CLEANUP_TRIGGER = 20
+
+QueuedTask = namedtuple('QueuedTask', 'num, user, added, task')
+
+
+def _get_main_thread():
+    for t in threading.enumerate():
+        if t.__class__.__name__ == '_MainThread':
+            return t
+    raise Exception("main thread not found?!")
+
+
+
+class ImprovedQueue(queue.Queue):
+    def to_list(self):
+        """
+        Returns a copy of all items in the queue without removing them.
+        """
+
+        with self.mutex:
+            return list(self.queue)
+
+#Class for all worker tasks in the background
+class WorkerThread(threading.Thread):
+    _instance = None
+
+    @classmethod
+    def getInstance(cls):
+        if cls._instance is None:
+            cls._instance = WorkerThread()
+        return cls._instance
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+
+        self.dequeued = list()
+
+        self.doLock = threading.Lock()
+        self.queue = ImprovedQueue()
+        self.num = 0
+        self.start()
+
+    @classmethod
+    def add(cls, user, task):
+        ins = cls.getInstance()
+        ins.num += 1
+        ins.queue.put(QueuedTask(
+            num=ins.num,
+            user=user,
+            added=datetime.now(),
+            task=task,
+        ))
+
+    @property
+    def tasks(self):
+        with self.doLock:
+            tasks = self.queue.to_list() + self.dequeued
+            return sorted(tasks, key=lambda x: x.num)
+
+    def cleanup_tasks(self):
+        with self.doLock:
+            dead = []
+            alive = []
+            for x in self.dequeued:
+                (dead if x.task.dead else alive).append(x)
+
+            # if the ones that we need to keep are within the trigger, do nothing else
+            delta = len(self.dequeued) - len(dead)
+            if delta > TASK_CLEANUP_TRIGGER:
+                ret = alive
+            else:
+                # otherwise, lop off the oldest dead tasks until we hit the target trigger
+                ret = sorted(dead, key=lambda x: x.task.end_time)[-TASK_CLEANUP_TRIGGER:] + alive
+
+            self.dequeued = sorted(ret, key=lambda x: x.num)
+
+    # Main thread loop starting the different tasks
+    def run(self):
+        main_thread = _get_main_thread()
+        while main_thread.is_alive():
+            try:
+                # this blocks until something is available. This can cause issues when the main thread dies - this
+                # thread will remain alive. We implement a timeout to unblock every second which allows us to check if
+                # the main thread is still alive.
+                # We don't use a daemon here because we don't want the tasks to just be abruptly halted, leading to
+                # possible file / database corruption
+                item = self.queue.get(timeout=1)
+            except queue.Empty as ex:
+                time.sleep(1)
+                continue
+
+            with self.doLock:
+                # add to list so that in-progress tasks show up
+                self.dequeued.append(item)
+
+            # once we hit our trigger, start cleaning up dead tasks
+            if len(self.dequeued) > TASK_CLEANUP_TRIGGER:
+                self.cleanup_tasks()
+
+            # sometimes tasks (like Upload) don't actually have work to do and are created as already finished
+            if item.task.stat is STAT_WAITING:
+                # CalibreTask.start() should wrap all exceptions in it's own error handling
+                item.task.start(self)
+
+            self.queue.task_done()
+
+
+class CalibreTask:
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, message):
+        self._progress = 0
+        self.stat = STAT_WAITING
+        self.error = None
+        self.start_time = None
+        self.end_time = None
+        self.message = message
+        self.id = uuid.uuid4()
+
+    @abc.abstractmethod
+    def run(self, worker_thread):
+        """Provides the caller some human-readable name for this class"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def name(self):
+        """Provides the caller some human-readable name for this class"""
+        raise NotImplementedError
+
+    def start(self, *args):
+        self.start_time = datetime.now()
+        self.stat = STAT_STARTED
+
+        # catch any unhandled exceptions in a task and automatically fail it
+        try:
+            self.run(*args)
+        except Exception as e:
+            self._handleError(str(e))
+            log.exception(e)
+
+        self.end_time = datetime.now()
+
+    @property
+    def stat(self):
+        return self._stat
+
+    @stat.setter
+    def stat(self, x):
+        self._stat = x
+
+    @property
+    def progress(self):
+        return self._progress
+
+    @progress.setter
+    def progress(self, x):
+        if not 0 <= x <= 1:
+            raise ValueError("Task progress should within [0, 1] range")
+        self._progress = x
+
+    @property
+    def error(self):
+        return self._error
+
+    @error.setter
+    def error(self, x):
+        self._error = x
+
+    @property
+    def runtime(self):
+        return (self.end_time or datetime.now()) - self.start_time
+
+    @property
+    def dead(self):
+        """Determines whether or not this task can be garbage collected
+
+        We have a separate dictating this because there may be certain tasks that want to override this
+        """
+        # By default, we're good to clean a task if it's "Done"
+        return self.stat in (STAT_FINISH_SUCCESS, STAT_FAIL)
+
+    @progress.setter
+    def progress(self, x):
+        # todo: throw error if outside of [0,1]
+        self._progress = x
+
+    def _handleError(self, error_message):
+        log.exception(error_message)
+        self.stat = STAT_FAIL
+        self.progress = 1
+        self.error = error_message
+
+    def _handleSuccess(self):
+        self.stat = STAT_FINISH_SUCCESS
+        self.progress = 1

cps/shelf.py

@@ -29,7 +29,7 @@ from flask_login import login_required, current_user
 from sqlalchemy.sql.expression import func
 from sqlalchemy.exc import OperationalError, InvalidRequestError
 
-from . import logger, ub, searched_ids, calibre_db
+from . import logger, ub, calibre_db
 from .web import login_required_if_no_ano, render_title_template
 
 
@@ -124,18 +124,18 @@ def search_to_shelf(shelf_id):
         flash(_(u"You are not allowed to add a book to the the shelf: %(name)s", name=shelf.name), category="error")
         return redirect(url_for('web.index'))
 
-    if current_user.id in searched_ids and searched_ids[current_user.id]:
+    if current_user.id in ub.searched_ids and ub.searched_ids[current_user.id]:
         books_for_shelf = list()
         books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id).all()
         if books_in_shelf:
             book_ids = list()
             for book_id in books_in_shelf:
                 book_ids.append(book_id.book_id)
-            for searchid in searched_ids[current_user.id]:
+            for searchid in ub.searched_ids[current_user.id]:
                 if searchid not in book_ids:
                     books_for_shelf.append(searchid)
         else:
-            books_for_shelf = searched_ids[current_user.id]
+            books_for_shelf = ub.searched_ids[current_user.id]
 
         if not books_for_shelf:
             log.error("Books are already part of %s", shelf)

cps/static/css/caliBlur.min.css

@@ -585,7 +585,7 @@ div.btn-group[role=group][aria-label="Download, send to Kindle, reading"] > .dow
     border-left: 2px solid rgba(0, 0, 0, .15)
 }
 
-div[aria-label="Edit/Delete book"] > .btn-warning {
+div[aria-label="Edit/Delete book"] > .btn {
     width: 50px;
     height: 60px;
     margin: 0;
@@ -600,7 +600,7 @@ div[aria-label="Edit/Delete book"] > .btn-warning {
     color: transparent
 }
 
-div[aria-label="Edit/Delete book"] > .btn-warning > span {
+div[aria-label="Edit/Delete book"] > .btn > span {
     visibility: visible;
     position: relative;
     display: inline-block;
@@ -616,7 +616,7 @@ div[aria-label="Edit/Delete book"] > .btn-warning > span {
     margin: auto
 }
 
-div[aria-label="Edit/Delete book"] > .btn-warning > span:before {
+div[aria-label="Edit/Delete book"] > .btn > span:before {
     content: "\EA5d";
     font-family: plex-icons;
     font-size: 20px;
@@ -625,7 +625,7 @@ div[aria-label="Edit/Delete book"] > .btn-warning > span:before {
     height: 60px
 }
 
-div[aria-label="Edit/Delete book"] > .btn-warning > span:hover {
+div[aria-label="Edit/Delete book"] > .btn > span:hover {
     color: #fff
 }
 
@@ -1939,7 +1939,9 @@ body > div.container-fluid > div > div.col-sm-10 > div.discover > form > .btn.bt
     z-index: 99999
 }
 
-.pagination:after, body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.next, body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.previous {
+body > div.container-fluid > div > div.col-sm-10 > div.pagination .page-next > a,
+body > div.container-fluid > div > div.col-sm-10 > div.pagination .page-previous > a
+{
     top: 0;
     font-family: plex-icons-new;
     font-weight: 100;
@@ -1947,7 +1949,8 @@ body > div.container-fluid > div > div.col-sm-10 > div.discover > form > .btn.bt
     line-height: 60px;
     height: 60px;
     font-style: normal;
-    -moz-osx-font-smoothing: grayscale
+    -moz-osx-font-smoothing: grayscale;
+    overflow: hidden;
 }
 
 .pagination > a {
@@ -1967,68 +1970,46 @@ body > div.container-fluid > div > div.col-sm-10 > div.discover > form > .btn.bt
     color: #fff !important
 }
 
-body > div.container-fluid > div > div.col-sm-10 > div.pagination > a, body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.previous + a, body > div.container-fluid > div > div.col-sm-10 > div.pagination > a[href*=page] {
+body > div.container-fluid > div > div.col-sm-10 > div.pagination > .page-item:not(.page-next):not(.page-previous)
+{
     display: none
 }
 
-body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.next, body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.previous {
+body > div.container-fluid > div > div.col-sm-10 > div.pagination > .page-next > a,
+body > div.container-fluid > div > div.col-sm-10 > div.pagination > .page-previous > a {
     color: transparent;
+    background-color:transparent;
     margin-left: 0;
     width: 65px;
     padding: 0;
     font-size: 15px;
-    position: absolute;
-    display: block !important
-}
-
-body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.next {
-    right: 0
-}
-
-body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.previous {
-    right: 65px
+    display: block !important;
+    border: none;
 }
 
-body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.next:before {
-    content: "\EA32";
+body > div.container-fluid > div > div.col-sm-10 > div.pagination > .page-next > a:before,
+body > div.container-fluid > div > div.col-sm-10 > div.pagination > .page-previous > a:before {
     visibility: visible;
     color: hsla(0, 0%, 100%, .35);
     height: 60px;
     line-height: 60px;
     border-left: 2px solid transparent;
     font-size: 20px;
-    padding: 20px 0 20px 20px;
-    margin-right: -27px
-}
-
-body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.previous:before {
-    content: "\EA33";
-    visibility: visible;
-    color: hsla(0, 0%, 100%, .65);
-    height: 60px;
-    line-height: 60px;
-    font-size: 20px;
-    padding: 20px 25px
+    padding: 20px 25px;
+    margin-right: -27px;
 }
 
-body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.next:hover:before, body > div.container-fluid > div > div.col-sm-10 > div.pagination > a.previous:hover:before {
-    color: #fff
+body > div.container-fluid > div > div.col-sm-10 > div.pagination > .page-next > a:before {
+    content: "\EA32";
 }
 
-.pagination > strong {
-    display: none
+body > div.container-fluid > div > div.col-sm-10 > div.pagination > .page-previous > a:before {
+    content: "\EA33";
 }
 
-.pagination:after {
-    content: "\EA32";
-    position: relative;
-    right: 0;
-    display: inline-block;
-    color: hsla(0, 0%, 100%, .55);
-    font-size: 20px;
-    padding: 0 23px;
-    margin-left: 20px;
-    z-index: -1
+body > div.container-fluid > div > div.col-sm-10 > div.pagination > .page-next > a:hover:before,
+body > div.container-fluid > div > div.col-sm-10 > div.pagination > .page-previous > a:hover:before {
+    color: #fff
 }
 
 .pagination > .ellipsis, .pagination > a:nth-last-of-type(2) {

cps/static/css/caliBlur_override.css

@@ -5,7 +5,7 @@ body.serieslist.grid-view div.container-fluid>div>div.col-sm-10:before{
 .cover .badge{
     position: absolute;
     top: 0;
-    right: 0;
+    left: 0;
     background-color: #cc7b19;
     border-radius: 0;
     padding: 0 8px;

cps/static/css/style.css

@@ -51,7 +51,22 @@ body h2 {
   color:#444;
 }
 
-a { color: #45b29d; }
+a, .danger,.book-remove, .editable-empty, .editable-empty:hover { color: #45b29d; }
+
+.book-remove:hover { color: #23527c; }
+
+.btn-default a { color: #444; }
+
+.btn-default a:hover {
+    color: #45b29d;
+    text-decoration: None;
+}
+
+.btn-default:hover {
+    color: #45b29d;
+}
+
+.editable-click, a.editable-click, a.editable-click:hover  { border-bottom: None; }
 
 .navigation .nav-head {
   text-transform: uppercase;
@@ -63,6 +78,7 @@ a { color: #45b29d; }
   border-top: 1px solid #ccc;
   padding-top: 20px;
 }
+
 .navigation li a {
   color: #444;
   text-decoration: none;

cps/static/js/archive/archive.js

@@ -411,6 +411,19 @@ bitjs.archive = bitjs.archive || {};
         return "unrar.js";
     };
 
+    /**
+     * Unrarrer5
+     * @extends {bitjs.archive.Unarchiver}
+     * @constructor
+     */
+    bitjs.archive.Unrarrer5 = function(arrayBuffer, optPathToBitJS) {
+        bitjs.base(this, arrayBuffer, optPathToBitJS);
+    };
+    bitjs.inherits(bitjs.archive.Unrarrer5, bitjs.archive.Unarchiver);
+    bitjs.archive.Unrarrer5.prototype.getScriptFileName = function() {
+        return "unrar5.js";
+    };
+
     /**
      * Untarrer
      * @extends {bitjs.archive.Unarchiver}

cps/static/js/archive/unrar.js

@@ -14,10 +14,10 @@
 /* global VM_FIXEDGLOBALSIZE, VM_GLOBALMEMSIZE, MAXWINMASK, VM_GLOBALMEMADDR, MAXWINSIZE */
 
 // This file expects to be invoked as a Worker (see onmessage below).
-importScripts("../io/bitstream.js");
+/*importScripts("../io/bitstream.js");
 importScripts("../io/bytebuffer.js");
 importScripts("archive.js");
-importScripts("rarvm.js");
+importScripts("rarvm.js");*/
 
 // Progress variables.
 var currentFilename = "";
@@ -29,19 +29,21 @@ var totalFilesInArchive = 0;
 
 // Helper functions.
 var info = function(str) {
-    postMessage(new bitjs.archive.UnarchiveInfoEvent(str));
+    console.log(str);
+    // postMessage(new bitjs.archive.UnarchiveInfoEvent(str));
 };
 var err = function(str) {
-    postMessage(new bitjs.archive.UnarchiveErrorEvent(str));
+    console.log(str);
+    // postMessage(new bitjs.archive.UnarchiveErrorEvent(str));
 };
 var postProgress = function() {
-    postMessage(new bitjs.archive.UnarchiveProgressEvent(
+    /*postMessage(new bitjs.archive.UnarchiveProgressEvent(
         currentFilename,
         currentFileNumber,
         currentBytesUnarchivedInFile,
         currentBytesUnarchived,
         totalUncompressedBytesInArchive,
-        totalFilesInArchive));
+        totalFilesInArchive));*/
 };
 
 // shows a byte value as its hex representation
@@ -1298,7 +1300,7 @@ var unrar = function(arrayBuffer) {
     totalUncompressedBytesInArchive = 0;
     totalFilesInArchive = 0;
 
-    postMessage(new bitjs.archive.UnarchiveStartEvent());
+    //postMessage(new bitjs.archive.UnarchiveStartEvent());
     var bstream = new bitjs.io.BitStream(arrayBuffer, false /* rtl */);
 
     var header = new RarVolumeHeader(bstream);
@@ -1348,7 +1350,7 @@ var unrar = function(arrayBuffer) {
                 localfile.unrar();
 
                 if (localfile.isValid) {
-                    postMessage(new bitjs.archive.UnarchiveExtractEvent(localfile));
+                    // postMessage(new bitjs.archive.UnarchiveExtractEvent(localfile));
                     postProgress();
                 }
             }
@@ -1358,7 +1360,7 @@ var unrar = function(arrayBuffer) {
     } else {
         err("Invalid RAR file");
     }
-    postMessage(new bitjs.archive.UnarchiveFinishEvent());
+    // postMessage(new bitjs.archive.UnarchiveFinishEvent());
 };
 
 // event.data.file has the ArrayBuffer.

cps/static/js/filter_grid.js

@@ -24,6 +24,14 @@ var $list = $("#list").isotope({
 });
 
 $("#desc").click(function() {
+    var page = $(this).data("id");
+    $.ajax({
+        method:"post",
+        contentType: "application/json; charset=utf-8",
+        dataType: "json",
+        url: window.location.pathname + "/../../ajax/view",
+        data: "{\"" + page + "\": {\"dir\": \"desc\"}}",
+    });
     $list.isotope({
         sortBy: "name",
         sortAscending: true
@@ -32,6 +40,14 @@ $("#desc").click(function() {
 });
 
 $("#asc").click(function() {
+    var page = $(this).data("id");
+    $.ajax({
+        method:"post",
+        contentType: "application/json; charset=utf-8",
+        dataType: "json",
+        url: window.location.pathname + "/../../ajax/view",
+        data: "{\"" + page + "\": {\"dir\": \"asc\"}}",
+    });
     $list.isotope({
         sortBy: "name",
         sortAscending: false

cps/static/js/filter_list.js

@@ -19,6 +19,17 @@ var direction = 0;  // Descending order
 var sort = 0;       // Show sorted entries
 
 $("#sort_name").click(function() {
+    var class_name = $("h1").attr('Class') + "_sort_name";
+    var obj = {};
+    obj[class_name] = sort;
+    /*$.ajax({
+        method:"post",
+        contentType: "application/json; charset=utf-8",
+        dataType: "json",
+        url: window.location.pathname + "/../../ajax/view",
+        data: JSON.stringify({obj}),
+    });*/
+
     var count = 0;
     var index = 0;
     var store;
@@ -40,9 +51,7 @@ $("#sort_name").click(function() {
             count++;
         }
     });
-    /*listItems.sort(function(a,b){
-        return $(a).children()[1].innerText.localeCompare($(b).children()[1].innerText)
-    });*/
+
     // Find count of middle element
     if (count > 20) {
         var middle = parseInt(count / 2, 10) + (count % 2);
@@ -66,6 +75,14 @@ $("#desc").click(function() {
     if (direction === 0) {
         return;
     }
+    var page = $(this).data("id");
+    $.ajax({
+        method:"post",
+        contentType: "application/json; charset=utf-8",
+        dataType: "json",
+        url: window.location.pathname + "/../../ajax/view",
+        data: "{\"" + page + "\": {\"dir\": \"desc\"}}",
+    });
     var index = 0;
     var list = $("#list");
     var second = $("#second");
@@ -102,9 +119,18 @@ $("#desc").click(function() {
 
 
 $("#asc").click(function() {
+
     if (direction === 1) {
         return;
     }
+    var page = $(this).data("id");
+    $.ajax({
+        method:"post",
+        contentType: "application/json; charset=utf-8",
+        dataType: "json",
+        url: window.location.pathname + "/../../ajax/view",
+        data: "{\"" + page + "\": {\"dir\": \"asc\"}}",
+    });
     var index = 0;
     var list = $("#list");
     var second = $("#second");
@@ -131,7 +157,6 @@ $("#asc").click(function() {
         });
 
         // middle = parseInt(elementLength / 2) + (elementLength % 2);
-
         list.append(reversed.slice(0, index));
         second.append(reversed.slice(index, elementLength));
     } else {

cps/static/js/kthoom.js

@@ -162,10 +162,15 @@ function initProgressClick() {
 function loadFromArrayBuffer(ab) {
     var start = (new Date).getTime();
     var h = new Uint8Array(ab, 0, 10);
+    unrar5(ab);
     var pathToBitJS = "../../static/js/archive/";
     var lastCompletion = 0;
-    if (h[0] === 0x52 && h[1] === 0x61 && h[2] === 0x72 && h[3] === 0x21) { //Rar!
-        unarchiver = new bitjs.archive.Unrarrer(ab, pathToBitJS);
+    /*if (h[0] === 0x52 && h[1] === 0x61 && h[2] === 0x72 && h[3] === 0x21) { //Rar!
+        if (h[7] === 0x01) {
+            unarchiver = new bitjs.archive.Unrarrer(ab, pathToBitJS);
+        } else {
+            unarchiver = new bitjs.archive.Unrarrer5(ab, pathToBitJS);
+        }
     } else if (h[0] === 80 && h[1] === 75) { //PK (Zip)
         unarchiver = new bitjs.archive.Unzipper(ab, pathToBitJS);
     } else if (h[0] === 255 && h[1] === 216) { // JPEG
@@ -229,7 +234,7 @@ function loadFromArrayBuffer(ab) {
         unarchiver.start();
     } else {
         alert("Some error");
-    }
+    }*/
 }
 
 function scrollTocToActive() {

cps/static/js/main.js

@@ -58,6 +58,60 @@ $(document).on("change", "select[data-controlall]", function() {
     }
 });
 
+$("#delete_confirm").click(function() {
+    //get data-id attribute of the clicked element
+    var pathname = document.getElementsByTagName("script"), src = pathname[pathname.length - 1].src;
+    var path = src.substring(0, src.lastIndexOf("/"));
+    var deleteId = $(this).data("delete-id");
+    var bookFormat = $(this).data("delete-format");
+    if (bookFormat) {
+        window.location.href = path + "/../../delete/" + deleteId + "/" + bookFormat;
+    } else {
+        if ($(this).data("delete-format")) {
+            path = path + "/../../ajax/delete/" + deleteId;
+            $.ajax({
+                method:"get",
+                url: path,
+                timeout: 900,
+                success:function(data) {
+                    data.forEach(function(item) {
+                        if (!jQuery.isEmptyObject(item)) {
+                            if (item.format != "") {
+                                $("button[data-delete-format='"+item.format+"']").addClass('hidden');
+                            }
+                            $( ".navbar" ).after( '<div class="row-fluid text-center" style="margin-top: -20px;">' +
+                                '<div id="flash_'+item.type+'" class="alert alert-'+item.type+'">'+item.message+'</div>' +
+                                '</div>');
+
+                        }
+                    });
+                }
+            });
+        } else {
+            window.location.href = path + "/../../delete/" + deleteId;
+
+        }
+    }
+
+});
+
+//triggered when modal is about to be shown
+$("#deleteModal").on("show.bs.modal", function(e) {
+    //get data-id attribute of the clicked element and store in button
+    var bookId = $(e.relatedTarget).data("delete-id");
+    var bookfomat = $(e.relatedTarget).data("delete-format");
+    if (bookfomat) {
+        $("#book_format").removeClass('hidden');
+        $("#book_complete").addClass('hidden');
+    } else {
+        $("#book_complete").removeClass('hidden');
+        $("#book_format").addClass('hidden');
+    }
+    $(e.currentTarget).find("#delete_confirm").data("delete-id", bookId);
+    $(e.currentTarget).find("#delete_confirm").data("delete-format", bookfomat);
+});
+
+
 
 $(function() {
     var updateTimerID;
@@ -324,16 +378,19 @@ $(function() {
     });
 
     $(".update-view").click(function(e) {
-        var target = $(this).data("target");
         var view = $(this).data("view");
 
         e.preventDefault();
         e.stopPropagation();
-        var data = {};
-        data[target] = view;
-        console.debug("Updating view data: ", data);
-        $.post( "/ajax/view", data).done(function( ) {
-            location.reload();
+        $.ajax({
+            method:"post",
+            contentType: "application/json; charset=utf-8",
+            dataType: "json",
+            url: window.location.pathname + "/../../ajax/view",
+            data: "{\"series\": {\"series_view\": \""+ view +"\"}}",
+            success: function success() {
+                location.reload();
+            }
         });
     });
 });

cps/static/js/table.js

@@ -1,5 +1,5 @@
 /* This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
- *    Copyright (C) 2018 OzzieIsaacs
+ *    Copyright (C) 2020 OzzieIsaacs
  *
  *  This program is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -15,10 +15,158 @@
  *  along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-/* exported TableActions, RestrictionActions*/
+/* exported TableActions, RestrictionActions, EbookActions, responseHandler */
+
+var selections = [];
 
 $(function() {
 
+    $("#books-table").on("check.bs.table check-all.bs.table uncheck.bs.table uncheck-all.bs.table",
+        function (e, rowsAfter, rowsBefore) {
+            var rows = rowsAfter;
+
+            if (e.type === "uncheck-all") {
+                rows = rowsBefore;
+            }
+
+            var ids = $.map(!$.isArray(rows) ? [rows] : rows, function (row) {
+                return row.id;
+            });
+
+            var func = $.inArray(e.type, ["check", "check-all"]) > -1 ? "union" : "difference";
+            selections = window._[func](selections, ids);
+            if (selections.length >= 2) {
+                $("#merge_books").removeClass("disabled");
+                $("#merge_books").attr("aria-disabled", false);
+            } else {
+                $("#merge_books").addClass("disabled");
+                $("#merge_books").attr("aria-disabled", true);
+            }
+            if (selections.length < 1) {
+                $("#delete_selection").addClass("disabled");
+                $("#delete_selection").attr("aria-disabled", true);
+            }
+            else{
+                $("#delete_selection").removeClass("disabled");
+                $("#delete_selection").attr("aria-disabled", false);
+            }
+        });
+    $("#delete_selection").click(function() {
+        $("#books-table").bootstrapTable('uncheckAll');
+    });
+
+    $("#merge_confirm").click(function() {
+        $.ajax({
+            method:"post",
+            contentType: "application/json; charset=utf-8",
+            dataType: "json",
+            url: window.location.pathname + "/../../ajax/mergebooks",
+            data: JSON.stringify({"Merge_books":selections}),
+            success: function success() {
+                $('#books-table').bootstrapTable('refresh');
+                $("#books-table").bootstrapTable('uncheckAll');
+            }
+        });
+    });
+
+    $("#merge_books").click(function() {
+        $.ajax({
+            method:"post",
+            contentType: "application/json; charset=utf-8",
+            dataType: "json",
+            url: window.location.pathname + "/../../ajax/simulatemerge",
+            data: JSON.stringify({"Merge_books":selections}),
+            success: function success(book_titles) {
+                $.each(book_titles.from, function(i, item) {
+                    $("<span>- " + item + "</span>").appendTo("#merge_from");
+                });
+                $('#merge_to').text("- " + book_titles.to);
+
+            }
+        });
+    });
+
+    var column = [];
+    $("#books-table > thead > tr > th").each(function() {
+        var element = {};
+        if ($(this).attr("data-edit")) {
+            element = {
+                editable: {
+                    mode: "inline",
+                    emptytext: "<span class='glyphicon glyphicon-plus'></span>",
+                }
+            };
+        }
+        var validateText = $(this).attr("data-edit-validate");
+        if (validateText) {
+            element.editable.validate = function (value) {
+                if ($.trim(value) === "") return validateText;
+            };
+        }
+        column.push(element);
+    });
+
+    $("#books-table").bootstrapTable({
+        sidePagination: "server",
+        pagination: true,
+        paginationLoop: false,
+        paginationDetailHAlign: " hidden",
+        paginationHAlign: "left",
+        idField: "id",
+        uniqueId: "id",
+        search: true,
+        showColumns: true,
+        searchAlign: "left",
+        showSearchButton : false,
+        searchOnEnterKey: true,
+        checkboxHeader: false,
+        maintainMetaData: true,
+        responseHandler: responseHandler,
+        columns: column,
+        formatNoMatches: function () {
+            return "";
+        },
+        onEditableSave: function (field, row, oldvalue, $el) {
+        if (field === 'title' || field === 'authors') {
+            $.ajax({
+                method:"get",
+                dataType: "json",
+                url: window.location.pathname + "/../../ajax/sort_value/" + field + '/' + row.id,
+                success: function success(data) {
+                    var key = Object.keys(data)[0]
+                    $("#books-table").bootstrapTable('updateCellByUniqueId', {
+                        id: row.id,
+                        field: key,
+                        value: data[key]
+                    });
+                    console.log(data);
+                }
+            });
+         }
+        },
+        onColumnSwitch: function (field, checked) {
+            var visible = $("#books-table").bootstrapTable('getVisibleColumns');
+            var hidden  = $("#books-table").bootstrapTable('getHiddenColumns');
+            var visibility =[]
+             var st = ""
+            visible.forEach(function(item) {
+                st += "\""+ item.field + "\":\"" +"true"+ "\","
+            });
+            hidden.forEach(function(item) {
+                st += "\""+ item.field + "\":\"" +"false"+ "\","
+            });
+            st = st.slice(0, -1);
+            $.ajax({
+                method:"post",
+                contentType: "application/json; charset=utf-8",
+                dataType: "json",
+                url: window.location.pathname + "/../../ajax/table_settings",
+                data: "{" + st + "}",
+            });
+        },
+    });
+
+
     $("#domain_allow_submit").click(function(event) {
         event.preventDefault();
         $("#domain_add_allow").ajaxForm();
@@ -33,6 +181,7 @@ $(function() {
             }
         });
     });
+
     $("#domain-allow-table").bootstrapTable({
         formatNoMatches: function () {
             return "";
@@ -205,6 +354,7 @@ function TableActions (value, row) {
     ].join("");
 }
 
+
 /* Function for deleting domain restrictions */
 function RestrictionActions (value, row) {
     return [
@@ -213,3 +363,20 @@ function RestrictionActions (value, row) {
         "</div>"
     ].join("");
 }
+
+/* Function for deleting books */
+function EbookActions (value, row) {
+    return [
+        "<div class=\"book-remove\" data-toggle=\"modal\" data-target=\"#deleteModal\" data-ajax=\"1\" data-delete-id=\"" + row.id + "\" title=\"Remove\">",
+        "<i class=\"glyphicon glyphicon-trash\"></i>",
+        "</div>"
+    ].join("");
+}
+
+/* Function for keeping checked rows */
+function responseHandler(res) {
+    $.each(res.rows, function (i, row) {
+        row.state = $.inArray(row.id, selections) !== -1;
+    });
+    return res;
+}

cps/tasks/convert.py

@@ -0,0 +1,217 @@
+from __future__ import division, print_function, unicode_literals
+import sys
+import os
+import re
+
+from glob import glob
+from shutil import copyfile
+
+from sqlalchemy.exc import SQLAlchemyError
+
+from cps.services.worker import CalibreTask, STAT_FINISH_SUCCESS
+from cps import calibre_db, db
+from cps import logger, config
+from cps.subproc_wrapper import process_open
+from flask_babel import gettext as _
+
+from cps.tasks.mail import TaskEmail
+from cps import gdriveutils
+log = logger.create()
+
+
+class TaskConvert(CalibreTask):
+    def __init__(self, file_path, bookid, taskMessage, settings, kindle_mail, user=None):
+        super(TaskConvert, self).__init__(taskMessage)
+        self.file_path = file_path
+        self.bookid = bookid
+        self.settings = settings
+        self.kindle_mail = kindle_mail
+        self.user = user
+
+        self.results = dict()
+
+    def run(self, worker_thread):
+        self.worker_thread = worker_thread
+        if config.config_use_google_drive:
+            cur_book = calibre_db.get_book(self.bookid)
+            data = calibre_db.get_book_format(self.bookid, self.settings['old_book_format'])
+            df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
+                                                     data.name + "." + self.settings['old_book_format'].lower())
+            if df:
+                datafile = os.path.join(config.config_calibre_dir,
+                                        cur_book.path,
+                                        data.name + u"." + self.settings['old_book_format'].lower())
+                if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)):
+                    os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path))
+                df.GetContentFile(datafile)
+            else:
+                error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
+                                  format=self.settings['old_book_format'],
+                                  fn=data.name + "." + self.settings['old_book_format'].lower())
+                return error_message
+
+        filename = self._convert_ebook_format()
+        if config.config_use_google_drive:
+            os.remove(self.file_path + u'.' + self.settings['old_book_format'].lower())
+
+        if filename:
+            if config.config_use_google_drive:
+                # Upload files to gdrive
+                gdriveutils.updateGdriveCalibreFromLocal()
+                self._handleSuccess()
+            if self.kindle_mail:
+                # if we're sending to kindle after converting, create a one-off task and run it immediately
+                # todo: figure out how to incorporate this into the progress
+                try:
+                    worker_thread.add(self.user, TaskEmail(self.settings['subject'], self.results["path"],
+                               filename, self.settings, self.kindle_mail,
+                               self.settings['subject'], self.settings['body'], internal=True))
+                except Exception as e:
+                    return self._handleError(str(e))
+
+    def _convert_ebook_format(self):
+        error_message = None
+        local_session = db.CalibreDB().session
+        file_path = self.file_path
+        book_id = self.bookid
+        format_old_ext = u'.' + self.settings['old_book_format'].lower()
+        format_new_ext = u'.' + self.settings['new_book_format'].lower()
+
+        # check to see if destination format already exists -
+        # if it does - mark the conversion task as complete and return a success
+        # this will allow send to kindle workflow to continue to work
+        if os.path.isfile(file_path + format_new_ext):
+            log.info("Book id %d already converted to %s", book_id, format_new_ext)
+            cur_book = calibre_db.get_book(book_id)
+            self.results['path'] = file_path
+            self.results['title'] = cur_book.title
+            self._handleSuccess()
+            return os.path.basename(file_path + format_new_ext)
+        else:
+            log.info("Book id %d - target format of %s does not exist. Moving forward with convert.",
+                     book_id,
+                     format_new_ext)
+
+        if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub':
+            check, error_message = self._convert_kepubify(file_path,
+                                                          format_old_ext,
+                                                          format_new_ext)
+        else:
+            # check if calibre converter-executable is existing
+            if not os.path.exists(config.config_converterpath):
+                # ToDo Text is not translated
+                self._handleError(_(u"Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
+                return
+            check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext)
+
+        if check == 0:
+            cur_book = calibre_db.get_book(book_id)
+            if os.path.isfile(file_path + format_new_ext):
+                # self.db_queue.join()
+                new_format = db.Data(name=cur_book.data[0].name,
+                                         book_format=self.settings['new_book_format'].upper(),
+                                         book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext))
+                try:
+                    local_session.merge(new_format)
+                    local_session.commit()
+                except SQLAlchemyError as e:
+                    local_session.rollback()
+                    log.error("Database error: %s", e)
+                    return
+                self.results['path'] = cur_book.path
+                self.results['title'] = cur_book.title
+                if not config.config_use_google_drive:
+                    self._handleSuccess()
+                return os.path.basename(file_path + format_new_ext)
+            else:
+                error_message = _('%(format)s format not found on disk', format=format_new_ext.upper())
+        log.info("ebook converter failed with error while converting book")
+        if not error_message:
+            error_message = _('Ebook converter failed with unknown error')
+        self._handleError(error_message)
+        return
+
+    def _convert_kepubify(self, file_path, format_old_ext, format_new_ext):
+        quotes = [1, 3]
+        command = [config.config_kepubifypath, (file_path + format_old_ext), '-o', os.path.dirname(file_path)]
+        try:
+            p = process_open(command, quotes)
+        except OSError as e:
+            return 1, _(u"Kepubify-converter failed: %(error)s", error=e)
+        self.progress = 0.01
+        while True:
+            nextline = p.stdout.readlines()
+            nextline = [x.strip('\n') for x in nextline if x != '\n']
+            if sys.version_info < (3, 0):
+                nextline = [x.decode('utf-8') for x in nextline]
+            for line in nextline:
+                log.debug(line)
+            if p.poll() is not None:
+                break
+
+        # ToD Handle
+        # process returncode
+        check = p.returncode
+
+        # move file
+        if check == 0:
+            converted_file = glob(os.path.join(os.path.dirname(file_path), "*.kepub.epub"))
+            if len(converted_file) == 1:
+                copyfile(converted_file[0], (file_path + format_new_ext))
+                os.unlink(converted_file[0])
+            else:
+                return 1, _(u"Converted file not found or more than one file in folder %(folder)s",
+                            folder=os.path.dirname(file_path))
+        return check, None
+
+    def _convert_calibre(self, file_path, format_old_ext, format_new_ext):
+        try:
+            # Linux py2.7 encode as list without quotes no empty element for parameters
+            # linux py3.x no encode and as list without quotes no empty element for parameters
+            # windows py2.7 encode as string with quotes empty element for parameters is okay
+            # windows py 3.x no encode and as string with quotes empty element for parameters is okay
+            # separate handling for windows and linux
+            quotes = [1, 2]
+            command = [config.config_converterpath, (file_path + format_old_ext),
+                       (file_path + format_new_ext)]
+            quotes_index = 3
+            if config.config_calibre:
+                parameters = config.config_calibre.split(" ")
+                for param in parameters:
+                    command.append(param)
+                    quotes.append(quotes_index)
+                    quotes_index += 1
+
+            p = process_open(command, quotes)
+        except OSError as e:
+            return 1, _(u"Ebook-converter failed: %(error)s", error=e)
+
+        while p.poll() is None:
+            nextline = p.stdout.readline()
+            if os.name == 'nt' and sys.version_info < (3, 0):
+                nextline = nextline.decode('windows-1252')
+            elif os.name == 'posix' and sys.version_info < (3, 0):
+                nextline = nextline.decode('utf-8')
+            log.debug(nextline.strip('\r\n'))
+            # parse progress string from calibre-converter
+            progress = re.search(r"(\d+)%\s.*", nextline)
+            if progress:
+                self.progress = int(progress.group(1)) / 100
+                if config.config_use_google_drive:
+                    self.progress *= 0.9
+
+        # process returncode
+        check = p.returncode
+        calibre_traceback = p.stderr.readlines()
+        error_message = ""
+        for ele in calibre_traceback:
+            if sys.version_info < (3, 0):
+                ele = ele.decode('utf-8')
+            log.debug(ele.strip('\n'))
+            if not ele.startswith('Traceback') and not ele.startswith('  File'):
+                error_message = _("Calibre failed with error: %(error)s", error=ele.strip('\n'))
+        return check, error_message
+
+    @property
+    def name(self):
+        return "Convert"

cps/tasks/mail.py

@@ -0,0 +1,241 @@
+from __future__ import division, print_function, unicode_literals
+import sys
+import os
+import smtplib
+import threading
+import socket
+
+try:
+    from StringIO import StringIO
+    from email.MIMEBase import MIMEBase
+    from email.MIMEMultipart import MIMEMultipart
+    from email.MIMEText import MIMEText
+except ImportError:
+    from io import StringIO
+    from email.mime.base import MIMEBase
+    from email.mime.multipart import MIMEMultipart
+    from email.mime.text import MIMEText
+
+from email import encoders
+from email.utils import formatdate, make_msgid
+from email.generator import Generator
+
+from cps.services.worker import CalibreTask
+from cps import logger, config
+
+from cps import gdriveutils
+
+log = logger.create()
+
+CHUNKSIZE = 8192
+
+
+# Class for sending email with ability to get current progress
+class EmailBase:
+
+    transferSize = 0
+    progress = 0
+
+    def data(self, msg):
+        self.transferSize = len(msg)
+        (code, resp) = smtplib.SMTP.data(self, msg)
+        self.progress = 0
+        return (code, resp)
+
+    def send(self, strg):
+        """Send `strg' to the server."""
+        log.debug('send: %r', strg[:300])
+        if hasattr(self, 'sock') and self.sock:
+            try:
+                if self.transferSize:
+                    lock=threading.Lock()
+                    lock.acquire()
+                    self.transferSize = len(strg)
+                    lock.release()
+                    for i in range(0, self.transferSize, CHUNKSIZE):
+                        if isinstance(strg, bytes):
+                            self.sock.send((strg[i:i + CHUNKSIZE]))
+                        else:
+                            self.sock.send((strg[i:i + CHUNKSIZE]).encode('utf-8'))
+                        lock.acquire()
+                        self.progress = i
+                        lock.release()
+                else:
+                    self.sock.sendall(strg.encode('utf-8'))
+            except socket.error:
+                self.close()
+                raise smtplib.SMTPServerDisconnected('Server not connected')
+        else:
+            raise smtplib.SMTPServerDisconnected('please run connect() first')
+
+    @classmethod
+    def _print_debug(cls, *args):
+        log.debug(args)
+
+    def getTransferStatus(self):
+        if self.transferSize:
+            lock2 = threading.Lock()
+            lock2.acquire()
+            value = int((float(self.progress) / float(self.transferSize))*100)
+            lock2.release()
+            return value / 100
+        else:
+            return 1
+
+
+# Class for sending email with ability to get current progress, derived from emailbase class
+class Email(EmailBase, smtplib.SMTP):
+
+    def __init__(self, *args, **kwargs):
+        smtplib.SMTP.__init__(self, *args, **kwargs)
+
+
+# Class for sending ssl encrypted email with ability to get current progress, , derived from emailbase class
+class EmailSSL(EmailBase, smtplib.SMTP_SSL):
+
+    def __init__(self, *args, **kwargs):
+        smtplib.SMTP_SSL.__init__(self, *args, **kwargs)
+
+
+class TaskEmail(CalibreTask):
+    def __init__(self, subject, filepath, attachment, settings, recipient, taskMessage, text, internal=False):
+        super(TaskEmail, self).__init__(taskMessage)
+        self.subject = subject
+        self.attachment = attachment
+        self.settings = settings
+        self.filepath = filepath
+        self.recipent = recipient
+        self.text = text
+        self.asyncSMTP = None
+
+        self.results = dict()
+
+    def run(self, worker_thread):
+        # create MIME message
+        msg = MIMEMultipart()
+        msg['Subject'] = self.subject
+        msg['Message-Id'] = make_msgid('calibre-web')
+        msg['Date'] = formatdate(localtime=True)
+        text = self.text
+        msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
+        if self.attachment:
+            result = self._get_attachment(self.filepath, self.attachment)
+            if result:
+                msg.attach(result)
+            else:
+                self._handleError(u"Attachment not found")
+                return
+
+        msg['From'] = self.settings["mail_from"]
+        msg['To'] = self.recipent
+
+        use_ssl = int(self.settings.get('mail_use_ssl', 0))
+        try:
+            # convert MIME message to string
+            fp = StringIO()
+            gen = Generator(fp, mangle_from_=False)
+            gen.flatten(msg)
+            msg = fp.getvalue()
+
+            # send email
+            timeout = 600  # set timeout to 5mins
+
+            # redirect output to logfile on python2 pn python3 debugoutput is caught with overwritten
+            # _print_debug function
+            if sys.version_info < (3, 0):
+                org_smtpstderr = smtplib.stderr
+                smtplib.stderr = logger.StderrLogger('worker.smtp')
+
+            if use_ssl == 2:
+                self.asyncSMTP = EmailSSL(self.settings["mail_server"], self.settings["mail_port"],
+                                           timeout=timeout)
+            else:
+                self.asyncSMTP = Email(self.settings["mail_server"], self.settings["mail_port"], timeout=timeout)
+
+            # link to logginglevel
+            if logger.is_debug_enabled():
+                self.asyncSMTP.set_debuglevel(1)
+            if use_ssl == 1:
+                self.asyncSMTP.starttls()
+            if self.settings["mail_password"]:
+                self.asyncSMTP.login(str(self.settings["mail_login"]), str(self.settings["mail_password"]))
+            self.asyncSMTP.sendmail(self.settings["mail_from"], self.recipent, msg)
+            self.asyncSMTP.quit()
+            self._handleSuccess()
+
+            if sys.version_info < (3, 0):
+                smtplib.stderr = org_smtpstderr
+
+        except (MemoryError) as e:
+            log.exception(e)
+            self._handleError(u'MemoryError sending email: ' + str(e))
+            # return None
+        except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
+            if hasattr(e, "smtp_error"):
+                text = e.smtp_error.decode('utf-8').replace("\n", '. ')
+            elif hasattr(e, "message"):
+                text = e.message
+            elif hasattr(e, "args"):
+                text = '\n'.join(e.args)
+            else:
+                log.exception(e)
+                text = ''
+            self._handleError(u'Smtplib Error sending email: ' + text)
+            # return None
+        except (socket.error) as e:
+            self._handleError(u'Socket Error sending email: ' + e.strerror)
+            # return None
+
+
+    @property
+    def progress(self):
+        if self.asyncSMTP is not None:
+            return self.asyncSMTP.getTransferStatus()
+        else:
+            return self._progress
+
+    @progress.setter
+    def progress(self, x):
+        """This gets explicitly set when handle(Success|Error) are called. In this case, remove the SMTP connection"""
+        if x == 1:
+            self.asyncSMTP = None
+            self._progress = x
+
+
+    @classmethod
+    def _get_attachment(cls, bookpath, filename):
+        """Get file as MIMEBase message"""
+        calibrepath = config.config_calibre_dir
+        if config.config_use_google_drive:
+            df = gdriveutils.getFileFromEbooksFolder(bookpath, filename)
+            if df:
+                datafile = os.path.join(calibrepath, bookpath, filename)
+                if not os.path.exists(os.path.join(calibrepath, bookpath)):
+                    os.makedirs(os.path.join(calibrepath, bookpath))
+                df.GetContentFile(datafile)
+            else:
+                return None
+            file_ = open(datafile, 'rb')
+            data = file_.read()
+            file_.close()
+            os.remove(datafile)
+        else:
+            try:
+                file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb')
+                data = file_.read()
+                file_.close()
+            except IOError as e:
+                log.exception(e)
+                log.error(u'The requested file could not be read. Maybe wrong permissions?')
+                return None
+
+        attachment = MIMEBase('application', 'octet-stream')
+        attachment.set_payload(data)
+        encoders.encode_base64(attachment)
+        attachment.add_header('Content-Disposition', 'attachment',
+                              filename=filename)
+        return attachment
+
+    @property
+    def name(self):
+        return "Email"

cps/tasks/upload.py

@@ -0,0 +1,19 @@
+from __future__ import division, print_function, unicode_literals
+
+from datetime import datetime
+from cps.services.worker import CalibreTask, STAT_FINISH_SUCCESS
+
+class TaskUpload(CalibreTask):
+    def __init__(self, taskMessage):
+        super(TaskUpload, self).__init__(taskMessage)
+        self.start_time = self.end_time = datetime.now()
+        self.stat = STAT_FINISH_SUCCESS
+        self.progress = 1
+
+    def run(self, worker_thread):
+        """Upload task doesn't have anything to do, it's simply a way to add information to the task list"""
+        pass
+
+    @property
+    def name(self):
+        return "Upload"

cps/templates/admin.html

@@ -161,8 +161,8 @@
       </table>
 
       <div class="hidden" id="update_error"> <span>{{update_error}}</span></div>
-      <div class="btn btn-default" id="check_for_update">{{_('Check for Update')}}</div>
-      <div class="btn btn-default hidden" id="perform_update" data-toggle="modal" data-target="#StatusDialog">{{_('Perform Update')}}</div>
+      <div class="btn btn-primary" id="check_for_update">{{_('Check for Update')}}</div>
+      <div class="btn btn-primary hidden" id="perform_update" data-toggle="modal" data-target="#StatusDialog">{{_('Perform Update')}}</div>
     </div>
   </div>
 </div>

cps/templates/author.html

@@ -23,14 +23,14 @@
     <h3>{{_("In Library")}}</h3>
   {% endif %}
     <div class="filterheader hidden-xs hidden-sm">
-      <a id="new" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort='new')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
-      <a id="old" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort='old')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
-      <a id="asc" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort='abc')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet"></span></a>
-      <a id="desc" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort='zyx')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></a>
-      <a id="pub_new" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort='pubnew')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
-      <a id="pub_old" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort='pubold')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
+      <a id="new" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort_param='new')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
+      <a id="old" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort_param='old')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
+      <a id="asc" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort_param='abc')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet"></span></a>
+      <a id="desc" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort_param='zyx')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></a>
+      <a id="pub_new" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort_param='pubnew')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
+      <a id="pub_old" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort_param='pubold')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
       <!--div class="btn-group character" role="group">
-        <a id="no_shelf" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort='pubold')}}"><span class="glyphicon glyphicon-list"></span><b>?</b></a>
+        <a id="no_shelf" class="btn btn-primary" href="{{url_for('web.books_list', data='author', book_id=id, sort_param='pubold')}}"><span class="glyphicon glyphicon-list"></span><b>?</b></a>
         <div id="all" class="btn btn-primary">{{_('All')}}</div>
       </div-->
     </div>
@@ -53,7 +53,7 @@
               {% if not loop.first %}
                 <span class="author-hidden-divider">&amp;</span>
 			  {% endif %}
-              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
               {% if loop.last %}
                 <a href="#" class="author-expand" data-authors-max="{{g.config_authors_max}}" data-collapse-caption="({{_('reduce')}})">(...)</a>
               {% endif %}
@@ -61,7 +61,7 @@
               {% if not loop.first %}
                 <span>&amp;</span>
               {% endif %}
-              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
             {% endif %}
           {% endfor %}
           {% for format in entry.data %}

cps/templates/book_edit.html

@@ -7,13 +7,13 @@
     </div>
 {% if g.user.role_delete_books() %}
     <div class="text-center">
-      <button type="button" class="btn btn-danger" id="delete" data-toggle="modal" data-target="#deleteModal">{{_("Delete Book")}}</button>
+      <button type="button" class="btn btn-danger" id="delete" data-toggle="modal" data-delete-id="{{ book.id }}" data-target="#deleteModal">{{_("Delete Book")}}</button>
     </div>
     {% if book.data|length > 1 %}
       <div class="text-center more-stuff"><h4>{{_('Delete formats:')}}</h4>
       {% for file in book.data %}
         <div class="form-group">
-          <a href="{{ url_for('editbook.delete_book', book_id=book.id, book_format=file.format) }}" class="btn btn-danger" type="button">{{_('Delete')}} - {{file.format}}</a>
+          <button type="button" class="btn btn-danger" id="delete_format" data-toggle="modal" data-delete-id="{{ book.id }}" data-delete-format="{{ file.format }}" data-target="#deleteModal">{{_('Delete')}} - {{file.format}}</button>
         </div>
       {% endfor %}
       </div>
@@ -197,34 +197,7 @@
 {% endblock %}
 
 {% block modal %}
-{% if g.user.role_delete_books() %}
-<div class="modal fade" id="deleteModal" role="dialog" aria-labelledby="metaDeleteLabel">
-  <div class="modal-dialog">
-    <div class="modal-content">
-      <div class="modal-header bg-danger text-center">
-          <span>{{_('Are you really sure?')}}</span>
-      </div>
-        <div class="modal-body text-center">
-          <p>
-          <span>{{_('This book will be permanently erased from database')}}</span>
-          <span>{{_('and hard disk')}}</span>
-		  </p>
-          {% if config.config_kobo_sync %}
-          <p>
-            <span>{{_('Important Kobo Note: deleted books will remain on any paired Kobo device.')}}</span>
-            <span>{{_('Books must first be archived and the device synced before a book can safely be deleted.')}}</span>
-          </p>
-          {% endif %}
-        </div>
-
-      <div class="modal-footer">
-        <a href="{{ url_for('editbook.delete_book', book_id=book.id) }}" id="delete_confirm" class="btn btn-danger">{{_('Delete')}}</a>
-        <button type="button" class="btn btn-default" data-dismiss="modal">{{_('Cancel')}}</button>
-      </div>
-    </div>
-  </div>
-</div>
-{% endif %}
+{{ delete_book(book.id) }}
 
 <div class="modal fade" id="metaModal" tabindex="-1" role="dialog" aria-labelledby="metaModalLabel">
   <div class="modal-dialog modal-lg" role="document">

cps/templates/book_table.html

@@ -1,59 +1,99 @@
 {% extends "layout.html" %}
-{% block body %}
-<h1 class="{{page}}">{{_(title)}}</h1>
+{% macro text_table_row(parameter, edit_text, show_text, validate) -%}
+<th data-field="{{ parameter }}" id="{{ parameter }}" data-sortable="true"
+    data-visible = "{{visiblility.get(parameter)}}"
+    {% if g.user.role_edit() %}
+        data-editable-type="text"
+        data-editable-url="{{ url_for('editbook.edit_list_book', param=parameter)}}"
+        data-editable-title="{{ edit_text }}"
+        data-edit="true"
+        {% if validate %}data-edit-validate="{{ _('This Field is Required') }}" {% endif %}
+    {% endif %}
+>{{ show_text }}</th>
+{%- endmacro %}
 
-    <div class="filterheader hidden-xs hidden-sm">
-      {% if entries.__len__() %}
-       {% if data == 'author' %}
-        <button id="sort_name" class="btn btn-primary"><b>B,A <-> A B</b></button>
-       {% endif %}
-      {% endif %}
-      <button id="desc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
-      <button id="asc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
-      {% if charlist|length %}
-      <button id="all" class="btn btn-primary">{{_('All')}}</button>
-      {% endif %}
-      <div class="btn-group character" role="group">
-        {% for char in charlist%}
-        <button class="btn btn-primary char">{{char.char}}</button>
-        {% endfor %}
+{% block header %}
+<link href="{{ url_for('static', filename='css/libs/bootstrap-table.min.css') }}" rel="stylesheet">
+<link href="{{ url_for('static', filename='css/libs/bootstrap-editable.css') }}" rel="stylesheet">
+{% endblock %}
+{% block body %}
+<h2 class="{{page}}">{{_(title)}}</h2>
+      <div class="col-xs-12 col-sm-6">
+        <div class="row">
+          <div class="btn btn-default disabled" id="merge_books" data-toggle="modal" data-target="#mergeModal" aria-disabled="true">{{_('Merge selected books')}}</div>
+          <div class="btn btn-default disabled" id="delete_selection" aria-disabled="true">{{_('Remove Selections')}}</div>
+        </div>
+      </div>
+     <div class="col-xs-12 col-sm-6">
+      <div class="row">
+        <input type="checkbox" id="autoupdate_titlesort" name="autoupdate_titlesort" checked>
+        <label for="autoupdate_titlesort">{{_('Update Title Sort automatically')}}</label>
+      </div>
+      <div class="row">
+        <input type="checkbox" id="autoupdate_autorsort" name="autoupdate_autorsort" checked>
+        <label for="autoupdate_autorsort">{{_('Update Author Sort automatically')}}</label>
       </div>
-
-      {% if title == "Series" %}
-      <button class="update-view btn btn-primary" href="#" data-target="series_view" data-view="grid">Grid</button>
-      {% endif %}
     </div>
-  <div class="container">
-    <div id="list" class="col-xs-12 col-sm-6">
-    {% for entry in entries %}
-      {% if loop.index0 == (loop.length/2+loop.length%2)|int and loop.length > 20 %}
+
+    <table id="books-table" class="table table-no-bordered table-striped"
+           data-url="{{url_for('web.list_books')}}">
+      <thead>
+        <tr>
+          {% if g.user.role_edit() %}
+            <th data-field="state" data-checkbox="true" data-sortable="true"></th>
+          {% endif %}
+            <th data-field="id" id="id" data-visible="false" data-switchable="false"></th>
+            {{ text_table_row('title', _('Enter Title'),_('Title'), true) }}
+            {{ text_table_row('sort', _('Enter Title Sort'),_('Title Sort'), false) }}
+            {{ text_table_row('author_sort', _('Enter Author Sort'),_('Author Sort'), false) }}
+            {{ text_table_row('authors', _('Enter Authors'),_('Authors'), true) }}
+            {{ text_table_row('tags', _('Enter Categories'),_('Categories'), false) }}
+            {{ text_table_row('series', _('Enter Series'),_('Series'), false) }}
+            <th data-field="series_index" id="series_index" data-visible="{{visiblility.get('series_index')}}" data-edit-validate="{{ _('This Field is Required') }}" data-sortable="true" {% if g.user.role_edit() %} data-editable-type="number" data-editable-placeholder="1" data-editable-step="0.01" data-editable-min="0" data-editable-url="{{ url_for('editbook.edit_list_book', param='series_index')}}" data-edit="true" data-editable-title="{{_('Enter title')}}"{% endif %}>{{_('Series Index')}}</th>
+            {{ text_table_row('languages', _('Enter Languages'),_('Languages'), false) }}
+            <!--th data-field="pubdate" data-type="date" data-visible="{{visiblility.get('pubdate')}}" data-viewformat="dd.mm.yyyy" id="pubdate" data-sortable="true">{{_('Publishing Date')}}</th-->
+            {{ text_table_row('publishers', _('Enter Publishers'),_('Publishers'), false) }}
+          {% if g.user.role_edit() %}
+            <th data-align="right" data-formatter="EbookActions" data-switchable="false">{{_('Delete')}}</th>
+          {% endif %}
+        </tr>
+      </thead>
+    </table>
+{% endblock %}
+{% block modal %}
+{{ delete_book(0) }}
+{% if g.user.role_edit() %}
+<div class="modal fade" id="mergeModal" role="dialog" aria-labelledby="metaMergeLabel">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header bg-danger text-center">
+          <span>{{_('Are you really sure?')}}</span>
+      </div>
+        <div class="modal-body">
+          <p></p>
+            <div class="text-left">{{_('Books with Title will be merged from:')}}</div>
+          <p></p>
+            <div class=text-left" id="merge_from"></div>
+          <p></p>
+            <div class="text-left">{{_('Into Book with Title:')}}</div>
+          <p></p>
+            <div class=text-left" id="merge_to"></div>
         </div>
-        <div id="second" class="col-xs-12 col-sm-6">
-      {% endif %}
-      <div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry.name %}{{entry.name}}{% else %}{{entry[0].name}}{% endif %}{% endif %}">
-        <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry.count}}</span></div>
-        <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{% if entry.format %}{{url_for('web.books_list', data=data, sort='new', book_id=entry.format )}}{% else %}{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].id )}}{% endif %}">
-          {% if entry.name %}
-          <div class="rating">
-          {% for number in range(entry.name) %}
-            <span class="glyphicon glyphicon-star good"></span>
-            {% if loop.last and loop.index < 5 %}
-              {% for numer in range(5 - loop.index) %}
-                <span class="glyphicon glyphicon-star"></span>
-              {% endfor %}
-            {% endif %}
-          {% endfor %}
-          </div>
-          {% else %}
-          {% if entry.format %}
-            {{entry.format}}
-          {% else %}
-            {{entry[0].name}}{% endif %}{% endif %}</a></div>
+      <div class="modal-footer">
+        <input type="button" class="btn btn-danger" value="{{_('Merge')}}" name="merge_confirm" id="merge_confirm" data-dismiss="modal">
+        <button type="button" class="btn btn-default" data-dismiss="modal">{{_('Cancel')}}</button>
       </div>
-    {% endfor %}
     </div>
   </div>
+</div>
+{% endif %}
+
 {% endblock %}
 {% block js %}
-<script src="{{ url_for('static', filename='js/filter_list.js') }}"></script>
+<script src="{{ url_for('static', filename='js/libs/bootstrap-table/bootstrap-table.min.js') }}"></script>
+<script src="{{ url_for('static', filename='js/libs/bootstrap-table/bootstrap-table-editable.min.js') }}"></script>
+<script src="{{ url_for('static', filename='js/libs/bootstrap-table/bootstrap-editable.min.js') }}"></script>
+<script src="{{ url_for('static', filename='js/table.js') }}"></script>
+<script>
+</script>
 {% endblock %}

cps/templates/detail.html

@@ -92,7 +92,7 @@
       <h2 id="title">{{entry.title|shortentitle(40)}}</h2>
       <p class="author">
           {% for author in entry.authors %}
-            <a href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id ) }}">{{author.name.replace('|',',')}}</a>
+            <a href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id ) }}">{{author.name.replace('|',',')}}</a>
             {% if not loop.last %}
               &amp;
             {% endif %}
@@ -114,7 +114,7 @@
       {% endif %}
 
       {% if entry.series|length > 0 %}
-        <p>{{_('Book')}} {{entry.series_index}} {{_('of')}} <a href="{{url_for('web.books_list', data='series',sort='abc', book_id=entry.series[0].id)}}">{{entry.series[0].name}}</a></p>
+        <p>{{_('Book')}} {{entry.series_index}} {{_('of')}} <a href="{{url_for('web.books_list', data='series', sort_param='abc', book_id=entry.series[0].id)}}">{{entry.series[0].name}}</a></p>
       {% endif %}
 
       {% if entry.languages.__len__() > 0 %}
@@ -143,7 +143,7 @@
         <span class="glyphicon glyphicon-tags"></span>
 
         {% for tag in entry.tags %}
-          <a href="{{ url_for('web.books_list', data='category', sort='new', book_id=tag.id) }}" class="btn btn-xs btn-info" role="button">{{tag.name}}</a>
+          <a href="{{ url_for('web.books_list', data='category', sort_param='new', book_id=tag.id) }}" class="btn btn-xs btn-info" role="button">{{tag.name}}</a>
         {%endfor%}
       </p>
 
@@ -154,13 +154,13 @@
       <div class="publishers">
         <p>
           <span>{{_('Publisher')}}:
-              <a href="{{url_for('web.books_list', data='publisher', sort='new', book_id=entry.publishers[0].id ) }}">{{entry.publishers[0].name}}</a>
+              <a href="{{url_for('web.books_list', data='publisher', sort_param='new', book_id=entry.publishers[0].id ) }}">{{entry.publishers[0].name}}</a>
           </span>
         </p>
       </div>
       {% endif %}
 
-      {% if entry.pubdate[:10] != '0101-01-01' %}
+      {% if (entry.pubdate|string)[:10] != '0101-01-01' %}
       <div class="publishing-date">
         <p>{{_('Published')}}: {{entry.pubdate|formatdate}} </p>
       </div>
@@ -281,7 +281,7 @@
       {% if g.user.role_edit() %}
       <div class="btn-toolbar" role="toolbar">
         <div class="btn-group" role="group" aria-label="Edit/Delete book">
-          <a href="{{ url_for('editbook.edit_book', book_id=entry.id) }}" class="btn btn-sm btn-warning" id="edit_book" role="button"><span class="glyphicon glyphicon-edit"></span> {{_('Edit Metadata')}}</a>
+          <a href="{{ url_for('editbook.edit_book', book_id=entry.id) }}" class="btn btn-sm btn-primary" id="edit_book" role="button"><span class="glyphicon glyphicon-edit"></span> {{_('Edit Metadata')}}</a>
         </div>
       </div>
       {% endif %}

cps/templates/discover.html

@@ -22,7 +22,7 @@
               {% if not loop.first %}
                 <span class="author-hidden-divider">&amp;</span>
 			  {% endif %}
-              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
               {% if loop.last %}
                 <a href="#" class="author-expand" data-authors-max="{{g.config_authors_max}}" data-collapse-caption="({{_('reduce')}})">(...)</a>
               {% endif %}
@@ -30,7 +30,7 @@
               {% if not loop.first %}
                 <span>&amp;</span>
               {% endif %}
-              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
             {% endif %}
           {% endfor %}
         </p>

cps/templates/grid.html

@@ -8,8 +8,8 @@
         <button id="sort_name" class="btn btn-primary"><b>B,A <-> A B</b></button>
        {% endif %}
       {% endif %}
-      <button id="desc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
-      <button id="asc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
+      <button id="desc" data-id="series" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
+      <button id="asc" data-id="series" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
       {% if charlist|length %}
       <button id="all" class="btn btn-primary">{{_('All')}}</button>
       {% endif %}
@@ -19,7 +19,7 @@
         {% endfor %}
       </div>
 
-        <button class="update-view btn btn-primary" href="#" data-target="series_view" data-view="list">List</button>
+        <button class="update-view btn btn-primary" href="#" data-target="series_view" id='list-button' data-view="list">List</button>
     </div>
 
     {% if entries[0] %}
@@ -27,13 +27,13 @@
           {% for entry in entries %}
               <div class="col-sm-3 col-lg-2 col-xs-6 book sortable" {% if entry[0].sort %}data-name="{{entry[0].series[0].name}}"{% endif %} data-id="{% if entry[0].series[0].name %}{{entry[0].series[0].name}}{% endif %}">
                   <div class="cover">
-                      <a href="{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].series[0].id )}}">
+                      <a href="{{url_for('web.books_list', data=data, sort_param='new', book_id=entry[0].series[0].id )}}">
                           <img src="{{ url_for('web.get_cover', book_id=entry[0].id) }}" alt="{{ entry[0].name }}"/>
                           <span class="badge">{{entry.count}}</span>
                       </a>
                   </div>
                   <div class="meta">
-                      <a href="{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].series[0].id )}}">
+                      <a href="{{url_for('web.books_list', data=data, sort_param='new', book_id=entry[0].series[0].id )}}">
                           <p class="title">{{entry[0].series[0].name|shortentitle}}</p>
                       </a>
                   </div>

cps/templates/index.html

@@ -21,7 +21,7 @@
               {% if not loop.first %}
                 <span class="author-hidden-divider">&amp;</span>
 			  {% endif %}
-              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
               {% if loop.last %}
                 <a href="#" class="author-expand" data-authors-max="{{g.config_authors_max}}" data-collapse-caption="({{_('reduce')}})">(...)</a>
               {% endif %}
@@ -29,7 +29,7 @@
               {% if not loop.first %}
                 <span>&amp;</span>
               {% endif %}
-              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
             {% endif %}
           {% endfor %}
         </p>
@@ -54,14 +54,14 @@
 <div class="discover load-more">
   <h2 class="{{title}}">{{_(title)}}</h2>
     <div class="filterheader hidden-xs hidden-sm">
-      <a data-toggle="tooltip" id="new" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort='new')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
-      <a id="old" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort='old')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
-      <a id="asc" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort='abc')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet"></span></a>
-      <a id="desc" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort='zyx')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></a>
-      <a id="pub_new" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort='pubnew')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
-      <a id="pub_old" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort='pubold')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
+      <a data-toggle="tooltip" id="new" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort_param='new')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
+      <a id="old" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort_param='old')}}"><span class="glyphicon glyphicon-book"></span> <span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
+      <a id="asc" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort_param='abc')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet"></span></a>
+      <a id="desc" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort_param='zyx')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></a>
+      <a id="pub_new" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort_param='pubnew')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
+      <a id="pub_old" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort_param='pubold')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
       <!--div class="btn-group character">
-        <a id="no_shelf" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort='pubold')}}"><span class="glyphicon glyphicon-list"></span> <b>{{_('Group by series')}}</b></a>
+        <a id="no_shelf" class="btn btn-primary" href="{{url_for('web.books_list', data=page, book_id=id, sort_param='pubold')}}"><span class="glyphicon glyphicon-list"></span> <b>{{_('Group by series')}}</b></a>
       </div-->
     </div>
 
@@ -84,7 +84,7 @@
               {% if not loop.first %}
                 <span class="author-hidden-divider">&amp;</span>
 			  {% endif %}
-              <a class="author-name author-hidden" href="{{url_for('web.books_list', data='author', book_id=author.id, sort='new') }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name author-hidden" href="{{url_for('web.books_list', data='author', book_id=author.id, sort_param='new') }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
               {% if loop.last %}
                 <a href="#" class="author-expand" data-authors-max="{{g.config_authors_max}}" data-collapse-caption="({{_('reduce')}})">(...)</a>
               {% endif %}
@@ -92,7 +92,7 @@
               {% if not loop.first %}
                 <span>&amp;</span>
               {% endif %}
-              <a class="author-name" href="{{url_for('web.books_list', data='author', book_id=author.id, sort='new') }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name" href="{{url_for('web.books_list', data='author', book_id=author.id, sort_param='new') }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
             {% endif %}
           {% endfor %}
           {% for format in entry.data %}

cps/templates/languages.html

@@ -10,7 +10,7 @@
     {% endif %}
     <div class="row">
         <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{lang_counter[loop.index0].bookcount}}</span></div>
-        <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{{url_for('web.books_list', book_id=lang.lang_code, data=data, sort='new')}}">{{lang.name}}</a></div>
+        <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{{url_for('web.books_list', book_id=lang.lang_code, data=data, sort_param='new')}}">{{lang.name}}</a></div>
     </div>
   {% endfor %}
   </div>

cps/templates/layout.html

@@ -1,4 +1,4 @@
-{% from 'modal_restriction.html' import restrict_modal %}
+{% from 'modal_dialogs.html' import restrict_modal, delete_book %}
 <!DOCTYPE html>
 <html lang="{{ g.user.locale }}">
   <head>
@@ -128,7 +128,7 @@
               <li class="nav-head hidden-xs">{{_('Browse')}}</li>
               {% for element in sidebar %}
                 {% if g.user.check_visibility(element['visibility']) and element['public'] %}
-                    <li id="nav_{{element['id']}}" {% if page == element['page'] %}class="active"{% endif %}><a href="{{url_for(element['link'], data=element['page'], sort='new')}}"><span class="glyphicon {{element['glyph']}}"></span>{{_(element['text'])}}</a></li>
+                    <li id="nav_{{element['id']}}" {% if page == element['page'] %}class="active"{% endif %}><a href="{{url_for(element['link'], data=element['page'], sort_param='stored')}}"><span class="glyphicon {{element['glyph']}}"></span>{{_(element['text'])}}</a></li>
                 {% endif %}
               {% endfor %}
               {% if g.user.is_authenticated or g.allow_anonymous %}
@@ -136,10 +136,6 @@
                 {% for shelf in g.shelves_access %}
                   <li><a href="{{url_for('shelf.show_shelf', shelf_id=shelf.id)}}"><span class="glyphicon glyphicon-list shelf"></span>{{shelf.name|shortentitle(40)}}{% if shelf.is_public == 1 %} {{_('(Public)')}}{% endif %}</a></li>
                 {% endfor %}
-                <!--li class="nav-head hidden-xs your-shelves">{{_('Your Shelves')}}</li>
-                {% for shelf in g.user.shelf %}
-                  <li><a href="{{url_for('shelf.show_shelf', shelf_id=shelf.id)}}"><span class="glyphicon glyphicon-list private_shelf"></span>{{shelf.name|shortentitle(40)}}{% if shelf.is_public == 1 %} {{_('(Public)')}}{% endif %}</a></li>
-                {% endfor %}-->
               {% if not g.user.is_anonymous %}
                 <li id="nav_createshelf" class="create-shelf"><a href="{{url_for('shelf.create_shelf')}}">{{_('Create a Shelf')}}</a></li>
                 <li id="nav_about" {% if page == 'stat' %}class="active"{% endif %}><a href="{{url_for('about.stats')}}"><span class="glyphicon glyphicon-info-sign"></span> {{_('About')}}</a></li>
@@ -155,29 +151,29 @@
           {% if pagination and (pagination.has_next or pagination.has_prev) %}
             <div class="pagination">
               {% if pagination.has_prev %}
-              <a class="previous" href="{{ (pagination.page - 1)|url_for_other_page
-                }}">&laquo; {{_('Previous')}}</a>
+              <li class="page-item page-previous"><a class="page-link" aria-label="next page" href="{{ (pagination.page - 1)|url_for_other_page
+                }}">&laquo; {{_('Previous')}}</a></li>
               {% endif %}
             {% for page in pagination.iter_pages() %}
               {% if page %}
                 {% if page != pagination.page %}
-                  <a href="{{ (page)|url_for_other_page }}">{{ page }}</a>
+                  <li class="page-item"><a class="page-link" aria-label="to page {{ page }}" href="{{ (page)|url_for_other_page }}">{{ page }}</a></li>
                 {% else %}
-                  <strong>{{ page }}</strong>
+                  <li class="page-item active"><a class="page-link" aria-label="to page {{ page }}" href="{{ (page)|url_for_other_page }}">{{ page }}</a></li>
                 {% endif %}
               {% else %}
-                <span class="ellipsis">…</span>
+                <li class="page-item page-last-separator disabled"><a class="page-link" aria-label="">…</a></li>
               {% endif %}
             {% endfor %}
             {% if pagination.has_next %}
-              <a class="next" href="{{ (pagination.page + 1)|url_for_other_page
-                }}">{{_('Next')}} &raquo;</a>
+              <li class="page-item page-next"><a class="page-link" aria-label="next page" href="{{ (pagination.page + 1)|url_for_other_page
+                }}">{{_('Next')}} &raquo;</a></li>
             {% endif %}
             </div>
           {% endif %}
-        </div>
       </div>
     </div>
+  </div>
     <div class="modal fade" id="bookDetailsModal" tabindex="-1" role="dialog" aria-labelledby="bookDetailsModalLabel">
       <div class="modal-dialog modal-lg" role="document">
         <div class="modal-content">
@@ -196,7 +192,6 @@
 
 
     <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
-    <!--script src="https://code.jquery.com/jquery.js"></script-->
     <script src="{{ url_for('static', filename='js/libs/jquery.min.js') }}"></script>
     <!-- Include all compiled plugins (below), or include individual files as needed -->
     <script src="{{ url_for('static', filename='js/libs/bootstrap.min.js') }}"></script>
@@ -227,9 +222,11 @@
             });
             $(document).ready(function() {
               var inp = $('#query').first()
-              var val = inp.val()
-              if (val !== "undefined") {
+              if (inp.length) {
+                var val = inp.val()
+                if (val.length) {
                   inp.val('').blur().focus().val(val)
+                }
               }
             });
         });

cps/templates/list.html

@@ -8,8 +8,8 @@
         <button id="sort_name" class="btn btn-primary"><b>B,A <-> A B</b></button>
        {% endif %}
       {% endif %}
-      <button id="desc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
-      <button id="asc" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
+      <button id="desc" data-id="{{ data }}" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet"></span></button>
+      <button id="asc" data-id="{{ data }}" class="btn btn-primary"><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></button>
       {% if charlist|length %}
       <button id="all" class="btn btn-primary">{{_('All')}}</button>
       {% endif %}
@@ -20,7 +20,7 @@
       </div>
 
       {% if data == "series" %}
-      <button class="update-view btn btn-primary" href="#" data-target="series_view" data-view="grid">Grid</button>
+      <button class="update-view btn btn-primary" href="#" data-target="series_view" id='grid-button' data-view="grid">Grid</button>
       {% endif %}
     </div>
   <div class="container">
@@ -32,7 +32,7 @@
       {% endif %}
       <div class="row" {% if entry[0].sort %}data-name="{{entry[0].name}}"{% endif %} data-id="{% if entry[0].sort %}{{entry[0].sort}}{% else %}{% if entry.name %}{{entry.name}}{% else %}{{entry[0].name}}{% endif %}{% endif %}">
         <div class="col-xs-2 col-sm-2 col-md-1" align="left"><span class="badge">{{entry.count}}</span></div>
-        <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{% if entry.format %}{{url_for('web.books_list', data=data, sort='new', book_id=entry.format )}}{% else %}{{url_for('web.books_list', data=data, sort='new', book_id=entry[0].id )}}{% endif %}">
+        <div class="col-xs-10 col-sm-10 col-md-11"><a id="list_{{loop.index0}}" href="{% if entry.format %}{{url_for('web.books_list', data=data, sort_param='new', book_id=entry.format )}}{% else %}{{url_for('web.books_list', data=data, sort_param='new', book_id=entry[0].id )}}{% endif %}">
           {% if entry.name %}
           <div class="rating">
           {% for number in range(entry.name) %}

cps/templates/modal_dialogs.html

@@ -37,3 +37,34 @@
   </div>
 </div>
 {% endmacro %}
+{% macro delete_book(bookid) %}
+{% if g.user.role_delete_books() %}
+<div class="modal fade" id="deleteModal" role="dialog" aria-labelledby="metaDeleteLabel">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header bg-danger text-center">
+          <span>{{_('Are you really sure?')}}</span>
+      </div>
+        <div class="modal-body text-center">
+          <p>
+          <span class="hidden" id="book_format">{{_('This book format will be permanently erased from database')}}</span>
+          <span class="hidden" id="book_complete">{{_('This book will be permanently erased from database')}}</span>
+          <span>{{_('and hard disk')}}</span>
+		  </p>
+          {% if config.config_kobo_sync %}
+          <p>
+            <span>{{_('Important Kobo Note: deleted books will remain on any paired Kobo device.')}}</span>
+            <span>{{_('Books must first be archived and the device synced before a book can safely be deleted.')}}</span>
+          </p>
+          {% endif %}
+        </div>
+
+      <div class="modal-footer">
+        <input type="button" class="btn btn-danger" value="{{_('Delete')}}" name="delete_confirm" id="delete_confirm" data-dismiss="modal">
+        <button type="button" class="btn btn-default" data-dismiss="modal">{{_('Cancel')}}</button>
+      </div>
+    </div>
+  </div>
+</div>
+{% endif %}
+{% endmacro %}

cps/templates/readcbr.html

@@ -14,8 +14,13 @@
 
   <script src="{{ url_for('static', filename='js/libs/jquery.min.js') }}"></script>
   <script src="{{ url_for('static', filename='js/libs/screenfull.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/io/bytestream.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/io/bytebuffer.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/io/bitstream.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/archive/archive.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/archive/rarvm.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/archive/unrar5.js') }}"></script>
   <script src="{{ url_for('static', filename='js/kthoom.js') }}"></script>
-	<script src="{{ url_for('static', filename='js/archive/archive.js') }}"></script>
   <script>
     var updateArrows = function() {
       if ($('input[name="direction"]:checked').val() === "0") {

cps/templates/search.html

@@ -5,7 +5,7 @@
       <h2>{{_('No Results Found')}} {{adv_searchterm}}</h2>
       <p>{{_('Search Term:')}} {{adv_searchterm}}</p>
     {% else %}
-      <h2>{{entries|length}} {{_('Results for:')}} {{adv_searchterm}}</h2>
+      <h2>{{result_count}} {{_('Results for:')}} {{adv_searchterm}}</h2>
       {% if g.user.is_authenticated %}
         {% if g.user.shelf.all() or g.shelves_access %}
           <div id="shelf-actions" class="btn-toolbar" role="toolbar">
@@ -25,18 +25,14 @@
           </div>
         {% endif %}
       {% endif %}
-      <!--div class="filterheader hidden-xs hidden-sm"--><!-- ToDo: Implement filter for search results -->
-        <!--a id="new" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort='new')}}"><span class="glyphicon glyphicon-sort-by-order"></span></a>
-        <a id="old" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort='old')}}"><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
-        <a id="asc" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort='abc')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet"></span></a>
-        <a id="desc" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort='zyx')}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></a>
-        <a id="pub_new" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort='pubnew')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
-        <a id="pub_old" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort='pubold')}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
+      <div class="filterheader hidden-xs hidden-sm"><!-- ToDo: Implement filter for search results -->
+        <a id="new" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort_param='new', query=query)}}"><span class="glyphicon glyphicon-sort-by-order"></span></a>
+        <a id="old" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort_param='old', query=query)}}"><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
+        <a id="asc" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort_param='abc', query=query)}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet"></span></a>
+        <a id="desc" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort_param='zyx', query=query)}}"><span class="glyphicon glyphicon-font"></span><span class="glyphicon glyphicon-sort-by-alphabet-alt"></span></a>
+        <a id="pub_new" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort_param='pubnew', query=query)}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order"></span></a>
+        <a id="pub_old" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort_param='pubold', query=query)}}"><span class="glyphicon glyphicon-calendar"></span><span class="glyphicon glyphicon-sort-by-order-alt"></span></a>
       </div>
-      <div class="btn-group character" role="group">
-        <a id="no_shelf" class="btn btn-primary" href="{{url_for('web.books_list', data=page, sort='pubold')}}"><span class="glyphicon glyphicon-list"></span><b>?</b></a>
-        <div id="all" class="btn btn-primary">{{_('All')}}</div>
-      </div-->
   {% endif %}
 
   <div class="row">
@@ -59,7 +55,7 @@
               {% if not loop.first %}
                 <span class="author-hidden-divider">&amp;</span>
               {% endif %}
-              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
               {% if loop.last %}
                 <a href="#" class="author-expand" data-authors-max="{{g.config_authors_max}}" data-collapse-caption="({{_('reduce')}})">(...)</a>
               {% endif %}
@@ -67,7 +63,7 @@
               {% if not loop.first %}
                 <span>&amp;</span>
               {% endif %}
-              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
             {% endif %}
           {% endfor %}
           {% for format in entry.data %}

cps/templates/search_form.html

@@ -1,7 +1,7 @@
 {% extends "layout.html" %}
 {% block body %}
 <div class="col-md-10 col-lg-6">
-  <form role="form" id="search" action="{{ url_for('web.advanced_search') }}" method="GET">
+  <form role="form" id="search" action="{{ url_for('web.advanced_search_form') }}" method="POST">
     <div class="form-group">
       <label for="book_title">{{_('Book Title')}}</label>
       <input type="text" class="form-control" name="book_title" id="book_title" value="">

cps/templates/shelf.html

@@ -31,7 +31,7 @@
               {% if not loop.first %}
                 <span class="author-hidden-divider">&amp;</span>
 			  {% endif %}
-              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name author-hidden" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
               {% if loop.last %}
                 <a href="#" class="author-expand" data-authors-max="{{g.config_authors_max}}" data-collapse-caption="({{_('reduce')}})">(...)</a>
               {% endif %}
@@ -39,7 +39,7 @@
               {% if not loop.first %}
                 <span>&amp;</span>
               {% endif %}
-              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
+              <a class="author-name" href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')|shortentitle(30)}}</a>
             {% endif %}
           {% endfor %}
         </p>

cps/templates/shelfdown.html

@@ -37,7 +37,7 @@
         <p class="title">{{entry.title|shortentitle}}</p>
         <p class="author">
           {% for author in entry.authors %}
-            <a href="{{url_for('web.books_list',  data='author', sort='new', book_id=author.id) }}">{{author.name.replace('|',',')}}</a>
+            <a href="{{url_for('web.books_list',  data='author', sort_param='new', book_id=author.id) }}">{{author.name.replace('|',',')}}</a>
             {% if not loop.last %}
               &amp;
             {% endif %}

cps/templates/user_edit.html

@@ -140,20 +140,8 @@
     {% endif %}
     </div>
   </form>
-
-  {% if downloads %}
-    <div class="col-sm-12">
-    <h2>{{_('Recent Downloads')}}</h2>
-    {% for entry in downloads %}
-      <div class="col-sm-2">
-        <a class="pull-left" href="{{ url_for('web.show_book', book_id=entry.id) }}">
-          <img class="media-object cover-small" src="{{ url_for('web.get_cover', book_id=entry.id) }}" alt="...">
-        </a>
-      </div>
-    {% endfor %}
-    </div>
-  {% endif %}
 </div>
+
     <div class="modal fade" id="modal_kobo_token" tabindex="-1" role="dialog" aria-labelledby="kobo_tokenModalLabel">
       <div class="modal-dialog modal-lg" role="document">
         <div class="modal-content">

cps/ub.py

@@ -23,11 +23,12 @@ import sys
 import datetime
 import itertools
 import uuid
+from flask import session as flask_session
 from binascii import hexlify
 
 from flask import g
 from flask_babel import gettext as _
-from flask_login import AnonymousUserMixin
+from flask_login import AnonymousUserMixin, current_user
 from werkzeug.local import LocalProxy
 try:
     from flask_dance.consumer.backend.sqla import OAuthConsumerMixin
@@ -41,8 +42,9 @@ except ImportError:
         oauth_support = False
 from sqlalchemy import create_engine, exc, exists, event
 from sqlalchemy import Column, ForeignKey
-from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float
+from sqlalchemy import String, Integer, SmallInteger, Boolean, DateTime, Float, JSON
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.orm import backref, relationship, sessionmaker, Session
 from werkzeug.security import generate_password_hash
 
@@ -52,6 +54,7 @@ from . import constants
 session = None
 app_DB_path = None
 Base = declarative_base()
+searched_ids = {}
 
 
 def get_sidebar_config(kwargs=None):
@@ -68,13 +71,17 @@ def get_sidebar_config(kwargs=None):
     sidebar.append({"glyph": "glyphicon-fire", "text": _('Hot Books'), "link": 'web.books_list', "id": "hot",
                     "visibility": constants.SIDEBAR_HOT, 'public': True, "page": "hot",
                     "show_text": _('Show Hot Books'), "config_show": True})
+    sidebar.append({"glyph": "glyphicon-download", "text": _('Downloaded Books'), "link": 'web.books_list',
+                    "id": "download", "visibility": constants.SIDEBAR_DOWNLOAD, 'public': (not g.user.is_anonymous),
+                    "page": "download", "show_text": _('Show Downloaded Books'),
+                    "config_show": content})
     sidebar.append(
         {"glyph": "glyphicon-star", "text": _('Top Rated Books'), "link": 'web.books_list', "id": "rated",
          "visibility": constants.SIDEBAR_BEST_RATED, 'public': True, "page": "rated",
          "show_text": _('Show Top Rated Books'), "config_show": True})
     sidebar.append({"glyph": "glyphicon-eye-open", "text": _('Read Books'), "link": 'web.books_list', "id": "read",
-                    "visibility": constants.SIDEBAR_READ_AND_UNREAD, 'public': (not g.user.is_anonymous), "page": "read",
-                    "show_text": _('Show read and unread'), "config_show": content})
+                    "visibility": constants.SIDEBAR_READ_AND_UNREAD, 'public': (not g.user.is_anonymous),
+                    "page": "read", "show_text": _('Show read and unread'), "config_show": content})
     sidebar.append(
         {"glyph": "glyphicon-eye-close", "text": _('Unread Books'), "link": 'web.books_list', "id": "unread",
          "visibility": constants.SIDEBAR_READ_AND_UNREAD, 'public': (not g.user.is_anonymous), "page": "unread",
@@ -109,14 +116,21 @@ def get_sidebar_config(kwargs=None):
         {"glyph": "glyphicon-trash", "text": _('Archived Books'), "link": 'web.books_list', "id": "archived",
          "visibility": constants.SIDEBAR_ARCHIVED, 'public': (not g.user.is_anonymous), "page": "archived",
          "show_text": _('Show archived books'), "config_show": content})
-    '''sidebar.append(
-        {"glyph": "glyphicon-th-list", "text": _('Books List'), "link": 'web.books_list', "id": "list",
+    sidebar.append(
+        {"glyph": "glyphicon-th-list", "text": _('Books List'), "link": 'web.books_table', "id": "list",
          "visibility": constants.SIDEBAR_LIST, 'public': (not g.user.is_anonymous), "page": "list",
-         "show_text": _('Show Books List'), "config_show": content})'''
+         "show_text": _('Show Books List'), "config_show": content})
 
     return sidebar
 
 
+def store_ids(result):
+    ids = list()
+    for element in result:
+        ids.append(element.id)
+    searched_ids[current_user.id] = ids
+
+
 class UserBase:
 
     @property
@@ -191,6 +205,25 @@ class UserBase:
         mct = self.allowed_column_value or ""
         return [t.strip() for t in mct.split(",")]
 
+    def get_view_property(self, page, property):
+        if not self.view_settings.get(page):
+            return None
+        return self.view_settings[page].get(property)
+
+    def set_view_property(self, page, property, value):
+        if not self.view_settings.get(page):
+            self.view_settings[page] = dict()
+        self.view_settings[page][property] = value
+        try:
+            flag_modified(self, "view_settings")
+        except AttributeError:
+            pass
+        try:
+            session.commit()
+        except (exc.OperationalError, exc.InvalidRequestError):
+            session.rollback()
+            # ToDo: Error message
+
     def __repr__(self):
         return '<User %r>' % self.nickname
 
@@ -218,7 +251,8 @@ class User(UserBase, Base):
     denied_column_value = Column(String, default="")
     allowed_column_value = Column(String, default="")
     remote_auth_token = relationship('RemoteAuthToken', backref='user', lazy='dynamic')
-    series_view = Column(String(10), default="list")
+    view_settings = Column(JSON, default={})
+
 
 
 if oauth_support:
@@ -259,7 +293,11 @@ class Anonymous(AnonymousUserMixin, UserBase):
         self.allowed_tags = data.allowed_tags
         self.denied_column_value = data.denied_column_value
         self.allowed_column_value = data.allowed_column_value
-        self.series_view = data.series_view
+        self.view_settings = data.view_settings
+        # Initialize flask_session once
+        if 'view' not in flask_session:
+            flask_session['view']={}
+
 
     def role_admin(self):
         return False
@@ -276,6 +314,16 @@ class Anonymous(AnonymousUserMixin, UserBase):
     def is_authenticated(self):
         return False
 
+    def get_view_property(self, page, prop):
+        if not flask_session['view'].get(page):
+            return None
+        return flask_session['view'][page].get(prop)
+
+    def set_view_property(self, page, prop, value):
+        if not flask_session['view'].get(page):
+            flask_session['view'][page] = dict()
+        flask_session['view'][page][prop] = value
+
 
 # Baseclass representing Shelfs in calibre-web in app.db
 class Shelf(Base):
@@ -567,10 +615,11 @@ def migrate_Database(session):
             conn.execute("ALTER TABLE user ADD column `allowed_column_value` String DEFAULT ''")
         session.commit()
     try:
-        session.query(exists().where(User.series_view)).scalar()
+        session.query(exists().where(User.view_settings)).scalar()
     except exc.OperationalError:
         with engine.connect() as conn:
-            conn.execute("ALTER TABLE user ADD column `series_view` VARCHAR(10) DEFAULT 'list'")
+            conn.execute("ALTER TABLE user ADD column `view_settings` VARCHAR(10) DEFAULT '{}'")
+        session.commit()
 
     if session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() \
         is None:
@@ -591,14 +640,15 @@ def migrate_Database(session):
                      "locale VARCHAR(2),"
                      "sidebar_view INTEGER,"
                      "default_language VARCHAR(3),"
-                     "series_view VARCHAR(10),"
+                     # "series_view VARCHAR(10),"
+                     "view_settings VARCHAR,"                     
                      "UNIQUE (nickname),"
                      "UNIQUE (email))")
             conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,locale,"
-                     "sidebar_view, default_language, series_view) "
+                     "sidebar_view, default_language, view_settings) "
                      "SELECT id, nickname, email, role, password, kindle_mail, locale,"
                      "sidebar_view, default_language FROM user")
-        # delete old user table and rename new user_id table to user:
+            # delete old user table and rename new user_id table to user:
             conn.execute("DROP TABLE user")
             conn.execute("ALTER TABLE user_id RENAME TO user")
         session.commit()

cps/updater.py

@@ -227,6 +227,7 @@ class Updater(threading.Thread):
             os.sep + 'vendor', os.sep + 'calibre-web.log', os.sep + '.git', os.sep + 'client_secrets.json',
             os.sep + 'gdrive_credentials', os.sep + 'settings.yaml', os.sep + 'venv', os.sep + 'virtualenv',
             os.sep + 'access.log', os.sep + 'access.log1', os.sep + 'access.log2',
+            os.sep + '.calibre-web.log.swp'
         )
         additional_path = self.is_venv()
         if additional_path:

cps/web.py

@@ -30,17 +30,22 @@ import traceback
 import binascii
 import re
 
-from babel import Locale as LC
 from babel.dates import format_date
+from babel import Locale as LC
 from babel.core import UnknownLocaleError
-from flask import Blueprint
+from flask import Blueprint, jsonify
 from flask import render_template, request, redirect, send_from_directory, make_response, g, flash, abort, url_for
+from flask import session as flask_session
 from flask_babel import gettext as _
 from flask_login import login_user, logout_user, login_required, current_user, confirm_login
 from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError
 from sqlalchemy.sql.expression import text, func, true, false, not_, and_, or_
+from sqlalchemy.orm.attributes import flag_modified
 from werkzeug.exceptions import default_exceptions, InternalServerError
 from sqlalchemy.sql.functions import coalesce
+
+from .services.worker import WorkerThread
+
 try:
     from werkzeug.exceptions import FailedDependency
 except ImportError:
@@ -48,11 +53,11 @@ except ImportError:
 from werkzeug.datastructures import Headers
 from werkzeug.security import generate_password_hash, check_password_hash
 
-from . import constants, logger, isoLanguages, services, worker, cli
-from . import searched_ids, lm, babel, db, ub, config, get_locale, app
+from . import constants, logger, isoLanguages, services
+from . import lm, babel, db, ub, config, get_locale, app
 from . import calibre_db
 from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
-from .helper import check_valid_domain, render_task_status, json_serial, \
+from .helper import check_valid_domain, render_task_status, \
     get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \
     send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password
 from .pagination import Pagination
@@ -230,9 +235,8 @@ def admin_required(f):
 
 def unconfigured(f):
     """
-    Checks if current_user.role == 1
+    Checks if calibre-web instance is not configured
     """
-
     @wraps(f)
     def inner(*args, **kwargs):
         if not config.db_configured:
@@ -285,14 +289,6 @@ def edit_required(f):
 # ################################### Helper functions ################################################################
 
 
-# Returns the template for rendering and includes the instance name
-def render_title_template(*args, **kwargs):
-    sidebar = ub.get_sidebar_config(kwargs)
-    return render_template(instance=config.config_calibre_web_title, sidebar=sidebar,
-                           accept=constants.EXTENSIONS_UPLOAD,
-                           *args, **kwargs)
-
-
 @web.before_app_request
 def before_request():
     if current_user.is_authenticated:
@@ -384,12 +380,8 @@ def import_ldap_users():
 @web.route("/ajax/emailstat")
 @login_required
 def get_email_status_json():
-    tasks = worker.get_taskstatus()
-    answer = render_task_status(tasks)
-    js = json.dumps(answer, default=json_serial)
-    response = make_response(js)
-    response.headers["Content-Type"] = "application/json; charset=utf-8"
-    return response
+    tasks = WorkerThread.getInstance().tasks
+    return jsonify(render_task_status(tasks))
 
 
 @web.route("/ajax/bookmark/<int:book_id>/<book_format>", methods=['POST'])
@@ -472,22 +464,17 @@ def toggle_archived(book_id):
 
 
 @web.route("/ajax/view", methods=["POST"])
-@login_required
+@login_required_if_no_ano
 def update_view():
-    to_save = request.form.to_dict()
-    allowed_view = ['grid', 'list']
-    if "series_view" in to_save and to_save["series_view"] in allowed_view:
-        current_user.series_view = to_save["series_view"]
-    else:
-        log.error("Invalid request received: %r %r", request, to_save)
-        return "Invalid request", 400
-
+    to_save = request.get_json()
     try:
-        ub.session.commit()
-    except InvalidRequestError:
-        log.error("Invalid request received: %r ", request, )
+        for element in to_save:
+            for param in to_save[element]:
+                current_user.set_view_property(element, param, to_save[element][param])
+    except Exception as e:
+        log.error("Could not save view_settings: %r %r: e", request, to_save, e)
         return "Invalid request", 400
-    return "", 200
+    return "1", 200
 
 
 '''
@@ -611,25 +598,20 @@ def get_matching_tags():
     return json_dumps
 
 
-# ################################### View Books list ##################################################################
-
-
-@web.route("/", defaults={'page': 1})
-@web.route('/page/<int:page>')
-@login_required_if_no_ano
-def index(page):
-    entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, [db.Books.timestamp.desc()])
-    return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
-                                 title=_(u"Recently Added Books"), page="root")
+# Returns the template for rendering and includes the instance name
+def render_title_template(*args, **kwargs):
+    sidebar = ub.get_sidebar_config(kwargs)
+    return render_template(instance=config.config_calibre_web_title, sidebar=sidebar,
+                           accept=constants.EXTENSIONS_UPLOAD,
+                           *args, **kwargs)
 
 
-@web.route('/<data>/<sort>', defaults={'page': 1, 'book_id': "1"})
-@web.route('/<data>/<sort>/', defaults={'page': 1, 'book_id': "1"})
-@web.route('/<data>/<sort>/<book_id>', defaults={'page': 1})
-@web.route('/<data>/<sort>/<book_id>/<int:page>')
-@login_required_if_no_ano
-def books_list(data, sort, book_id, page):
+def render_books_list(data, sort, book_id, page):
     order = [db.Books.timestamp.desc()]
+    if sort == 'stored':
+        sort = current_user.get_view_property(data, 'stored')
+    else:
+        current_user.set_view_property(data, 'stored', sort)
     if sort == 'pubnew':
         order = [db.Books.pubdate.desc()]
     if sort == 'pubold':
@@ -645,7 +627,7 @@ def books_list(data, sort, book_id, page):
 
     if data == "rated":
         if current_user.check_visibility(constants.SIDEBAR_BEST_RATED):
-            entries, random, pagination = calibre_db.fill_indexpage(page,
+            entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                                     db.Books,
                                                                     db.Books.ratings.any(db.Ratings.rating > 9),
                                                                     order)
@@ -655,7 +637,7 @@ def books_list(data, sort, book_id, page):
             abort(404)
     elif data == "discover":
         if current_user.check_visibility(constants.SIDEBAR_RANDOM):
-            entries, __, pagination = calibre_db.fill_indexpage(page, db.Books, True, [func.randomblob(2)])
+            entries, __, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, [func.randomblob(2)])
             pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page)
             return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id,
                                          title=_(u"Discover (Random Books)"), page="discover")
@@ -667,6 +649,8 @@ def books_list(data, sort, book_id, page):
         return render_read_books(page, True, order=order)
     elif data == "hot":
         return render_hot_books(page)
+    elif data == "download":
+        return render_downloaded_books(page, order)
     elif data == "author":
         return render_author_books(page, book_id, order)
     elif data == "publisher":
@@ -683,10 +667,19 @@ def books_list(data, sort, book_id, page):
         return render_language_books(page, book_id, order)
     elif data == "archived":
         return render_archived_books(page, order)
+    elif data == "search":
+        term = (request.args.get('query') or '')
+        offset = int(int(config.config_books_per_page) * (page - 1))
+        return render_search_results(term, offset, order, config.config_books_per_page)
+    elif data == "advsearch":
+        term = json.loads(flask_session['query'])
+        offset = int(int(config.config_books_per_page) * (page - 1))
+        return render_adv_search_results(term, offset, order, config.config_books_per_page)
     else:
-        entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, order)
+        website = data or "newest"
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
-                                     title=_(u"Books"), page="newest")
+                                     title=_(u"Books"), page=website)
 
 
 def render_hot_books(page):
@@ -718,8 +711,44 @@ def render_hot_books(page):
         abort(404)
 
 
+def render_downloaded_books(page, order):
+    if current_user.check_visibility(constants.SIDEBAR_DOWNLOAD):
+        # order = order or []
+        if current_user.show_detail_random():
+            random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \
+                .order_by(func.random()).limit(config.config_random_books)
+        else:
+            random = false()
+        # off = int(int(config.config_books_per_page) * (page - 1))
+        '''entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db_filter,
+                                                                order,
+                                                                ub.ReadBook, db.Books.id==ub.ReadBook.book_id)'''
+
+        entries, __, pagination = calibre_db.fill_indexpage(page,
+                                                            0,
+                                                            db.Books,
+                                                            ub.Downloads.user_id == int(current_user.id),
+                                                            order,
+                                                            ub.Downloads, db.Books.id == ub.Downloads.book_id)
+        for book in entries:
+            if not calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \
+                             .filter(db.Books.id == book.id).first():
+                ub.delete_download(book.id)
+
+        return render_title_template('index.html',
+                                     random=random,
+                                     entries=entries,
+                                     pagination=pagination,
+                                     title=_(u"Downloaded books by %(user)s",user=current_user.nickname),
+                                     page="download")
+    else:
+        abort(404)
+
+
 def render_author_books(page, author_id, order):
-    entries, __, pagination = calibre_db.fill_indexpage(page,
+    entries, __, pagination = calibre_db.fill_indexpage(page, 0,
                                                         db.Books,
                                                         db.Books.authors.any(db.Authors.id == author_id),
                                                         [order[0], db.Series.name, db.Books.series_index],
@@ -747,7 +776,7 @@ def render_author_books(page, author_id, order):
 def render_publisher_books(page, book_id, order):
     publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
     if publisher:
-        entries, random, pagination = calibre_db.fill_indexpage(page,
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                                 db.Books,
                                                                 db.Books.publishers.any(db.Publishers.id == book_id),
                                                                 [db.Series.name, order[0], db.Books.series_index],
@@ -762,10 +791,10 @@ def render_publisher_books(page, book_id, order):
 def render_series_books(page, book_id, order):
     name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
     if name:
-        entries, random, pagination = calibre_db.fill_indexpage(page,
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                                 db.Books,
                                                                 db.Books.series.any(db.Series.id == book_id),
-                                                                [db.Books.series_index, order[0]])
+                                                                [order[0]])
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
                                      title=_(u"Series: %(serie)s", serie=name.name), page="series")
     else:
@@ -774,7 +803,7 @@ def render_series_books(page, book_id, order):
 
 def render_ratings_books(page, book_id, order):
     name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
-    entries, random, pagination = calibre_db.fill_indexpage(page,
+    entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                             db.Books,
                                                             db.Books.ratings.any(db.Ratings.id == book_id),
                                                             [db.Books.timestamp.desc(), order[0]])
@@ -788,7 +817,7 @@ def render_ratings_books(page, book_id, order):
 def render_formats_books(page, book_id, order):
     name = calibre_db.session.query(db.Data).filter(db.Data.format == book_id.upper()).first()
     if name:
-        entries, random, pagination = calibre_db.fill_indexpage(page,
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                                 db.Books,
                                                                 db.Books.data.any(db.Data.format == book_id.upper()),
                                                                 [db.Books.timestamp.desc(), order[0]])
@@ -801,7 +830,7 @@ def render_formats_books(page, book_id, order):
 def render_category_books(page, book_id, order):
     name = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
     if name:
-        entries, random, pagination = calibre_db.fill_indexpage(page,
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                                 db.Books,
                                                                 db.Books.tags.any(db.Tags.id == book_id),
                                                                 [order[0], db.Series.name, db.Books.series_index],
@@ -821,27 +850,210 @@ def render_language_books(page, name, order):
             lang_name = _(isoLanguages.get(part3=name).name)
         except KeyError:
             abort(404)
-    entries, random, pagination = calibre_db.fill_indexpage(page,
+    entries, random, pagination = calibre_db.fill_indexpage(page, 0,
                                                             db.Books,
                                                             db.Books.languages.any(db.Languages.lang_code == name),
                                                             [db.Books.timestamp.desc(), order[0]])
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
                                  title=_(u"Language: %(name)s", name=lang_name), page="language")
 
+def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs):
+    order = order or []
+    if not config.config_read_column:
+        if are_read:
+            db_filter = and_(ub.ReadBook.user_id == int(current_user.id),
+                             ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED)
+        else:
+            db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED
+        entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                db.Books,
+                                                                db_filter,
+                                                                order,
+                                                                ub.ReadBook, db.Books.id==ub.ReadBook.book_id)
+    else:
+        try:
+            if are_read:
+                db_filter = db.cc_classes[config.config_read_column].value == True
+            else:
+                db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True
+            entries, random, pagination = calibre_db.fill_indexpage(page, 0,
+                                                                    db.Books,
+                                                                    db_filter,
+                                                                    order,
+                                                                    db.cc_classes[config.config_read_column])
+        except (KeyError, AttributeError):
+            log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column)
+            if not as_xml:
+                flash(_("Custom Column No.%(column)d is not existing in calibre database",
+                        column=config.config_read_column),
+                      category="error")
+                return redirect(url_for("web.index"))
+            # ToDo: Handle error Case for opds
+    if as_xml:
+        return entries, pagination
+    else:
+        if are_read:
+            name = _(u'Read Books') + ' (' + str(pagination.total_count) + ')'
+            pagename = "read"
+        else:
+            name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')'
+            pagename = "unread"
+        return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
+                                     title=name, page=pagename)
+
+
+def render_archived_books(page, order):
+    order = order or []
+    archived_books = (
+        ub.session.query(ub.ArchivedBook)
+        .filter(ub.ArchivedBook.user_id == int(current_user.id))
+        .filter(ub.ArchivedBook.is_archived == True)
+        .all()
+    )
+    archived_book_ids = [archived_book.book_id for archived_book in archived_books]
+
+    archived_filter = db.Books.id.in_(archived_book_ids)
+
+    entries, random, pagination = calibre_db.fill_indexpage_with_archived_books(page, 0,
+                                                                                db.Books,
+                                                                                archived_filter,
+                                                                                order,
+                                                                                allow_show_archived=True)
+
+    name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')'
+    pagename = "archived"
+    return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
+                                 title=name, page=pagename)
+
+
+def render_prepare_search_form(cc):
+    # prepare data for search-form
+    tags = calibre_db.session.query(db.Tags)\
+        .join(db.books_tags_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(text('books_tags_link.tag'))\
+        .order_by(db.Tags.name).all()
+    series = calibre_db.session.query(db.Series)\
+        .join(db.books_series_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(text('books_series_link.series'))\
+        .order_by(db.Series.name)\
+        .filter(calibre_db.common_filters()).all()
+    extensions = calibre_db.session.query(db.Data)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(db.Data.format)\
+        .order_by(db.Data.format).all()
+    if current_user.filter_language() == u"all":
+        languages = calibre_db.speaking_language()
+    else:
+        languages = None
+    return render_title_template('search_form.html', tags=tags, languages=languages, extensions=extensions,
+                                 series=series, title=_(u"search"), cc=cc, page="advsearch")
+
+
+def render_search_results(term, offset=None, order=None, limit=None):
+    entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit)
+    return render_title_template('search.html',
+                                 searchterm=term,
+                                 pagination=pagination,
+                                 query=term,
+                                 adv_searchterm=term,
+                                 entries=entries,
+                                 result_count=result_count,
+                                 title=_(u"Search"),
+                                 page="search")
 
-'''@web.route("/table")
+
+# ################################### View Books list ##################################################################
+
+
+@web.route("/", defaults={'page': 1})
+@web.route('/page/<int:page>')
+@login_required_if_no_ano
+def index(page):
+    sort_param = (request.args.get('sort') or 'stored').lower()
+    return render_books_list("newest", sort_param, 1, page)
+
+
+@web.route('/<data>/<sort_param>', defaults={'page': 1, 'book_id': "1"})
+@web.route('/<data>/<sort_param>/', defaults={'page': 1, 'book_id': "1"})
+@web.route('/<data>/<sort_param>/<book_id>', defaults={'page': 1})
+@web.route('/<data>/<sort_param>/<book_id>/<int:page>')
 @login_required_if_no_ano
+def books_list(data, sort_param, book_id, page):
+    return render_books_list(data, sort_param, book_id, page)
+
+
+@web.route("/table")
+@login_required
 def books_table():
-    return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
-                                 title=_(u"Language: %(name)s", name=lang_name), page="language")'''
+    visibility = current_user.view_settings.get('table', {})
+    return render_title_template('book_table.html', title=_(u"Books list"), page="book_table",
+                                 visiblility=visibility)
+
+@web.route("/ajax/listbooks")
+@login_required
+def list_books():
+    off = request.args.get("offset") or 0
+    limit = request.args.get("limit") or config.config_books_per_page
+    # sort = request.args.get("sort")
+    if request.args.get("order") == 'desc':
+        order = [db.Books.timestamp.desc()]
+    else:
+        order = [db.Books.timestamp.asc()]
+    search = request.args.get("search")
+    total_count = calibre_db.session.query(db.Books).count()
+    if search:
+        entries, filtered_count, pagination = calibre_db.get_search_results(search, off, order, limit)
+    else:
+        entries, __, __ = calibre_db.fill_indexpage((int(off) / (int(limit)) + 1), limit, db.Books, True, order)
+        filtered_count = total_count
+    for entry in entries:
+        for index in range(0, len(entry.languages)):
+            try:
+                entry.languages[index].language_name = LC.parse(entry.languages[index].lang_code)\
+                    .get_language_name(get_locale())
+            except UnknownLocaleError:
+                entry.languages[index].language_name = _(
+                    isoLanguages.get(part3=entry.languages[index].lang_code).name)
+    table_entries = {'totalNotFiltered': total_count, 'total': filtered_count, "rows": entries}
+    js_list = json.dumps(table_entries, cls=db.AlchemyEncoder)
+
+    response = make_response(js_list)
+    response.headers["Content-Type"] = "application/json; charset=utf-8"
+    return response
+
+@web.route("/ajax/table_settings", methods=['POST'])
+@login_required
+def update_table_settings():
+    # vals = request.get_json()
+    # ToDo: Save table settings
+    current_user.view_settings['table'] = json.loads(request.data)
+    try:
+        try:
+            flag_modified(current_user, "view_settings")
+        except AttributeError:
+            pass
+        ub.session.commit()
+    except InvalidRequestError:
+        log.error("Invalid request received: %r ", request, )
+        return "Invalid request", 400
+    return ""
 
 @web.route("/author")
 @login_required_if_no_ano
 def author_list():
     if current_user.check_visibility(constants.SIDEBAR_AUTHOR):
+        if current_user.get_view_property('author', 'dir') == 'desc':
+            order = db.Authors.sort.desc()
+        else:
+            order = db.Authors.sort.asc()
         entries = calibre_db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \
             .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
-            .group_by(text('books_authors_link.author')).order_by(db.Authors.sort).all()
+            .group_by(text('books_authors_link.author')).order_by(order).all()
         charlist = calibre_db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('char')) \
             .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Authors.sort, 1, 1))).all()
@@ -856,10 +1068,14 @@ def author_list():
 @web.route("/publisher")
 @login_required_if_no_ano
 def publisher_list():
+    if current_user.get_view_property('publisher', 'dir') == 'desc':
+        order = db.Publishers.name.desc()
+    else:
+        order = db.Publishers.name.asc()
     if current_user.check_visibility(constants.SIDEBAR_PUBLISHER):
         entries = calibre_db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \
             .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
-            .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.name).all()
+            .group_by(text('books_publishers_link.publisher')).order_by(order).all()
         charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \
             .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all()
@@ -873,10 +1089,14 @@ def publisher_list():
 @login_required_if_no_ano
 def series_list():
     if current_user.check_visibility(constants.SIDEBAR_SERIES):
-        if current_user.series_view == 'list':
+        if current_user.get_view_property('series', 'dir') == 'desc':
+            order = db.Series.sort.desc()
+        else:
+            order = db.Series.sort.asc()
+        if current_user.get_view_property('series', 'series_view') == 'list':
             entries = calibre_db.session.query(db.Series, func.count('books_series_link.book').label('count')) \
                 .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
-                .group_by(text('books_series_link.series')).order_by(db.Series.sort).all()
+                .group_by(text('books_series_link.series')).order_by(order).all()
             charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
                 .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
@@ -885,7 +1105,7 @@ def series_list():
         else:
             entries = calibre_db.session.query(db.Books, func.count('books_series_link').label('count')) \
                 .join(db.books_series_link).join(db.Series).filter(calibre_db.common_filters()) \
-                .group_by(text('books_series_link.series')).order_by(db.Series.sort).all()
+                .group_by(text('books_series_link.series')).order_by(order).all()
             charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
                 .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
@@ -900,10 +1120,14 @@ def series_list():
 @login_required_if_no_ano
 def ratings_list():
     if current_user.check_visibility(constants.SIDEBAR_RATING):
+        if current_user.get_view_property('ratings', 'dir') == 'desc':
+            order = db.Ratings.rating.desc()
+        else:
+            order = db.Ratings.rating.asc()
         entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
                                    (db.Ratings.rating / 2).label('name')) \
             .join(db.books_ratings_link).join(db.Books).filter(calibre_db.common_filters()) \
-            .group_by(text('books_ratings_link.rating')).order_by(db.Ratings.rating).all()
+            .group_by(text('books_ratings_link.rating')).order_by(order).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"Ratings list"), page="ratingslist", data="ratings")
     else:
@@ -914,11 +1138,15 @@ def ratings_list():
 @login_required_if_no_ano
 def formats_list():
     if current_user.check_visibility(constants.SIDEBAR_FORMAT):
+        if current_user.get_view_property('ratings', 'dir') == 'desc':
+            order = db.Data.format.desc()
+        else:
+            order = db.Data.format.asc()
         entries = calibre_db.session.query(db.Data,
                                            func.count('data.book').label('count'),
                                            db.Data.format.label('format')) \
             .join(db.Books).filter(calibre_db.common_filters()) \
-            .group_by(db.Data.format).order_by(db.Data.format).all()
+            .group_by(db.Data.format).order_by(order).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"File formats list"), page="formatslist", data="formats")
     else:
@@ -958,8 +1186,12 @@ def language_overview():
 @login_required_if_no_ano
 def category_list():
     if current_user.check_visibility(constants.SIDEBAR_CATEGORY):
+        if current_user.get_view_property('category', 'dir') == 'desc':
+            order = db.Tags.name.desc()
+        else:
+            order = db.Tags.name.asc()
         entries = calibre_db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \
-            .join(db.books_tags_link).join(db.Books).order_by(db.Tags.name).filter(calibre_db.common_filters()) \
+            .join(db.books_tags_link).join(db.Books).order_by(order).filter(calibre_db.common_filters()) \
             .group_by(text('books_tags_link.tag')).all()
         charlist = calibre_db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('char')) \
             .join(db.books_tags_link).join(db.Books).filter(calibre_db.common_filters()) \
@@ -977,7 +1209,7 @@ def category_list():
 @login_required
 def get_tasks_status():
     # if current user admin, show all email, otherwise only own emails
-    tasks = worker.get_taskstatus()
+    tasks = WorkerThread.getInstance().tasks
     answer = render_task_status(tasks)
     return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks")
 
@@ -990,55 +1222,51 @@ def reconnect():
 
 # ################################### Search functions ################################################################
 
-
 @web.route("/search", methods=["GET"])
 @login_required_if_no_ano
 def search():
     term = request.args.get("query")
     if term:
-        entries = calibre_db.get_search_results(term)
-        ids = list()
-        for element in entries:
-            ids.append(element.id)
-        searched_ids[current_user.id] = ids
-        return render_title_template('search.html',
-                                     searchterm=term,
-                                     adv_searchterm=term,
-                                     entries=entries,
-                                     title=_(u"Search"),
-                                     page="search")
+        return render_search_results(term, 0, None, config.config_books_per_page)
     else:
         return render_title_template('search.html',
                                      searchterm="",
+                                     result_count=0,
                                      title=_(u"Search"),
                                      page="search")
 
 
-@web.route("/advanced_search", methods=['GET'])
+@web.route("/advanced_search", methods=['POST'])
 @login_required_if_no_ano
 def advanced_search():
-    # Build custom columns names
+    term = request.form
+    return render_adv_search_results(term, 0, None, config.config_books_per_page)
+
+def render_adv_search_results(term, offset=None, order=None, limit=None):
+    order = order or [db.Books.sort]
+    pagination = None
+
     cc = get_cc_columns(filter_config_custom_read=True)
     calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
-    q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True)).order_by(db.Books.sort)
-
-    include_tag_inputs = request.args.getlist('include_tag')
-    exclude_tag_inputs = request.args.getlist('exclude_tag')
-    include_series_inputs = request.args.getlist('include_serie')
-    exclude_series_inputs = request.args.getlist('exclude_serie')
-    include_languages_inputs = request.args.getlist('include_language')
-    exclude_languages_inputs = request.args.getlist('exclude_language')
-    include_extension_inputs = request.args.getlist('include_extension')
-    exclude_extension_inputs = request.args.getlist('exclude_extension')
-
-    author_name = request.args.get("author_name")
-    book_title = request.args.get("book_title")
-    publisher = request.args.get("publisher")
-    pub_start = request.args.get("Publishstart")
-    pub_end = request.args.get("Publishend")
-    rating_low = request.args.get("ratinghigh")
-    rating_high = request.args.get("ratinglow")
-    description = request.args.get("comment")
+    q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True))
+
+    include_tag_inputs = request.form.getlist('include_tag')
+    exclude_tag_inputs = request.form.getlist('exclude_tag')
+    include_series_inputs = request.form.getlist('include_serie')
+    exclude_series_inputs = request.form.getlist('exclude_serie')
+    include_languages_inputs = request.form.getlist('include_language')
+    exclude_languages_inputs = request.form.getlist('exclude_language')
+    include_extension_inputs = request.form.getlist('include_extension')
+    exclude_extension_inputs = request.form.getlist('exclude_extension')
+
+    author_name = term.get("author_name")
+    book_title = term.get("book_title")
+    publisher = term.get("publisher")
+    pub_start = term.get("Publishstart")
+    pub_end = term.get("Publishend")
+    rating_low = term.get("ratinghigh")
+    rating_high = term.get("ratinglow")
+    description = term.get("comment")
     if author_name:
         author_name = author_name.strip().lower().replace(',', '|')
     if book_title:
@@ -1049,8 +1277,8 @@ def advanced_search():
     searchterm = []
     cc_present = False
     for c in cc:
-        if request.args.get('custom_column_' + str(c.id)):
-            searchterm.extend([(u"%s: %s" % (c.name, request.args.get('custom_column_' + str(c.id))))])
+        if request.form.get('custom_column_' + str(c.id)):
+            searchterm.extend([(u"%s: %s" % (c.name, request.form.get('custom_column_' + str(c.id))))])
             cc_present = True
 
     if include_tag_inputs or exclude_tag_inputs or include_series_inputs or exclude_series_inputs or \
@@ -1089,8 +1317,8 @@ def advanced_search():
         searchterm.extend(ext for ext in exclude_extension_inputs)
         # handle custom columns
         for c in cc:
-            if request.args.get('custom_column_' + str(c.id)):
-                searchterm.extend([(u"%s: %s" % (c.name, request.args.get('custom_column_' + str(c.id))))])
+            if request.form.get('custom_column_' + str(c.id)):
+                searchterm.extend([(u"%s: %s" % (c.name, request.form.get('custom_column_' + str(c.id))))])
         searchterm = " + ".join(filter(None, searchterm))
         q = q.filter()
         if author_name:
@@ -1133,7 +1361,7 @@ def advanced_search():
 
         # search custom culumns
         for c in cc:
-            custom_query = request.args.get('custom_column_' + str(c.id))
+            custom_query = request.form.get('custom_column_' + str(c.id))
             if custom_query != '' and custom_query is not None:
                 if c.datatype == 'bool':
                     q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
@@ -1147,107 +1375,34 @@ def advanced_search():
                 else:
                     q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
                         func.lower(db.cc_classes[c.id].value).ilike("%" + custom_query + "%")))
-        q = q.all()
-        ids = list()
-        for element in q:
-            ids.append(element.id)
-        searched_ids[current_user.id] = ids
-        return render_title_template('search.html', adv_searchterm=searchterm,
-                                     entries=q, title=_(u"search"), page="search")
-    # prepare data for search-form
-    tags = calibre_db.session.query(db.Tags)\
-        .join(db.books_tags_link)\
-        .join(db.Books)\
-        .filter(calibre_db.common_filters()) \
-        .group_by(text('books_tags_link.tag'))\
-        .order_by(db.Tags.name).all()
-    series = calibre_db.session.query(db.Series)\
-        .join(db.books_series_link)\
-        .join(db.Books)\
-        .filter(calibre_db.common_filters()) \
-        .group_by(text('books_series_link.series'))\
-        .order_by(db.Series.name)\
-        .filter(calibre_db.common_filters()).all()
-    extensions = calibre_db.session.query(db.Data)\
-        .join(db.Books)\
-        .filter(calibre_db.common_filters()) \
-        .group_by(db.Data.format)\
-        .order_by(db.Data.format).all()
-    if current_user.filter_language() == u"all":
-        languages = calibre_db.speaking_language()
-    else:
-        languages = None
-    return render_title_template('search_form.html', tags=tags, languages=languages, extensions=extensions,
-                                 series=series, title=_(u"search"), cc=cc, page="advsearch")
-
-
-def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs):
-    order = order or []
-    if not config.config_read_column:
-        if are_read:
-            db_filter = and_(ub.ReadBook.user_id == int(current_user.id),
-                             ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED)
-        else:
-            db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED
-        entries, random, pagination = calibre_db.fill_indexpage(page,
-                                                                db.Books,
-                                                                db_filter,
-                                                                order,
-                                                                ub.ReadBook, db.Books.id==ub.ReadBook.book_id)
-    else:
-        try:
-            if are_read:
-                db_filter = db.cc_classes[config.config_read_column].value == True
-            else:
-                db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True
-            entries, random, pagination = calibre_db.fill_indexpage(page,
-                                                                    db.Books,
-                                                                    db_filter,
-                                                                    order,
-                                                                    db.cc_classes[config.config_read_column])
-        except (KeyError, AttributeError):
-            log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column)
-            if not as_xml:
-                flash(_("Custom Column No.%(column)d is not existing in calibre database",
-                        column=config.config_read_column),
-                      category="error")
-                return redirect(url_for("web.index"))
-            # ToDo: Handle error Case for opds
-    if as_xml:
-        return entries, pagination
-    else:
-        if are_read:
-            name = _(u'Read Books') + ' (' + str(pagination.total_count) + ')'
-            pagename = "read"
+        q = q.order_by(*order).all()
+        flask_session['query'] = json.dumps(term)
+        ub.store_ids(q)
+        # entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit)
+        result_count = len(q)
+        if offset != None and limit != None:
+            offset = int(offset)
+            limit_all = offset + int(limit)
+            pagination = Pagination((offset / (int(limit)) + 1), limit, result_count)
         else:
-            name = _(u'Unread Books') + ' (' + str(pagination.total_count) + ')'
-            pagename = "unread"
-        return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
-                                     title=name, page=pagename)
+            offset = 0
+            limit_all = result_count
+    return render_title_template('search.html',
+                                 adv_searchterm=searchterm,
+                                 pagination=pagination,
+                                 entries=q[offset:limit_all],
+                                 result_count=result_count,
+                                 title=_(u"search"), page="advsearch")
 
 
-def render_archived_books(page, order):
-    order = order or []
-    archived_books = (
-        ub.session.query(ub.ArchivedBook)
-        .filter(ub.ArchivedBook.user_id == int(current_user.id))
-        .filter(ub.ArchivedBook.is_archived == True)
-        .all()
-    )
-    archived_book_ids = [archived_book.book_id for archived_book in archived_books]
-
-    archived_filter = db.Books.id.in_(archived_book_ids)
 
-    entries, random, pagination = calibre_db.fill_indexpage_with_archived_books(page,
-                                                                                db.Books,
-                                                                                archived_filter,
-                                                                                order,
-                                                                                allow_show_archived=True)
+@web.route("/advanced_search", methods=['GET'])
+@login_required_if_no_ano
+def advanced_search_form():
+    # Build custom columns names
+    cc = get_cc_columns(filter_config_custom_read=True)
+    return render_prepare_search_form(cc)
 
-    name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')'
-    pagename = "archived"
-    return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
-                                 title=name, page=pagename)
 
 # ################################### Download/Send ##################################################################
 
@@ -1551,21 +1706,24 @@ def token_verified():
 @web.route("/me", methods=["GET", "POST"])
 @login_required
 def profile():
-    downloads = list()
+    # downloads = list()
     languages = calibre_db.speaking_language()
     translations = babel.list_translations() + [LC('en')]
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
-    if feature_support['oauth']:
+    if feature_support['oauth'] and config.config_login_type == 2:
         oauth_status = get_oauth_status()
+        local_oauth_check = oauth_check
     else:
         oauth_status = None
+        local_oauth_check = {}
+
+    '''entries, __, pagination = calibre_db.fill_indexpage(page,
+                                                        0,
+                                                        db.Books,
+                                                        ub.Downloads.user_id == int(current_user.id), # True,
+                                                        [],
+                                                        ub.Downloads, db.Books.id == ub.Downloads.book_id)'''
 
-    for book in current_user.downloads:
-        downloadBook = calibre_db.get_book(book.book_id)
-        if downloadBook:
-            downloads.append(downloadBook)
-        else:
-            ub.delete_download(book.book_id)
     if request.method == "POST":
         to_save = request.form.to_dict()
         current_user.random_books = 0
@@ -1579,10 +1737,11 @@ def profile():
         if "email" in to_save and to_save["email"] != current_user.email:
             if config.config_public_reg and not check_valid_domain(to_save["email"]):
                 flash(_(u"E-mail is not from valid domain"), category="error")
-                return render_title_template("user_edit.html", content=current_user, downloads=downloads,
+                return render_title_template("user_edit.html", content=current_user,
                                              title=_(u"%(name)s's profile", name=current_user.nickname), page="me",
                                              kobo_support=kobo_support,
-                                             registered_oauth=oauth_check, oauth_status=oauth_status)
+                                             registered_oauth=local_oauth_check, oauth_status=oauth_status)
+            current_user.email = to_save["email"]
         if "nickname" in to_save and to_save["nickname"] != current_user.nickname:
             # Query User nickname, if not existing, change
             if not ub.session.query(ub.User).filter(ub.User.nickname == to_save["nickname"]).scalar():
@@ -1594,12 +1753,10 @@ def profile():
                                              languages=languages,
                                              kobo_support=kobo_support,
                                              new_user=0, content=current_user,
-                                             downloads=downloads,
-                                             registered_oauth=oauth_check,
+                                             registered_oauth=local_oauth_check,
                                              title=_(u"Edit User %(nick)s",
                                                      nick=current_user.nickname),
                                              page="edituser")
-            current_user.email = to_save["email"]
         if "show_random" in to_save and to_save["show_random"] == "on":
             current_user.random_books = 1
         if "default_language" in to_save:
@@ -1615,24 +1772,32 @@ def profile():
         if "Show_detail_random" in to_save:
             current_user.sidebar_view += constants.DETAIL_RANDOM
 
-        # current_user.mature_content = "Show_mature_content" in to_save
-
         try:
             ub.session.commit()
+            flash(_(u"Profile updated"), category="success")
+            log.debug(u"Profile updated")
         except IntegrityError:
             ub.session.rollback()
             flash(_(u"Found an existing account for this e-mail address."), category="error")
             log.debug(u"Found an existing account for this e-mail address.")
-            return render_title_template("user_edit.html", content=current_user, downloads=downloads,
-                                         translations=translations, kobo_support=kobo_support,
-                                         title=_(u"%(name)s's profile", name=current_user.nickname), page="me",
-                                         registered_oauth=oauth_check, oauth_status=oauth_status)
-        flash(_(u"Profile updated"), category="success")
-        log.debug(u"Profile updated")
-    return render_title_template("user_edit.html", translations=translations, profile=1, languages=languages,
-                                 content=current_user, downloads=downloads, kobo_support=kobo_support,
+            '''return render_title_template("user_edit.html",
+                                         content=current_user,
+                                         translations=translations,
+                                         kobo_support=kobo_support,
+                                         title=_(u"%(name)s's profile", name=current_user.nickname),
+                                         page="me",
+                                         registered_oauth=local_oauth_check,
+                                         oauth_status=oauth_status)'''
+    return render_title_template("user_edit.html",
+                                 translations=translations,
+                                 profile=1,
+                                 languages=languages,
+                                 content=current_user,
+                                 kobo_support=kobo_support,
                                  title=_(u"%(name)s's profile", name=current_user.nickname),
-                                 page="me", registered_oauth=oauth_check, oauth_status=oauth_status)
+                                 page="me",
+                                 registered_oauth=local_oauth_check,
+                                 oauth_status=oauth_status)
 
 
 # ###################################Show single book ##################################################################

cps/worker.py

@@ -1,602 +0,0 @@
-# -*- coding: utf-8 -*-
-
-#  This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
-#    Copyright (C) 2018-2019 OzzieIsaacs, bodybybuddha, janeczku
-#
-#  This program is free software: you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation, either version 3 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-from __future__ import division, print_function, unicode_literals
-import sys
-import os
-import re
-import smtplib
-import socket
-import time
-import threading
-try:
-    import queue
-except ImportError:
-    import Queue as queue
-from glob import glob
-from shutil import copyfile
-from datetime import datetime
-
-try:
-    from StringIO import StringIO
-    from email.MIMEBase import MIMEBase
-    from email.MIMEMultipart import MIMEMultipart
-    from email.MIMEText import MIMEText
-except ImportError:
-    from io import StringIO
-    from email.mime.base import MIMEBase
-    from email.mime.multipart import MIMEMultipart
-    from email.mime.text import MIMEText
-
-from email import encoders
-from email.utils import formatdate
-from email.utils import make_msgid
-from email.generator import Generator
-from flask_babel import gettext as _
-
-from . import calibre_db, db
-from . import logger, config
-from .subproc_wrapper import process_open
-from . import gdriveutils
-
-log = logger.create()
-
-chunksize = 8192
-# task 'status' consts
-STAT_WAITING = 0
-STAT_FAIL = 1
-STAT_STARTED = 2
-STAT_FINISH_SUCCESS = 3
-#taskType consts
-TASK_EMAIL = 1
-TASK_CONVERT = 2
-TASK_UPLOAD = 3
-TASK_CONVERT_ANY = 4
-
-RET_FAIL = 0
-RET_SUCCESS = 1
-
-
-def _get_main_thread():
-    for t in threading.enumerate():
-        if t.__class__.__name__ == '_MainThread':
-            return t
-    raise Exception("main thread not found?!")
-
-
-# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
-# it in MIME Base64 encoded to
-def get_attachment(bookpath, filename):
-    """Get file as MIMEBase message"""
-    calibrepath = config.config_calibre_dir
-    if config.config_use_google_drive:
-        df = gdriveutils.getFileFromEbooksFolder(bookpath, filename)
-        if df:
-            datafile = os.path.join(calibrepath, bookpath, filename)
-            if not os.path.exists(os.path.join(calibrepath, bookpath)):
-                os.makedirs(os.path.join(calibrepath, bookpath))
-            df.GetContentFile(datafile)
-        else:
-            return None
-        file_ = open(datafile, 'rb')
-        data = file_.read()
-        file_.close()
-        os.remove(datafile)
-    else:
-        try:
-            file_ = open(os.path.join(calibrepath, bookpath, filename), 'rb')
-            data = file_.read()
-            file_.close()
-        except IOError as e:
-            log.exception(e)
-            log.error(u'The requested file could not be read. Maybe wrong permissions?')
-            return None
-
-    attachment = MIMEBase('application', 'octet-stream')
-    attachment.set_payload(data)
-    encoders.encode_base64(attachment)
-    attachment.add_header('Content-Disposition', 'attachment',
-                          filename=filename)
-    return attachment
-
-
-# Class for sending email with ability to get current progress
-class emailbase():
-
-    transferSize = 0
-    progress = 0
-
-    def data(self, msg):
-        self.transferSize = len(msg)
-        (code, resp) = smtplib.SMTP.data(self, msg)
-        self.progress = 0
-        return (code, resp)
-
-    def send(self, strg):
-        """Send `strg' to the server."""
-        log.debug('send: %r', strg[:300])
-        if hasattr(self, 'sock') and self.sock:
-            try:
-                if self.transferSize:
-                    lock=threading.Lock()
-                    lock.acquire()
-                    self.transferSize = len(strg)
-                    lock.release()
-                    for i in range(0, self.transferSize, chunksize):
-                        if isinstance(strg, bytes):
-                            self.sock.send((strg[i:i+chunksize]))
-                        else:
-                            self.sock.send((strg[i:i + chunksize]).encode('utf-8'))
-                        lock.acquire()
-                        self.progress = i
-                        lock.release()
-                else:
-                    self.sock.sendall(strg.encode('utf-8'))
-            except socket.error:
-                self.close()
-                raise smtplib.SMTPServerDisconnected('Server not connected')
-        else:
-            raise smtplib.SMTPServerDisconnected('please run connect() first')
-
-    @classmethod
-    def _print_debug(self, *args):
-        log.debug(args)
-
-    def getTransferStatus(self):
-        if self.transferSize:
-            lock2 = threading.Lock()
-            lock2.acquire()
-            value = int((float(self.progress) / float(self.transferSize))*100)
-            lock2.release()
-            return str(value) + ' %'
-        else:
-            return "100 %"
-
-
-# Class for sending email with ability to get current progress, derived from emailbase class
-class email(emailbase, smtplib.SMTP):
-
-    def __init__(self, *args, **kwargs):
-        smtplib.SMTP.__init__(self, *args, **kwargs)
-
-
-# Class for sending ssl encrypted email with ability to get current progress, , derived from emailbase class
-class email_SSL(emailbase, smtplib.SMTP_SSL):
-
-    def __init__(self, *args, **kwargs):
-        smtplib.SMTP_SSL.__init__(self, *args, **kwargs)
-
-
-#Class for all worker tasks in the background
-class WorkerThread(threading.Thread):
-
-    def __init__(self):
-        threading.Thread.__init__(self)
-        self.status = 0
-        self.current = 0
-        self.last = 0
-        self.queue = list()
-        self.UIqueue = list()
-        self.asyncSMTP = None
-        self.id = 0
-        self.db_queue = queue.Queue()
-        calibre_db.add_queue(self.db_queue)
-        self.doLock = threading.Lock()
-
-    # Main thread loop starting the different tasks
-    def run(self):
-        main_thread = _get_main_thread()
-        while main_thread.is_alive():
-            try:
-                self.doLock.acquire()
-                if self.current != self.last:
-                    index = self.current
-                    log.info(index)
-                    log.info(len(self.queue))
-                    self.doLock.release()
-                    if self.queue[index]['taskType'] == TASK_EMAIL:
-                        self._send_raw_email()
-                    elif self.queue[index]['taskType'] in (TASK_CONVERT, TASK_CONVERT_ANY):
-                        self._convert_any_format()
-                    # TASK_UPLOAD is handled implicitly
-                    self.doLock.acquire()
-                    self.current += 1
-                    if self.current > self.last:
-                        self.current = self.last
-                    self.doLock.release()
-                else:
-                    self.doLock.release()
-            except Exception as e:
-                log.exception(e)
-                self.doLock.release()
-            if main_thread.is_alive():
-                time.sleep(1)
-
-    def get_send_status(self):
-        if self.asyncSMTP:
-            return self.asyncSMTP.getTransferStatus()
-        else:
-            return "0 %"
-
-    def _delete_completed_tasks(self):
-        for index, task in reversed(list(enumerate(self.UIqueue))):
-            if task['progress'] == "100 %":
-                # delete tasks
-                self.queue.pop(index)
-                self.UIqueue.pop(index)
-                # if we are deleting entries before the current index, adjust the index
-                if index <= self.current and self.current:
-                    self.current -= 1
-        self.last = len(self.queue)
-
-    def get_taskstatus(self):
-        self.doLock.acquire()
-        if self.current  < len(self.queue):
-            if self.UIqueue[self.current]['stat'] == STAT_STARTED:
-                if self.queue[self.current]['taskType'] == TASK_EMAIL:
-                    self.UIqueue[self.current]['progress'] = self.get_send_status()
-                self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime']
-                self.UIqueue[self.current]['rt'] = self.UIqueue[self.current]['formRuntime'].days*24*60 \
-                                                   + self.UIqueue[self.current]['formRuntime'].seconds \
-                                                   + self.UIqueue[self.current]['formRuntime'].microseconds
-        self.doLock.release()
-        return self.UIqueue
-
-    def _convert_any_format(self):
-        # convert book, and upload in case of google drive
-        self.doLock.acquire()
-        index = self.current
-        self.doLock.release()
-        self.UIqueue[index]['stat'] = STAT_STARTED
-        self.queue[index]['starttime'] = datetime.now()
-        self.UIqueue[index]['formStarttime'] = self.queue[index]['starttime']
-        curr_task = self.queue[index]['taskType']
-        filename = self._convert_ebook_format()
-        if filename:
-            if config.config_use_google_drive:
-                gdriveutils.updateGdriveCalibreFromLocal()
-            if curr_task == TASK_CONVERT:
-                self.add_email(self.queue[index]['settings']['subject'], self.queue[index]['path'],
-                                filename, self.queue[index]['settings'], self.queue[index]['kindle'],
-                                self.UIqueue[index]['user'], self.queue[index]['title'],
-                                self.queue[index]['settings']['body'], internal=True)
-
-
-    def _convert_ebook_format(self):
-        error_message = None
-        self.doLock.acquire()
-        index = self.current
-        self.doLock.release()
-        file_path = self.queue[index]['file_path']
-        book_id = self.queue[index]['bookid']
-        format_old_ext = u'.' + self.queue[index]['settings']['old_book_format'].lower()
-        format_new_ext = u'.' + self.queue[index]['settings']['new_book_format'].lower()
-
-        # check to see if destination format already exists -
-        # if it does - mark the conversion task as complete and return a success
-        # this will allow send to kindle workflow to continue to work
-        if os.path.isfile(file_path + format_new_ext):
-            log.info("Book id %d already converted to %s", book_id, format_new_ext)
-            cur_book = calibre_db.get_book(book_id)
-            self.queue[index]['path'] = file_path
-            self.queue[index]['title'] = cur_book.title
-            self._handleSuccess()
-            return file_path + format_new_ext
-        else:
-            log.info("Book id %d - target format of %s does not exist. Moving forward with convert.",
-                     book_id,
-                     format_new_ext)
-
-        if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub':
-            check, error_message = self._convert_kepubify(file_path,
-                                                          format_old_ext,
-                                                          format_new_ext,
-                                                          index)
-        else:
-            # check if calibre converter-executable is existing
-            if not os.path.exists(config.config_converterpath):
-                # ToDo Text is not translated
-                self._handleError(_(u"Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath))
-                return
-            check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, index)
-
-        if check == 0:
-            cur_book = calibre_db.get_book(book_id)
-            if os.path.isfile(file_path + format_new_ext):
-                # self.db_queue.join()
-                new_format = db.Data(name=cur_book.data[0].name,
-                                         book_format=self.queue[index]['settings']['new_book_format'].upper(),
-                                         book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext))
-                task = {'task':'add_format','id': book_id, 'format': new_format}
-                self.db_queue.put(task)
-                # To Do how to handle error?
-
-                '''cur_book.data.append(new_format)
-                try:
-                    # db.session.merge(cur_book)
-                    calibre_db.session.commit()
-                except OperationalError as e:
-                    calibre_db.session.rollback()
-                    log.error("Database error: %s", e)
-                    self._handleError(_(u"Database error: %(error)s.", error=e))
-                    return'''
-
-                self.queue[index]['path'] = cur_book.path
-                self.queue[index]['title'] = cur_book.title
-                if config.config_use_google_drive:
-                    os.remove(file_path + format_old_ext)
-                self._handleSuccess()
-                return file_path + format_new_ext
-            else:
-                error_message = format_new_ext.upper() + ' format not found on disk'
-        log.info("ebook converter failed with error while converting book")
-        if not error_message:
-            error_message = 'Ebook converter failed with unknown error'
-        self._handleError(error_message)
-        return
-
-
-    def _convert_calibre(self, file_path, format_old_ext, format_new_ext, index):
-        try:
-            # Linux py2.7 encode as list without quotes no empty element for parameters
-            # linux py3.x no encode and as list without quotes no empty element for parameters
-            # windows py2.7 encode as string with quotes empty element for parameters is okay
-            # windows py 3.x no encode and as string with quotes empty element for parameters is okay
-            # separate handling for windows and linux
-            quotes = [1, 2]
-            command = [config.config_converterpath, (file_path + format_old_ext),
-                       (file_path + format_new_ext)]
-            quotes_index = 3
-            if config.config_calibre:
-                parameters = config.config_calibre.split(" ")
-                for param in parameters:
-                    command.append(param)
-                    quotes.append(quotes_index)
-                    quotes_index += 1
-
-            p = process_open(command, quotes)
-        except OSError as e:
-            return 1, _(u"Ebook-converter failed: %(error)s", error=e)
-
-        while p.poll() is None:
-            nextline = p.stdout.readline()
-            if os.name == 'nt' and sys.version_info < (3, 0):
-                nextline = nextline.decode('windows-1252')
-            elif os.name == 'posix' and sys.version_info < (3, 0):
-                nextline = nextline.decode('utf-8')
-            log.debug(nextline.strip('\r\n'))
-            # parse progress string from calibre-converter
-            progress = re.search(r"(\d+)%\s.*", nextline)
-            if progress:
-                self.UIqueue[index]['progress'] = progress.group(1) + ' %'
-
-        # process returncode
-        check = p.returncode
-        calibre_traceback = p.stderr.readlines()
-        error_message = ""
-        for ele in calibre_traceback:
-            if sys.version_info < (3, 0):
-                ele = ele.decode('utf-8')
-            log.debug(ele.strip('\n'))
-            if not ele.startswith('Traceback') and not ele.startswith('  File'):
-                error_message = "Calibre failed with error: %s" % ele.strip('\n')
-        return check, error_message
-
-
-    def _convert_kepubify(self, file_path, format_old_ext, format_new_ext, index):
-        quotes = [1, 3]
-        command = [config.config_kepubifypath, (file_path + format_old_ext), '-o', os.path.dirname(file_path)]
-        try:
-            p = process_open(command, quotes)
-        except OSError as e:
-            return 1, _(u"Kepubify-converter failed: %(error)s", error=e)
-        self.UIqueue[index]['progress'] = '1 %'
-        while True:
-            nextline = p.stdout.readlines()
-            nextline = [x.strip('\n') for x in nextline if x != '\n']
-            if sys.version_info < (3, 0):
-                nextline = [x.decode('utf-8') for x in nextline]
-            for line in nextline:
-                log.debug(line)
-            if p.poll() is not None:
-                break
-
-        # ToD Handle
-        # process returncode
-        check = p.returncode
-
-        # move file
-        if check == 0:
-            converted_file = glob(os.path.join(os.path.dirname(file_path), "*.kepub.epub"))
-            if len(converted_file) == 1:
-                copyfile(converted_file[0], (file_path + format_new_ext))
-                os.unlink(converted_file[0])
-            else:
-                return 1, _(u"Converted file not found or more than one file in folder %(folder)s",
-                            folder=os.path.dirname(file_path))
-        return check, None
-
-
-    def add_convert(self, file_path, bookid, user_name, taskMessage, settings, kindle_mail=None):
-        self.doLock.acquire()
-        if self.last >= 20:
-            self._delete_completed_tasks()
-        # progress, runtime, and status = 0
-        self.id += 1
-        task = TASK_CONVERT_ANY
-        if kindle_mail:
-            task = TASK_CONVERT
-        self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, 'kindle': kindle_mail,
-                           'taskType': task, 'settings':settings})
-        self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'taskMess': taskMessage,
-                             'runtime': '0 s', 'stat': STAT_WAITING,'id': self.id, 'taskType': task } )
-
-        self.last=len(self.queue)
-        self.doLock.release()
-
-    def add_email(self, subject, filepath, attachment, settings, recipient, user_name, taskMessage,
-                  text, internal=False):
-        # if more than 20 entries in the list, clean the list
-        self.doLock.acquire()
-        if self.last >= 20:
-            self._delete_completed_tasks()
-            if internal:
-                self.current-= 1
-        # progress, runtime, and status = 0
-        self.id += 1
-        self.queue.append({'subject':subject, 'attachment':attachment, 'filepath':filepath,
-                           'settings':settings, 'recipent':recipient, 'starttime': 0,
-                           'taskType': TASK_EMAIL, 'text':text})
-        self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'taskMess': taskMessage,
-                             'runtime': '0 s', 'stat': STAT_WAITING,'id': self.id, 'taskType': TASK_EMAIL })
-        self.last=len(self.queue)
-        self.doLock.release()
-
-    def add_upload(self, user_name, taskMessage):
-        # if more than 20 entries in the list, clean the list
-        self.doLock.acquire()
-
-
-        if self.last >= 20:
-            self._delete_completed_tasks()
-        # progress=100%, runtime=0, and status finished
-        self.id += 1
-        starttime = datetime.now()
-        self.queue.append({'starttime': starttime, 'taskType': TASK_UPLOAD})
-        self.UIqueue.append({'user': user_name, 'formStarttime': starttime, 'progress': "100 %", 'taskMess': taskMessage,
-                             'runtime': '0 s', 'stat': STAT_FINISH_SUCCESS,'id': self.id, 'taskType': TASK_UPLOAD})
-        self.last=len(self.queue)
-        self.doLock.release()
-
-    def _send_raw_email(self):
-        self.doLock.acquire()
-        index = self.current
-        self.doLock.release()
-        self.queue[index]['starttime'] = datetime.now()
-        self.UIqueue[index]['formStarttime'] = self.queue[index]['starttime']
-        self.UIqueue[index]['stat'] = STAT_STARTED
-        obj=self.queue[index]
-        # create MIME message
-        msg = MIMEMultipart()
-        msg['Subject'] = self.queue[index]['subject']
-        msg['Message-Id'] = make_msgid('calibre-web')
-        msg['Date'] = formatdate(localtime=True)
-        text = self.queue[index]['text']
-        msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
-        if obj['attachment']:
-            result = get_attachment(obj['filepath'], obj['attachment'])
-            if result:
-                msg.attach(result)
-            else:
-                self._handleError(u"Attachment not found")
-                return
-
-        msg['From'] = obj['settings']["mail_from"]
-        msg['To'] = obj['recipent']
-
-        use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
-        try:
-            # convert MIME message to string
-            fp = StringIO()
-            gen = Generator(fp, mangle_from_=False)
-            gen.flatten(msg)
-            msg = fp.getvalue()
-
-            # send email
-            timeout = 600  # set timeout to 5mins
-
-            # redirect output to logfile on python2 pn python3 debugoutput is caught with overwritten
-            # _print_debug function
-            if sys.version_info < (3, 0):
-                org_smtpstderr = smtplib.stderr
-                smtplib.stderr = logger.StderrLogger('worker.smtp')
-
-            if use_ssl == 2:
-                self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout=timeout)
-            else:
-                self.asyncSMTP = email(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout=timeout)
-
-            # link to logginglevel
-            if logger.is_debug_enabled():
-                self.asyncSMTP.set_debuglevel(1)
-            if use_ssl == 1:
-                self.asyncSMTP.starttls()
-            if obj['settings']["mail_password"]:
-                self.asyncSMTP.login(str(obj['settings']["mail_login"]), str(obj['settings']["mail_password"]))
-            self.asyncSMTP.sendmail(obj['settings']["mail_from"], obj['recipent'], msg)
-            self.asyncSMTP.quit()
-            self._handleSuccess()
-
-            if sys.version_info < (3, 0):
-                smtplib.stderr = org_smtpstderr
-
-        except (MemoryError) as e:
-            log.exception(e)
-            self._handleError(u'MemoryError sending email: ' + str(e))
-            return None
-        except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
-            if hasattr(e, "smtp_error"):
-                text = e.smtp_error.decode('utf-8').replace("\n",'. ')
-            elif hasattr(e, "message"):
-                text = e.message
-            else:
-                log.exception(e)
-                text = ''
-            self._handleError(u'Smtplib Error sending email: ' + text)
-            return None
-        except (socket.error) as e:
-            self._handleError(u'Socket Error sending email: ' + e.strerror)
-            return None
-
-    def _handleError(self, error_message):
-        log.error(error_message)
-        self.doLock.acquire()
-        index = self.current
-        self.doLock.release()
-        self.UIqueue[index]['stat'] = STAT_FAIL
-        self.UIqueue[index]['progress'] = "100 %"
-        self.UIqueue[index]['formRuntime'] = datetime.now() - self.queue[index]['starttime']
-        self.UIqueue[index]['message'] = error_message
-
-    def _handleSuccess(self):
-        self.doLock.acquire()
-        index = self.current
-        self.doLock.release()
-        self.UIqueue[index]['stat'] = STAT_FINISH_SUCCESS
-        self.UIqueue[index]['progress'] = "100 %"
-        self.UIqueue[index]['formRuntime'] = datetime.now() - self.queue[index]['starttime']
-
-
-def get_taskstatus():
-    return _worker.get_taskstatus()
-
-
-def add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text):
-    return _worker.add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text)
-
-
-def add_upload(user_name, taskMessage):
-    return _worker.add_upload(user_name, taskMessage)
-
-
-def add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail=None):
-    return _worker.add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail)
-
-
-_worker = WorkerThread()
-_worker.start()