Refactored helper.py and db.py

cps/__init__.py

@@ -37,8 +37,6 @@ from . import config_sql, logger, cache_buster, cli, ub, db
 from .reverseproxy import ReverseProxied
 from .server import WebServer
 
-# import queue
-# queue = queue.Queue()
 
 mimetypes.init()
 mimetypes.add_type('application/xhtml+xml', '.xhtml')

cps/admin.py

@@ -39,7 +39,7 @@ from sqlalchemy.sql.expression import func
 
 from . import constants, logger, helper, services
 from . import db, calibre_db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils
-from .helper import speaking_language, check_valid_domain, send_test_mail, reset_password, generate_password_hash
+from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash
 from .gdriveutils import is_gdrive_ready, gdrive_support
 from .web import admin_required, render_title_template, before_request, unconfigured, login_required_if_no_ano
 
@@ -878,7 +878,7 @@ def _handle_edit_user(to_save, content,languages, translations, kobo_support, do
 @admin_required
 def new_user():
     content = ub.User()
-    languages = speaking_language()
+    languages = calibre_db.speaking_language()
     translations = [LC('en')] + babel.list_translations()
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
     if request.method == "POST":
@@ -942,11 +942,11 @@ def edit_user(user_id):
         flash(_(u"User not found"), category="error")
         return redirect(url_for('admin.admin'))
     downloads = list()
-    languages = speaking_language()
+    languages = calibre_db.speaking_language()
     translations = babel.list_translations() + [LC('en')]
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
     for book in content.downloads:
-        downloadbook = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+        downloadbook = calibre_db.get_book(book.book_id)
         if downloadbook:
             downloads.append(downloadbook)
         else:

cps/db.py

@@ -22,10 +22,9 @@ import sys
 import os
 import re
 import ast
+import json
 from datetime import datetime
 import threading
-import time
-import queue
 
 from sqlalchemy import create_engine
 from sqlalchemy import Table, Column, ForeignKey, CheckConstraint
@@ -33,12 +32,24 @@ from sqlalchemy import String, Integer, Boolean, TIMESTAMP, Float
 from sqlalchemy.orm import relationship, sessionmaker, scoped_session
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.exc import OperationalError
+from flask_login import current_user
+from sqlalchemy.sql.expression import and_, true, false, text, func, or_
+from babel import Locale as LC
+from babel.core import UnknownLocaleError
+from flask_babel import gettext as _
+
+from . import logger, ub, isoLanguages
+from .pagination import Pagination
+
+try:
+    import unidecode
+    use_unidecode = True
+except ImportError:
+    use_unidecode = False
+
 
-from . import logger
-# session = None
 cc_exceptions = ['datetime', 'comments', 'composite', 'series']
 cc_classes = {}
-# engine = None
 
 Base = declarative_base()
 
@@ -327,6 +338,7 @@ class CalibreDB(threading.Thread):
         self.session = None
         self.queue = None
         self.log = None
+        self.config = None
 
     def add_queue(self,queue):
         self.queue = queue
@@ -356,6 +368,7 @@ class CalibreDB(threading.Thread):
         self.queue.put('dummy')
 
     def setup_db(self, config, app_db_path):
+        self.config = config
         self.dispose()
         # global engine
 
@@ -441,6 +454,149 @@ class CalibreDB(threading.Thread):
         self.session = Session()
         return True
 
+    def get_book(self, book_id):
+        return self.session.query(Books).filter(Books.id == book_id).first()
+
+    def get_filtered_book(self, book_id, allow_show_archived=False):
+        return self.session.query(Books).filter(Books.id == book_id).\
+            filter(self.common_filters(allow_show_archived)).first()
+
+    def get_book_by_uuid(self, book_uuid):
+        return self.session.query(Books).filter(Books.uuid == book_uuid).first()
+
+    def get_book_format(self, book_id, format):
+        return self.session.query(Data).filter(Data.book == book_id).filter(Data.format == format).first()
+
+    # Language and content filters for displaying in the UI
+    def common_filters(self, allow_show_archived=False):
+        if not allow_show_archived:
+            archived_books = (
+                ub.session.query(ub.ArchivedBook)
+                    .filter(ub.ArchivedBook.user_id == int(current_user.id))
+                    .filter(ub.ArchivedBook.is_archived == True)
+                    .all()
+            )
+            archived_book_ids = [archived_book.book_id for archived_book in archived_books]
+            archived_filter = Books.id.notin_(archived_book_ids)
+        else:
+            archived_filter = true()
+
+        if current_user.filter_language() != "all":
+            lang_filter = Books.languages.any(Languages.lang_code == current_user.filter_language())
+        else:
+            lang_filter = true()
+        negtags_list = current_user.list_denied_tags()
+        postags_list = current_user.list_allowed_tags()
+        neg_content_tags_filter = false() if negtags_list == [''] else Books.tags.any(Tags.name.in_(negtags_list))
+        pos_content_tags_filter = true() if postags_list == [''] else Books.tags.any(Tags.name.in_(postags_list))
+        if self.config.config_restricted_column:
+            pos_cc_list = current_user.allowed_column_value.split(',')
+            pos_content_cc_filter = true() if pos_cc_list == [''] else \
+                getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
+                    any(cc_classes[self.config.config_restricted_column].value.in_(pos_cc_list))
+            neg_cc_list = current_user.denied_column_value.split(',')
+            neg_content_cc_filter = false() if neg_cc_list == [''] else \
+                getattr(Books, 'custom_column_' + str(self.config.config_restricted_column)). \
+                    any(cc_classes[self.config.config_restricted_column].value.in_(neg_cc_list))
+        else:
+            pos_content_cc_filter = true()
+            neg_content_cc_filter = false()
+        return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter,
+                    pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
+
+    # Fill indexpage with all requested data from database
+    def fill_indexpage(self, page, database, db_filter, order, *join):
+        return self.fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join)
+
+    def fill_indexpage_with_archived_books(self, page, database, db_filter, order, allow_show_archived, *join):
+        if current_user.show_detail_random():
+            randm = self.session.query(Books) \
+                .filter(self.common_filters(allow_show_archived)) \
+                .order_by(func.random()) \
+                .limit(self.config.config_random_books)
+        else:
+            randm = false()
+        off = int(int(self.config.config_books_per_page) * (page - 1))
+        query = self.session.query(database) \
+            .join(*join, isouter=True) \
+            .filter(db_filter) \
+            .filter(self.common_filters(allow_show_archived))
+        pagination = Pagination(page, self.config.config_books_per_page,
+                                len(query.all()))
+        entries = query.order_by(*order).offset(off).limit(self.config.config_books_per_page).all()
+        for book in entries:
+            book = self.order_authors(book)
+        return entries, randm, pagination
+
+    # Orders all Authors in the list according to authors sort
+    def order_authors(self, entry):
+        sort_authors = entry.author_sort.split('&')
+        authors_ordered = list()
+        error = False
+        for auth in sort_authors:
+            # ToDo: How to handle not found authorname
+            result = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).first()
+            if not result:
+                error = True
+                break
+            authors_ordered.append(result)
+        if not error:
+            entry.authors = authors_ordered
+        return entry
+
+    def get_typeahead(self, database, query, replace=('', ''), tag_filter=true()):
+        query = query or ''
+        self.session.connection().connection.connection.create_function("lower", 1, self.lcase)
+        entries = self.session.query(database).filter(tag_filter). \
+            filter(func.lower(database.name).ilike("%" + query + "%")).all()
+        json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries])
+        return json_dumps
+
+    def check_exists_book(self, authr, title):
+        self.session.connection().connection.connection.create_function("lower", 1, self.lcase)
+        q = list()
+        authorterms = re.split(r'\s*&\s*', authr)
+        for authorterm in authorterms:
+            q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
+
+        return self.session.query(Books)\
+            .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first()
+
+    # read search results from calibre-database and return it (function is used for feed and simple search
+    def get_search_results(self, term):
+        term.strip().lower()
+        self.session.connection().connection.connection.create_function("lower", 1, self.lcase)
+        q = list()
+        authorterms = re.split("[, ]+", term)
+        for authorterm in authorterms:
+            q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
+
+        return self.session.query(Books).filter(self.common_filters()).filter(
+            or_(Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")),
+                Books.series.any(func.lower(Series.name).ilike("%" + term + "%")),
+                Books.authors.any(and_(*q)),
+                Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")),
+                func.lower(Books.title).ilike("%" + term + "%")
+                )).order_by(Books.sort).all()
+
+    # Creates for all stored languages a translated speaking name in the array for the UI
+    def speaking_language(self, languages=None):
+        from . import get_locale
+
+        if not languages:
+            languages = self.session.query(Languages) \
+                .join(books_languages_link) \
+                .join(Books) \
+                .filter(self.common_filters()) \
+                .group_by(text('books_languages_link.lang_code')).all()
+        for lang in languages:
+            try:
+                cur_l = LC.parse(lang.lang_code)
+                lang.name = cur_l.get_language_name(get_locale())
+            except UnknownLocaleError:
+                lang.name = _(isoLanguages.get(part3=lang.lang_code).name)
+        return languages
+
     def update_title_sort(self, config, conn=None):
         # user defined sort function for calibre databases (Series, etc.)
         def _title_sort(title):
@@ -481,8 +637,13 @@ class CalibreDB(threading.Thread):
                 if table is not None:
                     Base.metadata.remove(table)
 
-
     def reconnect_db(self, config, app_db_path):
         self.session.close()
         self.engine.dispose()
         self.setup_db(config, app_db_path)
+
+    def lcase(self, s):
+        try:
+            return unidecode.unidecode(s.lower())
+        except Exception as e:
+            self.log.exception(e)

cps/editbooks.py

@@ -35,7 +35,6 @@ from sqlalchemy.exc import OperationalError
 from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
 from . import config, get_locale, ub, worker, db
 from . import calibre_db
-from .helper import order_authors, common_filters
 from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required
 
 
@@ -176,7 +175,7 @@ def modify_identifiers(input_identifiers, db_identifiers, db_session):
 @login_required
 def delete_book(book_id, book_format):
     if current_user.role_delete_books():
-        book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
+        book = calibre_db.get_book(book_id)
         if book:
             try:
                 result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper())
@@ -250,9 +249,7 @@ def delete_book(book_id, book_format):
 def render_edit_book(book_id):
     calibre_db.update_title_sort(config)
     cc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
-    book = calibre_db.session.query(db.Books)\
+    book = calibre_db.get_filtered_book(book_id)
-        .filter(db.Books.id == book_id).filter(common_filters()).first()
-
     if not book:
         flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error")
         return redirect(url_for("web.index"))
@@ -260,7 +257,7 @@ def render_edit_book(book_id):
     for lang in book.languages:
         lang.language_name = isoLanguages.get_language_name(get_locale(), lang.lang_code)
 
-    book = order_authors(book)
+    book = calibre_db.order_authors(book)
 
     author_names = []
     for authr in book.authors:
@@ -447,8 +444,7 @@ def upload_single_file(request, book, book_id):
                 return redirect(url_for('web.show_book', book_id=book.id))
 
             file_size = os.path.getsize(saved_filename)
-            is_format = calibre_db.session.query(db.Data).filter(db.Data.book == book_id).\
+            is_format = calibre_db.get_book_format(book_id, file_ext.upper())
-                filter(db.Data.format == file_ext.upper()).first()
 
             # Format entry already exists, no need to update the database
             if is_format:
@@ -500,8 +496,7 @@ def edit_book(book_id):
 
     # create the function for sorting...
     calibre_db.update_title_sort(config)
-    book = calibre_db.session.query(db.Books)\
+    book = calibre_db.get_filtered_book(book_id)
-        .filter(db.Books.id == book_id).filter(common_filters()).first()
 
     # Book not found
     if not book:
@@ -702,7 +697,7 @@ def upload():
                 series_index = meta.series_id
 
                 if title != _(u'Unknown') and authr != _(u'Unknown'):
-                    entry = helper.check_exists_book(authr, title)
+                    entry = calibre_db.check_exists_book(authr, title)
                     if entry:
                         log.info("Uploaded book probably exists in library")
                         flash(_(u"Uploaded book probably exists in the library, consider to change before upload new: ")
@@ -807,7 +802,7 @@ def upload():
                 calibre_db.update_title_sort(config)
                 # Reread book. It's important not to filter the result, as it could have language which hide it from
                 # current users view (tags are not stored/extracted from metadata and could also be limited)
-                book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
+                book = calibre_db.get_book(book_id)
                 # upload book to gdrive if nesseccary and add "(bookid)" to folder name
                 if config.config_use_google_drive:
                     gdriveutils.updateGdriveCalibreFromLocal()

cps/helper.py

@@ -74,8 +74,8 @@ log = logger.create()
 
 # Convert existing book entry to new format
 def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None):
-    book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
-    data = calibre_db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first()
+    data = calibre_db.get_book_format(book.id, old_book_format)
     if not data:
         error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
         log.error("convert_book_format: %s", error_message)
@@ -212,7 +212,7 @@ def check_read_formats(entry):
 # 3: If Pdf file is existing, it's directly send to kindle email
 def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
     """Send email with attachments"""
-    book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
 
     if convert == 1:
         # returns None if success, otherwise errormessage
@@ -324,7 +324,7 @@ def delete_book_file(book, calibrepath, book_format=None):
 
 
 def update_dir_structure_file(book_id, calibrepath, first_author):
-    localbook = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    localbook = calibre_db.get_book(book_id)
     path = os.path.join(calibrepath, localbook.path)
 
     authordir = localbook.path.split('/')[0]
@@ -383,7 +383,7 @@ def update_dir_structure_file(book_id, calibrepath, first_author):
 
 def update_dir_structure_gdrive(book_id, first_author):
     error = False
-    book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
     path = book.path
 
     authordir = book.path.split('/')[0]
@@ -494,18 +494,17 @@ def get_cover_on_failure(use_generic_cover):
 
 
 def get_book_cover(book_id):
-    book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters(allow_show_archived=True)).first()
+    book = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
     return get_book_cover_internal(book, use_generic_cover_on_failure=True)
 
 
 def get_book_cover_with_uuid(book_uuid,
                              use_generic_cover_on_failure=True):
-    book = calibre_db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     return get_book_cover_internal(book, use_generic_cover_on_failure)
 
 
-def get_book_cover_internal(book,
+def get_book_cover_internal(book, use_generic_cover_on_failure):
-                            use_generic_cover_on_failure):
     if book and book.has_cover:
         if config.config_use_google_drive:
             try:
@@ -725,66 +724,12 @@ def render_task_status(tasklist):
     return renderedtasklist
 
 
-# Language and content filters for displaying in the UI
-def common_filters(allow_show_archived=False):
-    if not allow_show_archived:
-        archived_books = (
-            ub.session.query(ub.ArchivedBook)
-            .filter(ub.ArchivedBook.user_id == int(current_user.id))
-            .filter(ub.ArchivedBook.is_archived == True)
-            .all()
-        )
-        archived_book_ids = [archived_book.book_id for archived_book in archived_books]
-        archived_filter = db.Books.id.notin_(archived_book_ids)
-    else:
-        archived_filter = true()
-
-    if current_user.filter_language() != "all":
-        lang_filter = db.Books.languages.any(db.Languages.lang_code == current_user.filter_language())
-    else:
-        lang_filter = true()
-    negtags_list = current_user.list_denied_tags()
-    postags_list = current_user.list_allowed_tags()
-    neg_content_tags_filter = false() if negtags_list == [''] else db.Books.tags.any(db.Tags.name.in_(negtags_list))
-    pos_content_tags_filter = true() if postags_list == [''] else db.Books.tags.any(db.Tags.name.in_(postags_list))
-    if config.config_restricted_column:
-        pos_cc_list = current_user.allowed_column_value.split(',')
-        pos_content_cc_filter = true() if pos_cc_list == [''] else \
-            getattr(db.Books, 'custom_column_' + str(config.config_restricted_column)).\
-                    any(db.cc_classes[config.config_restricted_column].value.in_(pos_cc_list))
-        neg_cc_list = current_user.denied_column_value.split(',')
-        neg_content_cc_filter = false() if neg_cc_list == [''] else \
-            getattr(db.Books, 'custom_column_' + str(config.config_restricted_column)).\
-                    any(db.cc_classes[config.config_restricted_column].value.in_(neg_cc_list))
-    else:
-        pos_content_cc_filter = true()
-        neg_content_cc_filter = false()
-    return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter,
-                pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
-
-
 def tags_filters():
     negtags_list = current_user.list_denied_tags()
     postags_list = current_user.list_allowed_tags()
     neg_content_tags_filter = false() if negtags_list == [''] else db.Tags.name.in_(negtags_list)
     pos_content_tags_filter = true() if postags_list == [''] else db.Tags.name.in_(postags_list)
     return and_(pos_content_tags_filter, ~neg_content_tags_filter)
-    # return ~(false()) if postags_list == [''] else db.Tags.in_(postags_list)
-
-
-# Creates for all stored languages a translated speaking name in the array for the UI
-def speaking_language(languages=None):
-    if not languages:
-        languages = calibre_db.session.query(db.Languages).join(db.books_languages_link).join(db.Books)\
-            .filter(common_filters())\
-            .group_by(text('books_languages_link.lang_code')).all()
-    for lang in languages:
-        try:
-            cur_l = LC.parse(lang.lang_code)
-            lang.name = cur_l.get_language_name(get_locale())
-        except UnknownLocaleError:
-            lang.name = _(isoLanguages.get(part3=lang.lang_code).name)
-    return languages
 
 
 # checks if domain is in database (including wildcards)
@@ -801,76 +746,9 @@ def check_valid_domain(domain_text):
     return not len(result)
 
 
-# Orders all Authors in the list according to authors sort
-def order_authors(entry):
-    sort_authors = entry.author_sort.split('&')
-    authors_ordered = list()
-    error = False
-    for auth in sort_authors:
-        # ToDo: How to handle not found authorname
-        result = calibre_db.session.query(db.Authors).filter(db.Authors.sort == auth.lstrip().strip()).first()
-        if not result:
-            error = True
-            break
-        authors_ordered.append(result)
-    if not error:
-        entry.authors = authors_ordered
-    return entry
-
-
-# Fill indexpage with all requested data from database
-def fill_indexpage(page, database, db_filter, order, *join):
-    return fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join)
-
-
-def fill_indexpage_with_archived_books(page, database, db_filter, order, allow_show_archived, *join):
-    if current_user.show_detail_random():
-        randm = calibre_db.session.query(db.Books).filter(common_filters(allow_show_archived))\
-            .order_by(func.random()).limit(config.config_random_books)
-    else:
-        randm = false()
-    off = int(int(config.config_books_per_page) * (page - 1))
-    query = calibre_db.session.query(database).join(*join, isouter=True).\
-        filter(db_filter).\
-        filter(common_filters(allow_show_archived))
-    pagination = Pagination(page, config.config_books_per_page,
-                            len(query.all()))
-    entries = query.order_by(*order).offset(off).limit(config.config_books_per_page).all()
-    for book in entries:
-        book = order_authors(book)
-    return entries, randm, pagination
-
-
-def get_typeahead(database, query, replace=('', ''), tag_filter=true()):
-    query = query or ''
-    calibre_db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    entries = calibre_db.session.query(database).filter(tag_filter).\
-        filter(func.lower(database.name).ilike("%" + query + "%")).all()
-    json_dumps = json.dumps([dict(name=r.name.replace(*replace)) for r in entries])
-    return json_dumps
-
-
-# read search results from calibre-database and return it (function is used for feed and simple search
-def get_search_results(term):
-    calibre_db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    q = list()
-    authorterms = re.split("[, ]+", term)
-    for authorterm in authorterms:
-        q.append(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + authorterm + "%")))
-
-    db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + term + "%"))
-
-    return calibre_db.session.query(db.Books).filter(common_filters()).filter(
-        or_(db.Books.tags.any(func.lower(db.Tags.name).ilike("%" + term + "%")),
-            db.Books.series.any(func.lower(db.Series.name).ilike("%" + term + "%")),
-            db.Books.authors.any(and_(*q)),
-            db.Books.publishers.any(func.lower(db.Publishers.name).ilike("%" + term + "%")),
-            func.lower(db.Books.title).ilike("%" + term + "%")
-            )).order_by(db.Books.sort).all()
-
-
 def get_cc_columns(filter_config_custom_read=False):
-    tmpcc = calibre_db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
+    tmpcc = calibre_db.session.query(db.Custom_Columns)\
+        .filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
     cc = []
     r = None
     if config.config_columns_to_ignore:
@@ -887,10 +765,9 @@ def get_cc_columns(filter_config_custom_read=False):
 
 def get_download_link(book_id, book_format, client):
     book_format = book_format.split(".")[0]
-    book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+    book = calibre_db.get_filtered_book(book_id)
     if book:
-        data1 = calibre_db.session.query(db.Data).filter(db.Data.book == book.id)\
+        data1 = data = calibre_db.get_book_format(book.id, book_format.upper())
-            .filter(db.Data.format == book_format.upper()).first()
     else:
         abort(404)
     if data1:
@@ -908,25 +785,3 @@ def get_download_link(book_id, book_format, client):
         return do_download_file(book, book_format, client, data1, headers)
     else:
         abort(404)
-
-
-def check_exists_book(authr, title):
-    calibre_db.session.connection().connection.connection.create_function("lower", 1, lcase)
-    q = list()
-    authorterms = re.split(r'\s*&\s*', authr)
-    for authorterm in authorterms:
-        q.append(db.Books.authors.any(func.lower(db.Authors.name).ilike("%" + authorterm + "%")))
-
-    return calibre_db.session.query(db.Books).filter(
-        and_(db.Books.authors.any(and_(*q)),
-             func.lower(db.Books.title).ilike("%" + title + "%")
-             )).first()
-
-############### Database Helper functions
-
-
-def lcase(s):
-    try:
-        return unidecode.unidecode(s.lower())
-    except Exception as e:
-        log.exception(e)

cps/jinjia.py

@@ -111,10 +111,3 @@ def timestamptodate(date, fmt=None):
 @jinjia.app_template_filter('yesno')
 def yesno(value, yes, no):
     return yes if value else no
-
-
-'''@jinjia.app_template_filter('canread')
-def canread(ext):
-    if isinstance(ext, db.Data):
-        ext = ext.format
-    return ext.lower() in EXTENSIONS_READER'''

cps/kobo.py

@@ -256,7 +256,7 @@ def HandleMetadataRequest(book_uuid):
     if not current_app.wsgi_app.is_proxied:
         log.debug('Kobo: Received unproxied request, changed request port to server port')
     log.info("Kobo library metadata request received for book %s" % book_uuid)
-    book = calibre_db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     if not book or not book.data:
         log.info(u"Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
@@ -474,7 +474,7 @@ def add_items_to_shelf(items, shelf):
                 items_unknown_to_calibre.append(item)
                 continue
 
-            book = calibre_db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none()
+            book = calibre_db.get_book_by_uuid(item["RevisionId"])
             if not book:
                 items_unknown_to_calibre.append(item)
                 continue
@@ -545,7 +545,7 @@ def HandleTagRemoveItem(tag_id):
                 items_unknown_to_calibre.append(item)
                 continue
 
-            book = calibre_db.session.query(db.Books).filter(db.Books.uuid == item["RevisionId"]).one_or_none()
+            book = calibre_db.get_book_by_uuid(item["RevisionId"])
             if not book:
                 items_unknown_to_calibre.append(item)
                 continue
@@ -613,7 +613,7 @@ def create_kobo_tag(shelf):
         "Type": "UserTag"
     }
     for book_shelf in shelf.books:
-        book = calibre_db.session.query(db.Books).filter(db.Books.id == book_shelf.book_id).one_or_none()
+        book = calibre_db.get_book_by_uuid(book_shelf.book_id)
         if not book:
             log.info(u"Book (id: %s) in BookShelf (id: %s) not found in book database",  book_shelf.book_id, shelf.id)
             continue
@@ -629,7 +629,7 @@ def create_kobo_tag(shelf):
 @kobo.route("/v1/library/<book_uuid>/state", methods=["GET", "PUT"])
 @login_required
 def HandleStateRequest(book_uuid):
-    book = calibre_db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     if not book or not book.data:
         log.info(u"Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()
@@ -804,7 +804,7 @@ def TopLevelEndpoint():
 @login_required
 def HandleBookDeletionRequest(book_uuid):
     log.info("Kobo book deletion request received for book %s" % book_uuid)
-    book = calibre_db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    book = calibre_db.get_book_by_uuid(book_uuid)
     if not book:
         log.info(u"Book %s not found in database", book_uuid)
         return redirect_or_proxy_request()

cps/opds.py

@@ -31,9 +31,9 @@ from sqlalchemy.sql.expression import func, text, or_, and_
 from werkzeug.security import check_password_hash
 
 from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages
-from .helper import fill_indexpage, get_download_link, get_book_cover, speaking_language
+from .helper import get_download_link, get_book_cover
 from .pagination import Pagination
-from .web import common_filters, get_search_results, render_read_books, download_required
+from .web import render_read_books, download_required
 from flask_babel import gettext as _
 from babel import Locale as LC
 from babel.core import UnknownLocaleError
@@ -100,7 +100,7 @@ def feed_normal_search():
 @requires_basic_auth_if_no_ano
 def feed_new():
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books, True, [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
 
@@ -108,7 +108,7 @@ def feed_new():
 @opds.route("/opds/discover")
 @requires_basic_auth_if_no_ano
 def feed_discover():
-    entries = calibre_db.session.query(db.Books).filter(common_filters()).order_by(func.random())\
+    entries = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).order_by(func.random())\
         .limit(config.config_books_per_page)
     pagination = Pagination(1, config.config_books_per_page, int(config.config_books_per_page))
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
@@ -118,7 +118,7 @@ def feed_discover():
 @requires_basic_auth_if_no_ano
 def feed_best_rated():
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books, db.Books.ratings.any(db.Ratings.rating > 9),
                                                         [db.Books.timestamp.desc()])
     return render_xml_template('feed.xml', entries=entries, pagination=pagination)
@@ -133,16 +133,13 @@ def feed_hot():
     hot_books = all_books.offset(off).limit(config.config_books_per_page)
     entries = list()
     for book in hot_books:
-        downloadBook = calibre_db.session.query(db.Books).filter(db.Books.id == book.Downloads.book_id).first()
+        downloadBook = calibre_db.get_book(book.Downloads.book_id)
         if downloadBook:
             entries.append(
-                calibre_db.session.query(db.Books).filter(common_filters())
+                calibre_db.get_filtered_book(book.Downloads.book_id)
-                .filter(db.Books.id == book.Downloads.book_id).first()
             )
         else:
             ub.delete_download(book.Downloads.book_id)
-            # ub.session.query(ub.Downloads).filter(book.Downloads.book_id == ub.Downloads.book_id).delete()
-            # ub.session.commit()
     numBooks = entries.__len__()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1),
                             config.config_books_per_page, numBooks)
@@ -153,8 +150,10 @@ def feed_hot():
 @requires_basic_auth_if_no_ano
 def feed_authorindex():
     off = request.args.get("offset") or 0
-    entries = calibre_db.session.query(db.Authors).join(db.books_authors_link).join(db.Books).filter(common_filters())\
+    entries = calibre_db.session.query(db.Authors).join(db.books_authors_link).join(db.Books)\
-        .group_by(text('books_authors_link.author')).order_by(db.Authors.sort).limit(config.config_books_per_page)\
+        .filter(calibre_db.common_filters())\
+        .group_by(text('books_authors_link.author'))\
+        .order_by(db.Authors.sort).limit(config.config_books_per_page)\
         .offset(off)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(calibre_db.session.query(db.Authors).all()))
@@ -165,7 +164,7 @@ def feed_authorindex():
 @requires_basic_auth_if_no_ano
 def feed_author(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.authors.any(db.Authors.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -176,8 +175,11 @@ def feed_author(book_id):
 @requires_basic_auth_if_no_ano
 def feed_publisherindex():
     off = request.args.get("offset") or 0
-    entries = calibre_db.session.query(db.Publishers).join(db.books_publishers_link).join(db.Books).filter(common_filters())\
+    entries = calibre_db.session.query(db.Publishers)\
-        .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.sort)\
+        .join(db.books_publishers_link)\
+        .join(db.Books).filter(calibre_db.common_filters())\
+        .group_by(text('books_publishers_link.publisher'))\
+        .order_by(db.Publishers.sort)\
         .limit(config.config_books_per_page).offset(off)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(calibre_db.session.query(db.Publishers).all()))
@@ -188,7 +190,7 @@ def feed_publisherindex():
 @requires_basic_auth_if_no_ano
 def feed_publisher(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.publishers.any(db.Publishers.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -199,8 +201,14 @@ def feed_publisher(book_id):
 @requires_basic_auth_if_no_ano
 def feed_categoryindex():
     off = request.args.get("offset") or 0
-    entries = calibre_db.session.query(db.Tags).join(db.books_tags_link).join(db.Books).filter(common_filters())\
+    entries = calibre_db.session.query(db.Tags)\
-        .group_by(text('books_tags_link.tag')).order_by(db.Tags.name).offset(off).limit(config.config_books_per_page)
+        .join(db.books_tags_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters())\
+        .group_by(text('books_tags_link.tag'))\
+        .order_by(db.Tags.name)\
+        .offset(off)\
+        .limit(config.config_books_per_page)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(calibre_db.session.query(db.Tags).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_category', pagination=pagination)
@@ -210,7 +218,7 @@ def feed_categoryindex():
 @requires_basic_auth_if_no_ano
 def feed_category(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.tags.any(db.Tags.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -221,8 +229,13 @@ def feed_category(book_id):
 @requires_basic_auth_if_no_ano
 def feed_seriesindex():
     off = request.args.get("offset") or 0
-    entries = calibre_db.session.query(db.Series).join(db.books_series_link).join(db.Books).filter(common_filters())\
+    entries = calibre_db.session.query(db.Series)\
-        .group_by(text('books_series_link.series')).order_by(db.Series.sort).offset(off).all()
+        .join(db.books_series_link)\
+        .join(db.Books)\
+        .filter(calibre_db.common_filters())\
+        .group_by(text('books_series_link.series'))\
+        .order_by(db.Series.sort)\
+        .offset(off).all()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(calibre_db.session.query(db.Series).all()))
     return render_xml_template('feed.xml', listelements=entries, folder='opds.feed_series', pagination=pagination)
@@ -232,7 +245,7 @@ def feed_seriesindex():
 @requires_basic_auth_if_no_ano
 def feed_series(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.series.any(db.Series.id == book_id),
                                                         [db.Books.series_index])
@@ -245,8 +258,11 @@ def feed_ratingindex():
     off = request.args.get("offset") or 0
     entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
                                (db.Ratings.rating / 2).label('name')) \
-        .join(db.books_ratings_link).join(db.Books).filter(common_filters()) \
+        .join(db.books_ratings_link)\
-        .group_by(text('books_ratings_link.rating')).order_by(db.Ratings.rating).all()
+        .join(db.Books)\
+        .filter(calibre_db.common_filters()) \
+        .group_by(text('books_ratings_link.rating'))\
+        .order_by(db.Ratings.rating).all()
 
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(entries))
@@ -260,7 +276,7 @@ def feed_ratingindex():
 @requires_basic_auth_if_no_ano
 def feed_ratings(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.ratings.any(db.Ratings.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -271,8 +287,10 @@ def feed_ratings(book_id):
 @requires_basic_auth_if_no_ano
 def feed_formatindex():
     off = request.args.get("offset") or 0
-    entries = calibre_db.session.query(db.Data).join(db.Books).filter(common_filters()) \
+    entries = calibre_db.session.query(db.Data).join(db.Books)\
-        .group_by(db.Data.format).order_by(db.Data.format).all()
+        .filter(calibre_db.common_filters()) \
+        .group_by(db.Data.format)\
+        .order_by(db.Data.format).all()
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(entries))
 
@@ -286,7 +304,7 @@ def feed_formatindex():
 @requires_basic_auth_if_no_ano
 def feed_format(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.data.any(db.Data.format == book_id.upper()),
                                                         [db.Books.timestamp.desc()])
@@ -299,7 +317,7 @@ def feed_format(book_id):
 def feed_languagesindex():
     off = request.args.get("offset") or 0
     if current_user.filter_language() == u"all":
-        languages = speaking_language()
+        languages = calibre_db.speaking_language()
     else:
         try:
             cur_l = LC.parse(current_user.filter_language())
@@ -320,7 +338,7 @@ def feed_languagesindex():
 @requires_basic_auth_if_no_ano
 def feed_languages(book_id):
     off = request.args.get("offset") or 0
-    entries, __, pagination = fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
+    entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1),
                                                         db.Books,
                                                         db.Books.languages.any(db.Languages.id == book_id),
                                                         [db.Books.timestamp.desc()])
@@ -356,7 +374,7 @@ def feed_shelf(book_id):
         books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == book_id).order_by(
             ub.BookShelf.order.asc()).all()
         for book in books_in_shelf:
-            cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+            cur_book = calibre_db.get_book(book.book_id)
             result.append(cur_book)
     pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page,
                             len(result))
@@ -390,8 +408,7 @@ def get_metadata_calibre_companion(uuid, library):
 
 def feed_search(term):
     if term:
-        term = term.strip().lower()
+        entries = calibre_db.get_search_results(term)
-        entries = get_search_results(term)
         entriescount = len(entries) if len(entries) > 0 else 1
         pagination = Pagination(1, entriescount, entriescount)
         return render_xml_template('feed.xml', searchterm=term, entries=entries, pagination=pagination)

cps/shelf.py

@@ -30,7 +30,6 @@ from sqlalchemy.sql.expression import func
 
 from . import logger, ub, searched_ids, db, calibre_db
 from .web import render_title_template
-from .helper import common_filters
 
 
 shelf = Blueprint('shelf', __name__)
@@ -320,11 +319,11 @@ def show_shelf(shelf_type, shelf_id):
         books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id)\
             .order_by(ub.BookShelf.order.asc()).all()
         for book in books_in_shelf:
-            cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first()
+            cur_book = calibre_db.get_filtered_book(book.book_id)
             if cur_book:
                 result.append(cur_book)
             else:
-                cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+                cur_book = calibre_db.get_book(book.book_id)
                 if not cur_book:
                     log.info('Not existing book %s in %s deleted', book.book_id, shelf)
                     ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book.book_id).delete()
@@ -356,7 +355,7 @@ def order_shelf(shelf_id):
         books_in_shelf2 = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == shelf_id) \
             .order_by(ub.BookShelf.order.asc()).all()
         for book in books_in_shelf2:
-            cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).filter(common_filters()).first()
+            cur_book = calibre_db.get_filtered_book(book.book_id)
             if cur_book:
                 result.append({'title': cur_book.title,
                                'id': cur_book.id,
@@ -364,7 +363,7 @@ def order_shelf(shelf_id):
                                'series': cur_book.series,
                                'series_index': cur_book.series_index})
             else:
-                cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+                cur_book = calibre_db.get_book(book.book_id)
                 result.append({'title': _('Hidden Book'),
                                'id': cur_book.id,
                                'author': [],

cps/web.py

@@ -52,10 +52,9 @@ from . import constants, logger, isoLanguages, services, worker
 from . import searched_ids, lm, babel, db, ub, config, get_locale, app
 from . import calibre_db
 from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
-from .helper import common_filters, get_search_results, fill_indexpage, fill_indexpage_with_archived_books, \
+from .helper import check_valid_domain, render_task_status, json_serial, \
-    speaking_language, check_valid_domain, order_authors, get_typeahead, render_task_status, json_serial, \
     get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \
-    send_registration_mail, check_send_to_kindle, check_read_formats, lcase, tags_filters, reset_password
+    send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password
 from .pagination import Pagination
 from .redirect import redirect_back
 
@@ -440,7 +439,7 @@ def toggle_read(book_id):
     else:
         try:
             calibre_db.update_title_sort(config)
-            book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+            book = calibre_db.get_filtered_book(book_id)
             read_status = getattr(book, 'custom_column_' + str(config.config_read_column))
             if len(read_status):
                 read_status[0].value = not read_status[0].value
@@ -497,7 +496,7 @@ def update_view():
 @web.route("/ajax/getcomic/<int:book_id>/<book_format>/<int:page>")
 @login_required
 def get_comic_book(book_id, book_format, page):
-    book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
     if not book:
         return "", 204
     else:
@@ -551,25 +550,25 @@ def get_comic_book(book_id, book_format, page):
 @web.route("/get_authors_json", methods=['GET'])
 @login_required_if_no_ano
 def get_authors_json():
-    return get_typeahead(db.Authors, request.args.get('q'), ('|', ','))
+    return calibre_db.get_typeahead(db.Authors, request.args.get('q'), ('|', ','))
 
 
 @web.route("/get_publishers_json", methods=['GET'])
 @login_required_if_no_ano
 def get_publishers_json():
-    return get_typeahead(db.Publishers, request.args.get('q'), ('|', ','))
+    return calibre_db.get_typeahead(db.Publishers, request.args.get('q'), ('|', ','))
 
 
 @web.route("/get_tags_json", methods=['GET'])
 @login_required_if_no_ano
 def get_tags_json():
-    return get_typeahead(db.Tags, request.args.get('q'), tag_filter=tags_filters())
+    return calibre_db.get_typeahead(db.Tags, request.args.get('q'), tag_filter=tags_filters())
 
 
 @web.route("/get_series_json", methods=['GET'])
 @login_required_if_no_ano
 def get_series_json():
-    return get_typeahead(db.Series, request.args.get('q'))
+    return calibre_db.get_typeahead(db.Series, request.args.get('q'))
 
 
 @web.route("/get_languages_json", methods=['GET'])
@@ -591,7 +590,7 @@ def get_languages_json():
 def get_matching_tags():
     tag_dict = {'tags': []}
     q = calibre_db.session.query(db.Books)
-    calibre_db.session.connection().connection.connection.create_function("lower", 1, lcase)
+    calibre_db.session.connection().connection.connection.create_function("lower", 1, calibre_db.lcase)
     author_input = request.args.get('author_name') or ''
     title_input = request.args.get('book_title') or ''
     include_tag_inputs = request.args.getlist('include_tag') or ''
@@ -621,7 +620,7 @@ def get_matching_tags():
 @web.route('/page/<int:page>')
 @login_required_if_no_ano
 def index(page):
-    entries, random, pagination = fill_indexpage(page, db.Books, True, [db.Books.timestamp.desc()])
+    entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, [db.Books.timestamp.desc()])
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                  title=_(u"Recently Added Books"), page="root")
 
@@ -648,7 +647,9 @@ def books_list(data, sort, book_id, page):
 
     if data == "rated":
         if current_user.check_visibility(constants.SIDEBAR_BEST_RATED):
-            entries, random, pagination = fill_indexpage(page, db.Books, db.Books.ratings.any(db.Ratings.rating > 9),
+            entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                    db.Books,
+                                                                    db.Books.ratings.any(db.Ratings.rating > 9),
                                                                     order)
             return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                          id=book_id, title=_(u"Top Rated Books"), page="rated")
@@ -656,7 +657,7 @@ def books_list(data, sort, book_id, page):
             abort(404)
     elif data == "discover":
         if current_user.check_visibility(constants.SIDEBAR_RANDOM):
-            entries, __, pagination = fill_indexpage(page, db.Books, True, [func.randomblob(2)])
+            entries, __, pagination = calibre_db.calibre_db.fill_indexpage(page, db.Books, True, [func.randomblob(2)])
             pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page)
             return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id,
                                          title=_(u"Discover (Random Books)"), page="discover")
@@ -685,7 +686,7 @@ def books_list(data, sort, book_id, page):
     elif data == "archived":
         return render_archived_books(page, order)
     else:
-        entries, random, pagination = fill_indexpage(page, db.Books, True, order)
+        entries, random, pagination = calibre_db.fill_indexpage(page, db.Books, True, order)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
                                      title=_(u"Books"), page="newest")
 
@@ -693,7 +694,7 @@ def books_list(data, sort, book_id, page):
 def render_hot_books(page):
     if current_user.check_visibility(constants.SIDEBAR_HOT):
         if current_user.show_detail_random():
-            random = calibre_db.session.query(db.Books).filter(common_filters()) \
+            random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \
                 .order_by(func.random()).limit(config.config_random_books)
         else:
             random = false()
@@ -703,7 +704,7 @@ def render_hot_books(page):
         hot_books = all_books.offset(off).limit(config.config_books_per_page)
         entries = list()
         for book in hot_books:
-            downloadBook = calibre_db.session.query(db.Books).filter(common_filters()).filter(
+            downloadBook = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).filter(
                 db.Books.id == book.Downloads.book_id).first()
             if downloadBook:
                 entries.append(downloadBook)
@@ -720,9 +721,12 @@ def render_hot_books(page):
 
 
 def render_author_books(page, author_id, order):
-    entries, __, pagination = fill_indexpage(page, db.Books, db.Books.authors.any(db.Authors.id == author_id),
+    entries, __, pagination = calibre_db.fill_indexpage(page,
+                                                        db.Books,
+                                                        db.Books.authors.any(db.Authors.id == author_id),
                                                         [order[0], db.Series.name, db.Books.series_index],
-                                             db.books_series_link, db.Series)
+                                                        db.books_series_link,
+                                                        db.Series)
     if entries is None or not len(entries):
         flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"),
               category="error")
@@ -745,10 +749,12 @@ def render_author_books(page, author_id, order):
 def render_publisher_books(page, book_id, order):
     publisher = calibre_db.session.query(db.Publishers).filter(db.Publishers.id == book_id).first()
     if publisher:
-        entries, random, pagination = fill_indexpage(page, db.Books,
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
                                                                 db.Books.publishers.any(db.Publishers.id == book_id),
                                                                 [db.Series.name, order[0], db.Books.series_index],
-                                                     db.books_series_link, db.Series)
+                                                                db.books_series_link,
+                                                                db.Series)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
                                      title=_(u"Publisher: %(name)s", name=publisher.name), page="publisher")
     else:
@@ -758,7 +764,9 @@ def render_publisher_books(page, book_id, order):
 def render_series_books(page, book_id, order):
     name = calibre_db.session.query(db.Series).filter(db.Series.id == book_id).first()
     if name:
-        entries, random, pagination = fill_indexpage(page, db.Books, db.Books.series.any(db.Series.id == book_id),
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db.Books.series.any(db.Series.id == book_id),
                                                                 [db.Books.series_index, order[0]])
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
                                      title=_(u"Series: %(serie)s", serie=name.name), page="series")
@@ -768,7 +776,9 @@ def render_series_books(page, book_id, order):
 
 def render_ratings_books(page, book_id, order):
     name = calibre_db.session.query(db.Ratings).filter(db.Ratings.id == book_id).first()
-    entries, random, pagination = fill_indexpage(page, db.Books, db.Books.ratings.any(db.Ratings.id == book_id),
+    entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                            db.Books,
+                                                            db.Books.ratings.any(db.Ratings.id == book_id),
                                                             [db.Books.timestamp.desc(), order[0]])
     if name and name.rating <= 10:
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
@@ -780,7 +790,8 @@ def render_ratings_books(page, book_id, order):
 def render_formats_books(page, book_id, order):
     name = calibre_db.session.query(db.Data).filter(db.Data.format == book_id.upper()).first()
     if name:
-        entries, random, pagination = fill_indexpage(page, db.Books,
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
                                                                 db.Books.data.any(db.Data.format == book_id.upper()),
                                                                 [db.Books.timestamp.desc(), order[0]])
         return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id,
@@ -792,7 +803,9 @@ def render_formats_books(page, book_id, order):
 def render_category_books(page, book_id, order):
     name = calibre_db.session.query(db.Tags).filter(db.Tags.id == book_id).first()
     if name:
-        entries, random, pagination = fill_indexpage(page, db.Books, db.Books.tags.any(db.Tags.id == book_id),
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
+                                                                db.Books.tags.any(db.Tags.id == book_id),
                                                                 [order[0], db.Series.name, db.Books.series_index],
                                                                 db.books_series_link, db.Series)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=book_id,
@@ -810,7 +823,9 @@ def render_language_books(page, name, order):
             lang_name = _(isoLanguages.get(part3=name).name)
         except KeyError:
             abort(404)
-    entries, random, pagination = fill_indexpage(page, db.Books, db.Books.languages.any(db.Languages.lang_code == name),
+    entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                            db.Books,
+                                                            db.Books.languages.any(db.Languages.lang_code == name),
                                                             [db.Books.timestamp.desc(), order[0]])
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name,
                                  title=_(u"Language: %(name)s", name=lang_name), page="language")
@@ -827,10 +842,10 @@ def books_table():
 def author_list():
     if current_user.check_visibility(constants.SIDEBAR_AUTHOR):
         entries = calibre_db.session.query(db.Authors, func.count('books_authors_link.book').label('count')) \
-            .join(db.books_authors_link).join(db.Books).filter(common_filters()) \
+            .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_authors_link.author')).order_by(db.Authors.sort).all()
         charlist = calibre_db.session.query(func.upper(func.substr(db.Authors.sort, 1, 1)).label('char')) \
-            .join(db.books_authors_link).join(db.Books).filter(common_filters()) \
+            .join(db.books_authors_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Authors.sort, 1, 1))).all()
         for entry in entries:
             entry.Authors.name = entry.Authors.name.replace('|', ',')
@@ -845,10 +860,10 @@ def author_list():
 def publisher_list():
     if current_user.check_visibility(constants.SIDEBAR_PUBLISHER):
         entries = calibre_db.session.query(db.Publishers, func.count('books_publishers_link.book').label('count')) \
-            .join(db.books_publishers_link).join(db.Books).filter(common_filters()) \
+            .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_publishers_link.publisher')).order_by(db.Publishers.name).all()
         charlist = calibre_db.session.query(func.upper(func.substr(db.Publishers.name, 1, 1)).label('char')) \
-            .join(db.books_publishers_link).join(db.Books).filter(common_filters()) \
+            .join(db.books_publishers_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Publishers.name, 1, 1))).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
                                      title=_(u"Publishers"), page="publisherlist", data="publisher")
@@ -862,19 +877,19 @@ def series_list():
     if current_user.check_visibility(constants.SIDEBAR_SERIES):
         if current_user.series_view == 'list':
             entries = calibre_db.session.query(db.Series, func.count('books_series_link.book').label('count')) \
-                .join(db.books_series_link).join(db.Books).filter(common_filters()) \
+                .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(text('books_series_link.series')).order_by(db.Series.sort).all()
             charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
-                .join(db.books_series_link).join(db.Books).filter(common_filters()) \
+                .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
             return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
                                          title=_(u"Series"), page="serieslist", data="series")
         else:
             entries = calibre_db.session.query(db.Books, func.count('books_series_link').label('count')) \
-                .join(db.books_series_link).join(db.Series).filter(common_filters()) \
+                .join(db.books_series_link).join(db.Series).filter(calibre_db.common_filters()) \
                 .group_by(text('books_series_link.series')).order_by(db.Series.sort).all()
             charlist = calibre_db.session.query(func.upper(func.substr(db.Series.sort, 1, 1)).label('char')) \
-                .join(db.books_series_link).join(db.Books).filter(common_filters()) \
+                .join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
                 .group_by(func.upper(func.substr(db.Series.sort, 1, 1))).all()
 
             return render_title_template('grid.html', entries=entries, folder='web.books_list', charlist=charlist,
@@ -889,7 +904,7 @@ def ratings_list():
     if current_user.check_visibility(constants.SIDEBAR_RATING):
         entries = calibre_db.session.query(db.Ratings, func.count('books_ratings_link.book').label('count'),
                                    (db.Ratings.rating / 2).label('name')) \
-            .join(db.books_ratings_link).join(db.Books).filter(common_filters()) \
+            .join(db.books_ratings_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(text('books_ratings_link.rating')).order_by(db.Ratings.rating).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"Ratings list"), page="ratingslist", data="ratings")
@@ -901,8 +916,10 @@ def ratings_list():
 @login_required_if_no_ano
 def formats_list():
     if current_user.check_visibility(constants.SIDEBAR_FORMAT):
-        entries = calibre_db.session.query(db.Data, func.count('data.book').label('count'), db.Data.format.label('format')) \
+        entries = calibre_db.session.query(db.Data,
-            .join(db.Books).filter(common_filters()) \
+                                           func.count('data.book').label('count'),
+                                           db.Data.format.label('format')) \
+            .join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(db.Data.format).order_by(db.Data.format).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=list(),
                                      title=_(u"File formats list"), page="formatslist", data="formats")
@@ -916,7 +933,7 @@ def language_overview():
     if current_user.check_visibility(constants.SIDEBAR_LANGUAGE):
         charlist = list()
         if current_user.filter_language() == u"all":
-            languages = speaking_language()
+            languages = calibre_db.speaking_language()
             # ToDo: generate first character list for languages
         else:
             try:
@@ -944,10 +961,10 @@ def language_overview():
 def category_list():
     if current_user.check_visibility(constants.SIDEBAR_CATEGORY):
         entries = calibre_db.session.query(db.Tags, func.count('books_tags_link.book').label('count')) \
-            .join(db.books_tags_link).join(db.Books).order_by(db.Tags.name).filter(common_filters()) \
+            .join(db.books_tags_link).join(db.Books).order_by(db.Tags.name).filter(calibre_db.common_filters()) \
             .group_by(text('books_tags_link.tag')).all()
         charlist = calibre_db.session.query(func.upper(func.substr(db.Tags.name, 1, 1)).label('char')) \
-            .join(db.books_tags_link).join(db.Books).filter(common_filters()) \
+            .join(db.books_tags_link).join(db.Books).filter(calibre_db.common_filters()) \
             .group_by(func.upper(func.substr(db.Tags.name, 1, 1))).all()
         return render_title_template('list.html', entries=entries, folder='web.books_list', charlist=charlist,
                                      title=_(u"Categories"), page="catlist", data="category")
@@ -980,8 +997,7 @@ def reconnect():
 def search():
     term = request.args.get("query")
     if term:
-        term.strip().lower()
+        entries = calibre_db.get_search_results(term)
-        entries = get_search_results(term)
         ids = list()
         for element in entries:
             ids.append(element.id)
@@ -1004,8 +1020,8 @@ def search():
 def advanced_search():
     # Build custom columns names
     cc = get_cc_columns()
-    calibre_db.session.connection().connection.connection.create_function("lower", 1, lcase)
+    calibre_db.session.connection().connection.connection.create_function("lower", 1, calibre_db.lcase)
-    q = calibre_db.session.query(db.Books).filter(common_filters()).order_by(db.Books.sort)
+    q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).order_by(db.Books.sort)
 
     include_tag_inputs = request.args.getlist('include_tag')
     exclude_tag_inputs = request.args.getlist('exclude_tag')
@@ -1064,7 +1080,7 @@ def advanced_search():
         searchterm.extend(serie.name for serie in serie_names)
         language_names = calibre_db.session.query(db.Languages).filter(db.Languages.id.in_(include_languages_inputs)).all()
         if language_names:
-            language_names = speaking_language(language_names)
+            language_names = calibre_db.speaking_language(language_names)
         searchterm.extend(language.name for language in language_names)
         if rating_high:
             searchterm.extend([_(u"Rating <= %(rating)s", rating=rating_high)])
@@ -1137,15 +1153,15 @@ def advanced_search():
         return render_title_template('search.html', adv_searchterm=searchterm,
                                      entries=q, title=_(u"search"), page="search")
     # prepare data for search-form
-    tags = calibre_db.session.query(db.Tags).join(db.books_tags_link).join(db.Books).filter(common_filters()) \
+    tags = calibre_db.session.query(db.Tags).join(db.books_tags_link).join(db.Books).filter(calibre_db.common_filters()) \
         .group_by(text('books_tags_link.tag')).order_by(db.Tags.name).all()
-    series = calibre_db.session.query(db.Series).join(db.books_series_link).join(db.Books).filter(common_filters()) \
+    series = calibre_db.session.query(db.Series).join(db.books_series_link).join(db.Books).filter(calibre_db.common_filters()) \
-        .group_by(text('books_series_link.series')).order_by(db.Series.name).filter(common_filters()).all()
+        .group_by(text('books_series_link.series')).order_by(db.Series.name).filter(calibre_db.common_filters()).all()
-    extensions = calibre_db.session.query(db.Data).join(db.Books).filter(common_filters()) \
+    extensions = calibre_db.session.query(db.Data).join(db.Books).filter(calibre_db.common_filters()) \
         .group_by(db.Data.format).order_by(db.Data.format).all()
 
     if current_user.filter_language() == u"all":
-        languages = speaking_language()
+        languages = calibre_db.speaking_language()
     else:
         languages = None
     return render_title_template('search_form.html', tags=tags, languages=languages, extensions=extensions,
@@ -1160,7 +1176,8 @@ def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs)
                              ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED)
         else:
             db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED
-        entries, random, pagination = fill_indexpage(page, db.Books,
+        entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                db.Books,
                                                                 db_filter,
                                                                 order,
                                                                 ub.ReadBook, db.Books.id==ub.ReadBook.book_id)
@@ -1170,7 +1187,8 @@ def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs)
                 db_filter = db.cc_classes[config.config_read_column].value == True
             else:
                 db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True
-            entries, random, pagination = fill_indexpage(page, db.Books,
+            entries, random, pagination = calibre_db.fill_indexpage(page,
+                                                                    db.Books,
                                                                     db_filter,
                                                                     order,
                                                                     db.cc_classes[config.config_read_column])
@@ -1207,7 +1225,10 @@ def render_archived_books(page, order):
 
     archived_filter = db.Books.id.in_(archived_book_ids)
 
-    entries, random, pagination = fill_indexpage_with_archived_books(page, db.Books, archived_filter, order,
+    entries, random, pagination = calibre_db.fill_indexpage_with_archived_books(page,
+                                                                                db.Books,
+                                                                                archived_filter,
+                                                                                order,
                                                                                 allow_show_archived=True)
 
     name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')'
@@ -1230,9 +1251,8 @@ def get_cover(book_id):
 @viewer_required
 def serve_book(book_id, book_format, anyname):
     book_format = book_format.split(".")[0]
-    book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).first()
+    book = calibre_db.get_book(book_id)
-    data = calibre_db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == book_format.upper()) \
+    data = calibre_db.get_book_format(book.id, book_format.upper())
-        .first()
     log.info('Serving book: %s', data.name)
     if config.config_use_google_drive:
         headers = Headers()
@@ -1514,7 +1534,7 @@ def token_verified():
 @login_required
 def profile():
     downloads = list()
-    languages = speaking_language()
+    languages = calibre_db.speaking_language()
     translations = babel.list_translations() + [LC('en')]
     kobo_support = feature_support['kobo'] and config.config_kobo_sync
     if feature_support['oauth']:
@@ -1523,9 +1543,9 @@ def profile():
         oauth_status = None
 
     for book in current_user.downloads:
-        downloadBook = calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).first()
+        downloadBook = calibre_db.get_book(book.book_id)
         if downloadBook:
-            downloads.append(calibre_db.session.query(db.Books).filter(db.Books.id == book.book_id).first())
+            downloads.append(downloadBook)
         else:
             ub.delete_download(book.book_id)
     if request.method == "POST":
@@ -1604,7 +1624,7 @@ def profile():
 @login_required_if_no_ano
 @viewer_required
 def read_book(book_id, book_format):
-    book = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+    book = calibre_db.get_filtered_book(book_id)
     if not book:
         flash(_(u"Error opening eBook. File does not exist or file is not accessible:"), category="error")
         log.debug(u"Error opening eBook. File does not exist or file is not accessible:")
@@ -1628,7 +1648,7 @@ def read_book(book_id, book_format):
     else:
         for fileExt in constants.EXTENSIONS_AUDIO:
             if book_format.lower() == fileExt:
-                entries = calibre_db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+                entries = calibre_db.get_filtered_book(book_id)
                 log.debug(u"Start mp3 listening for %d", book_id)
                 return render_title_template('listenmp3.html', mp3file=book_id, audioformat=book_format.lower(),
                                              title=_(u"Read a Book"), entry=entries, bookmark=bookmark)
@@ -1654,8 +1674,7 @@ def read_book(book_id, book_format):
 @web.route("/book/<int:book_id>")
 @login_required_if_no_ano
 def show_book(book_id):
-    entries = calibre_db.session.query(db.Books).filter(and_(db.Books.id == book_id,
+    entries = calibre_db.get_filtered_book(book_id, allow_show_archived=True)
-                                                     common_filters(allow_show_archived=True))).first()
     if entries:
         for index in range(0, len(entries.languages)):
             try:

cps/worker.py

@@ -291,7 +291,7 @@ class WorkerThread(threading.Thread):
         w_session = Session()
         engine.execute("attach database '{}' as calibre;".format(dbpath))'''
         file_path = self.queue[index]['file_path']
-        bookid = self.queue[index]['bookid']
+        book_id = self.queue[index]['bookid']
         format_old_ext = u'.' + self.queue[index]['settings']['old_book_format'].lower()
         format_new_ext = u'.' + self.queue[index]['settings']['new_book_format'].lower()
 
@@ -299,15 +299,15 @@ class WorkerThread(threading.Thread):
         # if it does - mark the conversion task as complete and return a success
         # this will allow send to kindle workflow to continue to work
         if os.path.isfile(file_path + format_new_ext):
-            log.info("Book id %d already converted to %s", bookid, format_new_ext)
+            log.info("Book id %d already converted to %s", book_id, format_new_ext)
-            cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == bookid).first()
+            cur_book = calibre_db.get_book(book_id)
             self.queue[index]['path'] = file_path
             self.queue[index]['title'] = cur_book.title
             self._handleSuccess()
             return file_path + format_new_ext
         else:
             log.info("Book id %d - target format of %s does not exist. Moving forward with convert.",
-                     bookid,
+                     book_id,
                      format_new_ext)
 
         if config.config_kepubifypath and format_old_ext == '.epub' and format_new_ext == '.kepub':
@@ -324,13 +324,13 @@ class WorkerThread(threading.Thread):
             check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, index)
 
         if check == 0:
-            cur_book = calibre_db.session.query(db.Books).filter(db.Books.id == bookid).first()
+            cur_book = calibre_db.get_book(book_id)
             if os.path.isfile(file_path + format_new_ext):
                 # self.db_queue.join()
                 new_format = db.Data(name=cur_book.data[0].name,
                                          book_format=self.queue[index]['settings']['new_book_format'].upper(),
-                                         book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext))
+                                         book=book_id, uncompressed_size=os.path.getsize(file_path + format_new_ext))
-                task = {'task':'add_format','id': bookid, 'format': new_format}
+                task = {'task':'add_format','id': book_id, 'format': new_format}
                 self.db_queue.put(task)
                 # To Do how to handle error?