Merge remote-tracking branch 'worker/db' into Develop

# Conflicts:
#	cps/tasks/convert.py

cps/__init__.py

@@ -83,6 +83,8 @@ log = logger.create()
 
 from . import services
 
+db.CalibreDB.setup_db(config, cli.settingspath)
+
 calibre_db = db.CalibreDB()
 
 def create_app():
@@ -101,7 +103,6 @@ def create_app():
     app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))
 
     web_server.init_app(app, config)
-    calibre_db.setup_db(config, cli.settingspath)
 
     babel.init_app(app)
     _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations())

cps/db.py

@@ -42,14 +42,15 @@ from flask_babel import gettext as _
 from . import logger, ub, isoLanguages
 from .pagination import Pagination
 
+from weakref import WeakSet
+
 try:
     import unidecode
+
     use_unidecode = True
 except ImportError:
     use_unidecode = False
 
-Session = None
-
 cc_exceptions = ['datetime', 'comments', 'composite', 'series']
 cc_classes = {}
 
@@ -366,6 +367,7 @@ class Books(Base):
     def atom_timestamp(self):
         return (self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '')
 
+
 class Custom_Columns(Base):
     __tablename__ = 'custom_columns'
 
@@ -385,6 +387,7 @@ class Custom_Columns(Base):
             display_dict['enum_values'] = [x.decode('unicode_escape') for x in display_dict['enum_values']]
         return display_dict
 
+
 class AlchemyEncoder(json.JSONEncoder):
 
     def default(self, obj):
@@ -420,17 +423,32 @@ class AlchemyEncoder(json.JSONEncoder):
 
 
 class CalibreDB():
+    _init = False
+    engine = None
+    config = None
+    session_factory = None
+    # This is a WeakSet so that references here don't keep other CalibreDB
+    # instances alive once they reach the end of their respective scopes
+    instances = WeakSet()
 
     def __init__(self):
-        self.engine = None
+        """ Initialize a new CalibreDB session
+        """
         self.session = None
-        self.log = None
+        if self._init:
-        self.config = None
+            self.initSession()
 
-    def setup_db(self, config, app_db_path):
+        self.instances.add(self)
-        self.config = config
+
-        self.dispose()
+
-        global Session
+    def initSession(self):
+        self.session = self.session_factory()
+        self.update_title_sort(self.config)
+
+    @classmethod
+    def setup_db(cls, config, app_db_path):
+        cls.config = config
+        cls.dispose()
 
         if not config.config_calibre_dir:
             config.invalidate()
@@ -442,22 +460,21 @@ class CalibreDB():
             return False
 
         try:
-            self.engine = create_engine('sqlite://',
+            cls.engine = create_engine('sqlite://',
                                        echo=False,
                                        isolation_level="SERIALIZABLE",
                                        connect_args={'check_same_thread': False},
                                        poolclass=StaticPool)
-            self.engine.execute("attach database '{}' as calibre;".format(dbpath))
+            cls.engine.execute("attach database '{}' as calibre;".format(dbpath))
-            self.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
+            cls.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
 
-            conn = self.engine.connect()
+            conn = cls.engine.connect()
             # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
         except Exception as e:
             config.invalidate(e)
             return False
 
         config.db_configured = True
-        self.update_title_sort(config, conn.connection)
 
         if not cc_classes:
             cc = conn.execute("SELECT id, datatype FROM custom_columns")
@@ -527,10 +544,13 @@ class CalibreDB():
                                          secondary=books_custom_column_links[cc_id[0]],
                                          backref='books'))
 
-        Session = scoped_session(sessionmaker(autocommit=False,
+        cls.session_factory = scoped_session(sessionmaker(autocommit=False,
                                                           autoflush=True,
-                                              bind=self.engine))
+                                                          bind=cls.engine))
-        self.session = Session()
+        for inst in cls.instances:
+            inst.initSession()
+            
+        cls._init = True
         return True
 
     def get_book(self, book_id):
@@ -587,7 +607,8 @@ class CalibreDB():
     def fill_indexpage(self, page, pagesize, database, db_filter, order, *join):
         return self.fill_indexpage_with_archived_books(page, pagesize, database, db_filter, order, False, *join)
 
-    def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived, *join):
+    def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived,
+                                           *join):
         pagesize = pagesize or self.config.config_books_per_page
         if current_user.show_detail_random():
             randm = self.session.query(Books) \
@@ -702,17 +723,23 @@ class CalibreDB():
         conn = conn or self.session.connection().connection.connection
         conn.create_function("title_sort", 1, _title_sort)
 
-    def dispose(self):
+    @classmethod
+    def dispose(cls):
         # global session
 
-        old_session = self.session
+        for inst in cls.instances:
-        self.session = None
+            old_session = inst.session
+            inst.session = None
             if old_session:
-            try: old_session.close()
+                try:
-            except: pass
+                    old_session.close()
+                except:
+                    pass
                 if old_session.bind:
-                try: old_session.bind.dispose()
+                    try:
-                except Exception: pass
+                        old_session.bind.dispose()
+                    except Exception:
+                        pass
 
         for attr in list(Books.__dict__.keys()):
             if attr.startswith("custom_column_"):
@@ -729,10 +756,11 @@ class CalibreDB():
                     Base.metadata.remove(table)
 
     def reconnect_db(self, config, app_db_path):
-        self.session.close()
+        self.dispose()
         self.engine.dispose()
         self.setup_db(config, app_db_path)
 
+
 def lcase(s):
     try:
         return unidecode.unidecode(s.lower())

cps/tasks/convert.py

@@ -70,7 +70,8 @@ class TaskConvert(CalibreTask):
                     return self._handleError(str(e))
 
     def _convert_ebook_format(self):
-        local_session = db.Session()
+        error_message = None
+        local_session = db.CalibreDB()
         file_path = self.file_path
         book_id = self.bookid
         format_old_ext = u'.' + self.settings['old_book_format'].lower()