Merge remote-tracking branch 'worker/db' into Develop

# Conflicts:
#	cps/tasks/convert.py

cps/__init__.py

@@ -83,6 +83,8 @@ log = logger.create()
 
 from . import services
 
+db.CalibreDB.setup_db(config, cli.settingspath)
+
 calibre_db = db.CalibreDB()
 
 def create_app():
@@ -91,7 +93,7 @@ def create_app():
     if sys.version_info < (3, 0):
         app.static_folder = app.static_folder.decode('utf-8')
         app.root_path = app.root_path.decode('utf-8')
-        app.instance_path = app.instance_path .decode('utf-8')
+        app.instance_path = app.instance_path.decode('utf-8')
 
     cache_buster.init_cache_busting(app)
 
@@ -101,7 +103,6 @@ def create_app():
     app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))
 
     web_server.init_app(app, config)
-    calibre_db.setup_db(config, cli.settingspath)
 
     babel.init_app(app)
     _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations())

cps/db.py

@@ -42,14 +42,15 @@ from flask_babel import gettext as _
 from . import logger, ub, isoLanguages
 from .pagination import Pagination
 
+from weakref import WeakSet
+
 try:
     import unidecode
+
     use_unidecode = True
 except ImportError:
     use_unidecode = False
 
-Session = None
-
 cc_exceptions = ['datetime', 'comments', 'composite', 'series']
 cc_classes = {}
 
@@ -99,7 +100,7 @@ class Identifiers(Base):
         self.type = id_type
         self.book = book
 
-    #def get(self):
+    # def get(self):
     #    return {self.type: self.val}
 
     def formatType(self):
@@ -290,7 +291,7 @@ class Publishers(Base):
 
 class Data(Base):
     __tablename__ = 'data'
-    __table_args__ = {'schema':'calibre'}
+    __table_args__ = {'schema': 'calibre'}
 
     id = Column(Integer, primary_key=True)
     book = Column(Integer, ForeignKey('books.id'), nullable=False)
@@ -354,7 +355,7 @@ class Books(Base):
         self.path = path
         self.has_cover = (has_cover != None)
 
-    #def as_dict(self):
+    # def as_dict(self):
     #    return {c.name: getattr(self, c.name) for c in self.__table__.columns}
 
     def __repr__(self):
@@ -366,6 +367,7 @@ class Books(Base):
     def atom_timestamp(self):
         return (self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '')
 
+
 class Custom_Columns(Base):
     __tablename__ = 'custom_columns'
 
@@ -385,6 +387,7 @@ class Custom_Columns(Base):
             display_dict['enum_values'] = [x.decode('unicode_escape') for x in display_dict['enum_values']]
         return display_dict
 
+
 class AlchemyEncoder(json.JSONEncoder):
 
     def default(self, obj):
@@ -397,15 +400,15 @@ class AlchemyEncoder(json.JSONEncoder):
                 data = obj.__getattribute__(field)
                 try:
                     if isinstance(data, str):
-                        data = data.replace("'","\'")
+                        data = data.replace("'", "\'")
                     elif isinstance(data, InstrumentedList):
-                        el =list()
+                        el = list()
                         for ele in data:
                             if ele.get:
                                 el.append(ele.get())
                             else:
                                 el.append(json.dumps(ele, cls=AlchemyEncoder))
-                        data =",".join(el)
+                        data = ",".join(el)
                         if data == '[]':
                             data = ""
                     else:
@@ -420,17 +423,32 @@ class AlchemyEncoder(json.JSONEncoder):
 
 
 class CalibreDB():
+    _init = False
+    engine = None
+    config = None
+    session_factory = None
+    # This is a WeakSet so that references here don't keep other CalibreDB
+    # instances alive once they reach the end of their respective scopes
+    instances = WeakSet()
 
     def __init__(self):
-        self.engine = None
+        """ Initialize a new CalibreDB session
+        """
         self.session = None
-        self.log = None
-        self.config = None
+        if self._init:
+            self.initSession()
 
-    def setup_db(self, config, app_db_path):
-        self.config = config
-        self.dispose()
-        global Session
+        self.instances.add(self)
+
+
+    def initSession(self):
+        self.session = self.session_factory()
+        self.update_title_sort(self.config)
+
+    @classmethod
+    def setup_db(cls, config, app_db_path):
+        cls.config = config
+        cls.dispose()
 
         if not config.config_calibre_dir:
             config.invalidate()
@@ -442,22 +460,21 @@ class CalibreDB():
             return False
 
         try:
-            self.engine = create_engine('sqlite://',
+            cls.engine = create_engine('sqlite://',
                                        echo=False,
                                        isolation_level="SERIALIZABLE",
                                        connect_args={'check_same_thread': False},
                                        poolclass=StaticPool)
-            self.engine.execute("attach database '{}' as calibre;".format(dbpath))
-            self.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
+            cls.engine.execute("attach database '{}' as calibre;".format(dbpath))
+            cls.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
 
-            conn = self.engine.connect()
+            conn = cls.engine.connect()
             # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
         except Exception as e:
             config.invalidate(e)
             return False
 
         config.db_configured = True
-        self.update_title_sort(config, conn.connection)
 
         if not cc_classes:
             cc = conn.execute("SELECT id, datatype FROM custom_columns")
@@ -476,8 +493,8 @@ class CalibreDB():
                                                                     str(row.id) + '.id'),
                                                          primary_key=True),
                                      'extra': Column(Float),
-                                     'asoc' : relationship('custom_column_' + str(row.id), uselist=False),
-                                     'value' : association_proxy('asoc', 'value')
+                                     'asoc': relationship('custom_column_' + str(row.id), uselist=False),
+                                     'value': association_proxy('asoc', 'value')
                                      }
                         books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'),
                                                                  (Base,), dicttable)
@@ -527,17 +544,20 @@ class CalibreDB():
                                          secondary=books_custom_column_links[cc_id[0]],
                                          backref='books'))
 
-        Session = scoped_session(sessionmaker(autocommit=False,
+        cls.session_factory = scoped_session(sessionmaker(autocommit=False,
                                                           autoflush=True,
-                                              bind=self.engine))
-        self.session = Session()
+                                                          bind=cls.engine))
+        for inst in cls.instances:
+            inst.initSession()
+            
+        cls._init = True
         return True
 
     def get_book(self, book_id):
         return self.session.query(Books).filter(Books.id == book_id).first()
 
     def get_filtered_book(self, book_id, allow_show_archived=False):
-        return self.session.query(Books).filter(Books.id == book_id).\
+        return self.session.query(Books).filter(Books.id == book_id). \
             filter(self.common_filters(allow_show_archived)).first()
 
     def get_book_by_uuid(self, book_uuid):
@@ -587,7 +607,8 @@ class CalibreDB():
     def fill_indexpage(self, page, pagesize, database, db_filter, order, *join):
         return self.fill_indexpage_with_archived_books(page, pagesize, database, db_filter, order, False, *join)
 
-    def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived, *join):
+    def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived,
+                                           *join):
         pagesize = pagesize or self.config.config_books_per_page
         if current_user.show_detail_random():
             randm = self.session.query(Books) \
@@ -642,7 +663,7 @@ class CalibreDB():
         for authorterm in authorterms:
             q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
 
-        return self.session.query(Books)\
+        return self.session.query(Books) \
             .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first()
 
     # read search results from calibre-database and return it (function is used for feed and simple search
@@ -702,17 +723,23 @@ class CalibreDB():
         conn = conn or self.session.connection().connection.connection
         conn.create_function("title_sort", 1, _title_sort)
 
-    def dispose(self):
+    @classmethod
+    def dispose(cls):
         # global session
 
-        old_session = self.session
-        self.session = None
+        for inst in cls.instances:
+            old_session = inst.session
+            inst.session = None
             if old_session:
-            try: old_session.close()
-            except: pass
+                try:
+                    old_session.close()
+                except:
+                    pass
                 if old_session.bind:
-                try: old_session.bind.dispose()
-                except Exception: pass
+                    try:
+                        old_session.bind.dispose()
+                    except Exception:
+                        pass
 
         for attr in list(Books.__dict__.keys()):
             if attr.startswith("custom_column_"):
@@ -729,10 +756,11 @@ class CalibreDB():
                     Base.metadata.remove(table)
 
     def reconnect_db(self, config, app_db_path):
-        self.session.close()
+        self.dispose()
         self.engine.dispose()
         self.setup_db(config, app_db_path)
 
+
 def lcase(s):
     try:
         return unidecode.unidecode(s.lower())

cps/tasks/convert.py

@@ -70,7 +70,8 @@ class TaskConvert(CalibreTask):
                     return self._handleError(str(e))
 
     def _convert_ebook_format(self):
-        local_session = db.Session()
+        error_message = None
+        local_session = db.CalibreDB()
         file_path = self.file_path
         book_id = self.bookid
         format_old_ext = u'.' + self.settings['old_book_format'].lower()