From 18d16f9a8bdb443a1d18d652952a9232978da18f Mon Sep 17 00:00:00 2001 From: blitzmann Date: Fri, 11 Sep 2020 22:52:40 -0400 Subject: [PATCH 1/4] Initial attempt at setting up CalibreDB as a class that carries the engine and DB connection, and the instance being the session --- cps/__init__.py | 5 +- cps/db.py | 156 ++++++++++++++++++++++++++++-------------------- 2 files changed, 93 insertions(+), 68 deletions(-) diff --git a/cps/__init__.py b/cps/__init__.py index a879da0a..28c2ebd4 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -83,6 +83,8 @@ log = logger.create() from . import services +db.CalibreDB.setup_db(config, cli.settingspath) + calibre_db = db.CalibreDB() def create_app(): @@ -91,7 +93,7 @@ def create_app(): if sys.version_info < (3, 0): app.static_folder = app.static_folder.decode('utf-8') app.root_path = app.root_path.decode('utf-8') - app.instance_path = app.instance_path .decode('utf-8') + app.instance_path = app.instance_path.decode('utf-8') cache_buster.init_cache_busting(app) @@ -101,7 +103,6 @@ def create_app(): app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session)) web_server.init_app(app, config) - calibre_db.setup_db(config, cli.settingspath) babel.init_app(app) _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations()) diff --git a/cps/db.py b/cps/db.py index debfe288..88388e44 100644 --- a/cps/db.py +++ b/cps/db.py @@ -42,8 +42,11 @@ from flask_babel import gettext as _ from . import logger, ub, isoLanguages from .pagination import Pagination +from weakref import WeakSet + try: import unidecode + use_unidecode = True except ImportError: use_unidecode = False @@ -56,34 +59,34 @@ cc_classes = {} Base = declarative_base() books_authors_link = Table('books_authors_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('author', Integer, ForeignKey('authors.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('author', Integer, ForeignKey('authors.id'), primary_key=True) + ) books_tags_link = Table('books_tags_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('tag', Integer, ForeignKey('tags.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('tag', Integer, ForeignKey('tags.id'), primary_key=True) + ) books_series_link = Table('books_series_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('series', Integer, ForeignKey('series.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('series', Integer, ForeignKey('series.id'), primary_key=True) + ) books_ratings_link = Table('books_ratings_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True) + ) books_languages_link = Table('books_languages_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True) + ) books_publishers_link = Table('books_publishers_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True) + ) class Identifiers(Base): @@ -99,7 +102,7 @@ class Identifiers(Base): self.type = id_type self.book = book - #def get(self): + # def get(self): # return {self.type: self.val} def formatType(self): @@ -278,7 +281,7 @@ class Publishers(Base): class Data(Base): __tablename__ = 'data' - __table_args__ = {'schema':'calibre'} + __table_args__ = {'schema': 'calibre'} id = Column(Integer, primary_key=True) book = Column(Integer, ForeignKey('books.id'), nullable=False) @@ -303,7 +306,7 @@ class Data(Base): class Books(Base): __tablename__ = 'books' - DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0) # ("0101-01-01 00:00:00+00:00") + DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0) # ("0101-01-01 00:00:00+00:00") id = Column(Integer, primary_key=True, autoincrement=True) title = Column(String(collation='NOCASE'), nullable=False, default='Unknown') @@ -342,7 +345,7 @@ class Books(Base): self.path = path self.has_cover = (has_cover != None) - #def as_dict(self): + # def as_dict(self): # return {c.name: getattr(self, c.name) for c in self.__table__.columns} def __repr__(self): @@ -354,6 +357,7 @@ class Books(Base): def atom_timestamp(self): return (self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '') + class Custom_Columns(Base): __tablename__ = 'custom_columns' @@ -373,6 +377,7 @@ class Custom_Columns(Base): display_dict['enum_values'] = [x.decode('unicode_escape') for x in display_dict['enum_values']] return display_dict + class AlchemyEncoder(json.JSONEncoder): def default(self, obj): @@ -385,15 +390,15 @@ class AlchemyEncoder(json.JSONEncoder): data = obj.__getattribute__(field) try: if isinstance(data, str): - data = data.replace("'","\'") + data = data.replace("'", "\'") elif isinstance(data, InstrumentedList): - el =list() + el = list() for ele in data: if ele.get: el.append(ele.get()) else: el.append(json.dumps(ele, cls=AlchemyEncoder)) - data =",".join(el) + data = ",".join(el) if data == '[]': data = "" else: @@ -408,16 +413,28 @@ class AlchemyEncoder(json.JSONEncoder): class CalibreDB(): + _init = False + engine = None + log = None # todo: ??? this isn't used, and even then, not sure if it's supposed to be per session or what + config = None + session_factory = None + instances = WeakSet() def __init__(self): - self.engine = None - self.session = None - self.log = None - self.config = None - - def setup_db(self, config, app_db_path): - self.config = config - self.dispose() + """ Initialize a new CalibreDB session + """ + if not self._init: + raise Exception("CalibreDB not initialized") + self.session = self.session_factory() + self.instances.add(self) + self.update_title_sort(self.config) + + @classmethod + def setup_db(cls, config, app_db_path): + cls.config = config + cls.dispose() + + # todo: remove...? global Session if not config.config_calibre_dir: @@ -430,22 +447,21 @@ class CalibreDB(): return False try: - self.engine = create_engine('sqlite://', - echo=False, - isolation_level="SERIALIZABLE", - connect_args={'check_same_thread': False}, - poolclass=StaticPool) - self.engine.execute("attach database '{}' as calibre;".format(dbpath)) - self.engine.execute("attach database '{}' as app_settings;".format(app_db_path)) - - conn = self.engine.connect() + cls.engine = create_engine('sqlite://', + echo=False, + isolation_level="SERIALIZABLE", + connect_args={'check_same_thread': False}, + poolclass=StaticPool) + cls.engine.execute("attach database '{}' as calibre;".format(dbpath)) + cls.engine.execute("attach database '{}' as app_settings;".format(app_db_path)) + + conn = cls.engine.connect() # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302 except Exception as e: config.invalidate(e) return False config.db_configured = True - self.update_title_sort(config, conn.connection) if not cc_classes: cc = conn.execute("SELECT id, datatype FROM custom_columns") @@ -460,12 +476,12 @@ class CalibreDB(): 'book': Column(Integer, ForeignKey('books.id'), primary_key=True), 'map_value': Column('value', Integer, - ForeignKey('custom_column_' + - str(row.id) + '.id'), - primary_key=True), + ForeignKey('custom_column_' + + str(row.id) + '.id'), + primary_key=True), 'extra': Column(Float), - 'asoc' : relationship('custom_column_' + str(row.id), uselist=False), - 'value' : association_proxy('asoc', 'value') + 'asoc': relationship('custom_column_' + str(row.id), uselist=False), + 'value': association_proxy('asoc', 'value') } books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'), (Base,), dicttable) @@ -501,7 +517,7 @@ class CalibreDB(): 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]], primaryjoin=( - Books.id == cc_classes[cc_id[0]].book), + Books.id == cc_classes[cc_id[0]].book), backref='books')) elif (cc_id[1] == 'series'): setattr(Books, @@ -515,17 +531,17 @@ class CalibreDB(): secondary=books_custom_column_links[cc_id[0]], backref='books')) - Session = scoped_session(sessionmaker(autocommit=False, - autoflush=True, - bind=self.engine)) - self.session = Session() + cls.session_factory = scoped_session(sessionmaker(autocommit=False, + autoflush=True, + bind=cls.engine)) + cls._init = True return True def get_book(self, book_id): return self.session.query(Books).filter(Books.id == book_id).first() def get_filtered_book(self, book_id, allow_show_archived=False): - return self.session.query(Books).filter(Books.id == book_id).\ + return self.session.query(Books).filter(Books.id == book_id). \ filter(self.common_filters(allow_show_archived)).first() def get_book_by_uuid(self, book_uuid): @@ -575,7 +591,8 @@ class CalibreDB(): def fill_indexpage(self, page, pagesize, database, db_filter, order, *join): return self.fill_indexpage_with_archived_books(page, pagesize, database, db_filter, order, False, *join) - def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived, *join): + def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived, + *join): pagesize = pagesize or self.config.config_books_per_page if current_user.show_detail_random(): randm = self.session.query(Books) \ @@ -630,7 +647,7 @@ class CalibreDB(): for authorterm in authorterms: q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) - return self.session.query(Books)\ + return self.session.query(Books) \ .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first() # read search results from calibre-database and return it (function is used for feed and simple search @@ -687,17 +704,23 @@ class CalibreDB(): conn = conn or self.session.connection().connection.connection conn.create_function("title_sort", 1, _title_sort) - def dispose(self): + @classmethod + def dispose(cls): # global session - old_session = self.session - self.session = None - if old_session: - try: old_session.close() - except: pass - if old_session.bind: - try: old_session.bind.dispose() - except Exception: pass + for inst in cls.instances: + old_session = inst.session + inst.session = None + if old_session: + try: + old_session.close() + except: + pass + if old_session.bind: + try: + old_session.bind.dispose() + except Exception: + pass for attr in list(Books.__dict__.keys()): if attr.startswith("custom_column_"): @@ -714,10 +737,11 @@ class CalibreDB(): Base.metadata.remove(table) def reconnect_db(self, config, app_db_path): - self.session.close() + self.dispose() self.engine.dispose() self.setup_db(config, app_db_path) + def lcase(s): try: return unidecode.unidecode(s.lower()) From 032cb593880febabf9b4af02f2102767ab7595d6 Mon Sep 17 00:00:00 2001 From: blitzmann Date: Sun, 13 Sep 2020 13:16:11 -0400 Subject: [PATCH 2/4] Fix resetting the session when first configuring the calibre-db on first boot up --- cps/db.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cps/db.py b/cps/db.py index 88388e44..48d2247d 100644 --- a/cps/db.py +++ b/cps/db.py @@ -423,10 +423,15 @@ class CalibreDB(): def __init__(self): """ Initialize a new CalibreDB session """ - if not self._init: - raise Exception("CalibreDB not initialized") - self.session = self.session_factory() + self.session = None + if self._init: + self.initSession() + self.instances.add(self) + + + def initSession(self): + self.session = self.session_factory() self.update_title_sort(self.config) @classmethod @@ -534,6 +539,9 @@ class CalibreDB(): cls.session_factory = scoped_session(sessionmaker(autocommit=False, autoflush=True, bind=cls.engine)) + for inst in cls.instances: + inst.initSession() + cls._init = True return True From 76c724c78355605849bf02bdc68e68fedcf79d2d Mon Sep 17 00:00:00 2001 From: blitzmann Date: Sun, 13 Sep 2020 21:37:31 -0400 Subject: [PATCH 3/4] Remove global session object, this is now wrapped in the CalibreDB class --- cps/db.py | 5 ----- cps/tasks/convert.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cps/db.py b/cps/db.py index 48d2247d..66795f7e 100644 --- a/cps/db.py +++ b/cps/db.py @@ -51,8 +51,6 @@ try: except ImportError: use_unidecode = False -Session = None - cc_exceptions = ['datetime', 'comments', 'composite', 'series'] cc_classes = {} @@ -439,9 +437,6 @@ class CalibreDB(): cls.config = config cls.dispose() - # todo: remove...? - global Session - if not config.config_calibre_dir: config.invalidate() return False diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 2b679fc0..8179de9f 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -53,7 +53,7 @@ class TaskConvert(CalibreTask): def _convert_ebook_format(self): error_message = None - local_session = db.Session() + local_session = db.CalibreDB() file_path = self.file_path book_id = self.bookid format_old_ext = u'.' + self.settings['old_book_format'].lower() From 0480edce2a834b9fa1e92999341126699b28f208 Mon Sep 17 00:00:00 2001 From: blitzmann Date: Fri, 18 Sep 2020 21:52:45 -0400 Subject: [PATCH 4/4] Clarify need for WeakSet --- cps/db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cps/db.py b/cps/db.py index 66795f7e..891b43a3 100644 --- a/cps/db.py +++ b/cps/db.py @@ -413,9 +413,10 @@ class AlchemyEncoder(json.JSONEncoder): class CalibreDB(): _init = False engine = None - log = None # todo: ??? this isn't used, and even then, not sure if it's supposed to be per session or what config = None session_factory = None + # This is a WeakSet so that references here don't keep other CalibreDB + # instances alive once they reach the end of their respective scopes instances = WeakSet() def __init__(self):