diff --git a/cps/__init__.py b/cps/__init__.py index a879da0a..28c2ebd4 100644 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -83,6 +83,8 @@ log = logger.create() from . import services +db.CalibreDB.setup_db(config, cli.settingspath) + calibre_db = db.CalibreDB() def create_app(): @@ -91,7 +93,7 @@ def create_app(): if sys.version_info < (3, 0): app.static_folder = app.static_folder.decode('utf-8') app.root_path = app.root_path.decode('utf-8') - app.instance_path = app.instance_path .decode('utf-8') + app.instance_path = app.instance_path.decode('utf-8') cache_buster.init_cache_busting(app) @@ -101,7 +103,6 @@ def create_app(): app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session)) web_server.init_app(app, config) - calibre_db.setup_db(config, cli.settingspath) babel.init_app(app) _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations()) diff --git a/cps/db.py b/cps/db.py index 28b21c45..d89a38ed 100644 --- a/cps/db.py +++ b/cps/db.py @@ -42,48 +42,49 @@ from flask_babel import gettext as _ from . import logger, ub, isoLanguages from .pagination import Pagination +from weakref import WeakSet + try: import unidecode + use_unidecode = True except ImportError: use_unidecode = False -Session = None - cc_exceptions = ['datetime', 'comments', 'composite', 'series'] cc_classes = {} Base = declarative_base() books_authors_link = Table('books_authors_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('author', Integer, ForeignKey('authors.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('author', Integer, ForeignKey('authors.id'), primary_key=True) + ) books_tags_link = Table('books_tags_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('tag', Integer, ForeignKey('tags.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('tag', Integer, ForeignKey('tags.id'), primary_key=True) + ) books_series_link = Table('books_series_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('series', Integer, ForeignKey('series.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('series', Integer, ForeignKey('series.id'), primary_key=True) + ) books_ratings_link = Table('books_ratings_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True) + ) books_languages_link = Table('books_languages_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True) + ) books_publishers_link = Table('books_publishers_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True) - ) + Column('book', Integer, ForeignKey('books.id'), primary_key=True), + Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True) + ) class Identifiers(Base): @@ -99,7 +100,7 @@ class Identifiers(Base): self.type = id_type self.book = book - #def get(self): + # def get(self): # return {self.type: self.val} def formatType(self): @@ -290,7 +291,7 @@ class Publishers(Base): class Data(Base): __tablename__ = 'data' - __table_args__ = {'schema':'calibre'} + __table_args__ = {'schema': 'calibre'} id = Column(Integer, primary_key=True) book = Column(Integer, ForeignKey('books.id'), nullable=False) @@ -315,7 +316,7 @@ class Data(Base): class Books(Base): __tablename__ = 'books' - DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0) # ("0101-01-01 00:00:00+00:00") + DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0) # ("0101-01-01 00:00:00+00:00") id = Column(Integer, primary_key=True, autoincrement=True) title = Column(String(collation='NOCASE'), nullable=False, default='Unknown') @@ -354,7 +355,7 @@ class Books(Base): self.path = path self.has_cover = (has_cover != None) - #def as_dict(self): + # def as_dict(self): # return {c.name: getattr(self, c.name) for c in self.__table__.columns} def __repr__(self): @@ -366,6 +367,7 @@ class Books(Base): def atom_timestamp(self): return (self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '') + class Custom_Columns(Base): __tablename__ = 'custom_columns' @@ -385,6 +387,7 @@ class Custom_Columns(Base): display_dict['enum_values'] = [x.decode('unicode_escape') for x in display_dict['enum_values']] return display_dict + class AlchemyEncoder(json.JSONEncoder): def default(self, obj): @@ -397,15 +400,15 @@ class AlchemyEncoder(json.JSONEncoder): data = obj.__getattribute__(field) try: if isinstance(data, str): - data = data.replace("'","\'") + data = data.replace("'", "\'") elif isinstance(data, InstrumentedList): - el =list() + el = list() for ele in data: if ele.get: el.append(ele.get()) else: el.append(json.dumps(ele, cls=AlchemyEncoder)) - data =",".join(el) + data = ",".join(el) if data == '[]': data = "" else: @@ -420,17 +423,32 @@ class AlchemyEncoder(json.JSONEncoder): class CalibreDB(): + _init = False + engine = None + config = None + session_factory = None + # This is a WeakSet so that references here don't keep other CalibreDB + # instances alive once they reach the end of their respective scopes + instances = WeakSet() def __init__(self): - self.engine = None + """ Initialize a new CalibreDB session + """ self.session = None - self.log = None - self.config = None + if self._init: + self.initSession() + + self.instances.add(self) - def setup_db(self, config, app_db_path): - self.config = config - self.dispose() - global Session + + def initSession(self): + self.session = self.session_factory() + self.update_title_sort(self.config) + + @classmethod + def setup_db(cls, config, app_db_path): + cls.config = config + cls.dispose() if not config.config_calibre_dir: config.invalidate() @@ -442,22 +460,21 @@ class CalibreDB(): return False try: - self.engine = create_engine('sqlite://', - echo=False, - isolation_level="SERIALIZABLE", - connect_args={'check_same_thread': False}, - poolclass=StaticPool) - self.engine.execute("attach database '{}' as calibre;".format(dbpath)) - self.engine.execute("attach database '{}' as app_settings;".format(app_db_path)) - - conn = self.engine.connect() + cls.engine = create_engine('sqlite://', + echo=False, + isolation_level="SERIALIZABLE", + connect_args={'check_same_thread': False}, + poolclass=StaticPool) + cls.engine.execute("attach database '{}' as calibre;".format(dbpath)) + cls.engine.execute("attach database '{}' as app_settings;".format(app_db_path)) + + conn = cls.engine.connect() # conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302 except Exception as e: config.invalidate(e) return False config.db_configured = True - self.update_title_sort(config, conn.connection) if not cc_classes: cc = conn.execute("SELECT id, datatype FROM custom_columns") @@ -472,12 +489,12 @@ class CalibreDB(): 'book': Column(Integer, ForeignKey('books.id'), primary_key=True), 'map_value': Column('value', Integer, - ForeignKey('custom_column_' + - str(row.id) + '.id'), - primary_key=True), + ForeignKey('custom_column_' + + str(row.id) + '.id'), + primary_key=True), 'extra': Column(Float), - 'asoc' : relationship('custom_column_' + str(row.id), uselist=False), - 'value' : association_proxy('asoc', 'value') + 'asoc': relationship('custom_column_' + str(row.id), uselist=False), + 'value': association_proxy('asoc', 'value') } books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'), (Base,), dicttable) @@ -513,7 +530,7 @@ class CalibreDB(): 'custom_column_' + str(cc_id[0]), relationship(cc_classes[cc_id[0]], primaryjoin=( - Books.id == cc_classes[cc_id[0]].book), + Books.id == cc_classes[cc_id[0]].book), backref='books')) elif (cc_id[1] == 'series'): setattr(Books, @@ -527,17 +544,20 @@ class CalibreDB(): secondary=books_custom_column_links[cc_id[0]], backref='books')) - Session = scoped_session(sessionmaker(autocommit=False, - autoflush=True, - bind=self.engine)) - self.session = Session() + cls.session_factory = scoped_session(sessionmaker(autocommit=False, + autoflush=True, + bind=cls.engine)) + for inst in cls.instances: + inst.initSession() + + cls._init = True return True def get_book(self, book_id): return self.session.query(Books).filter(Books.id == book_id).first() def get_filtered_book(self, book_id, allow_show_archived=False): - return self.session.query(Books).filter(Books.id == book_id).\ + return self.session.query(Books).filter(Books.id == book_id). \ filter(self.common_filters(allow_show_archived)).first() def get_book_by_uuid(self, book_uuid): @@ -587,7 +607,8 @@ class CalibreDB(): def fill_indexpage(self, page, pagesize, database, db_filter, order, *join): return self.fill_indexpage_with_archived_books(page, pagesize, database, db_filter, order, False, *join) - def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived, *join): + def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived, + *join): pagesize = pagesize or self.config.config_books_per_page if current_user.show_detail_random(): randm = self.session.query(Books) \ @@ -642,7 +663,7 @@ class CalibreDB(): for authorterm in authorterms: q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) - return self.session.query(Books)\ + return self.session.query(Books) \ .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first() # read search results from calibre-database and return it (function is used for feed and simple search @@ -702,17 +723,23 @@ class CalibreDB(): conn = conn or self.session.connection().connection.connection conn.create_function("title_sort", 1, _title_sort) - def dispose(self): + @classmethod + def dispose(cls): # global session - old_session = self.session - self.session = None - if old_session: - try: old_session.close() - except: pass - if old_session.bind: - try: old_session.bind.dispose() - except Exception: pass + for inst in cls.instances: + old_session = inst.session + inst.session = None + if old_session: + try: + old_session.close() + except: + pass + if old_session.bind: + try: + old_session.bind.dispose() + except Exception: + pass for attr in list(Books.__dict__.keys()): if attr.startswith("custom_column_"): @@ -729,10 +756,11 @@ class CalibreDB(): Base.metadata.remove(table) def reconnect_db(self, config, app_db_path): - self.session.close() + self.dispose() self.engine.dispose() self.setup_db(config, app_db_path) + def lcase(s): try: return unidecode.unidecode(s.lower()) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index e3857389..20f41a43 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -70,7 +70,8 @@ class TaskConvert(CalibreTask): return self._handleError(str(e)) def _convert_ebook_format(self): - local_session = db.Session() + error_message = None + local_session = db.CalibreDB() file_path = self.file_path book_id = self.bookid format_old_ext = u'.' + self.settings['old_book_format'].lower() diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 0fc840dc..0bd45645 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2020-09-29 07:15:44

+

Start Time: 2020-10-04 19:26:20

-

Stop Time: 2020-09-29 09:05:33

+

Stop Time: 2020-10-04 21:13:13

-

Duration: 1h 32 min

+

Duration: 1h 26 min

@@ -102,12 +102,12 @@ - + TestAnonymous 13 - 13 - 0 + 12 0 + 1 0 Detail @@ -215,11 +215,31 @@ - +
TestAnonymous - test_guest_restricted_settings_visibility
- PASS + +
+ ERROR +
+ + + + @@ -511,21 +531,90 @@ AssertionError: False is not true : BMP file is not detected + + TestEbookConvertGDrive + 6 + 6 + 0 + 0 + 0 + + Detail + + + + + + + +
TestEbookConvertGDrive - test_convert_email
+ + PASS + + + + + + +
TestEbookConvertGDrive - test_convert_failed_and_email
+ + PASS + + + + + + +
TestEbookConvertGDrive - test_convert_only
+ + PASS + + + + + + +
TestEbookConvertGDrive - test_convert_parameter
+ + PASS + + + + + + +
TestEbookConvertGDrive - test_email_failed
+ + PASS + + + + + + +
TestEbookConvertGDrive - test_email_only
+ + PASS + + + + + TestEditAdditionalBooks + 12 11 - 10 0 0 1 - Detail + Detail - +
TestEditAdditionalBooks - test_change_upload_formats
@@ -534,7 +623,7 @@ AssertionError: False is not true : BMP file is not detected - +
TestEditAdditionalBooks - test_delete_book
@@ -543,7 +632,16 @@ AssertionError: False is not true : BMP file is not detected - + + +
TestEditAdditionalBooks - test_delete_role
+ + PASS + + + + +
TestEditAdditionalBooks - test_edit_book_identifier
@@ -552,7 +650,7 @@ AssertionError: False is not true : BMP file is not detected - +
TestEditAdditionalBooks - test_edit_book_identifier_capital
@@ -561,7 +659,7 @@ AssertionError: False is not true : BMP file is not detected - +
TestEditAdditionalBooks - test_edit_book_identifier_standard
@@ -570,7 +668,7 @@ AssertionError: False is not true : BMP file is not detected - +
TestEditAdditionalBooks - test_edit_special_book_identifier
@@ -579,7 +677,7 @@ AssertionError: False is not true : BMP file is not detected - +
TestEditAdditionalBooks - test_upload_edit_role
@@ -588,7 +686,7 @@ AssertionError: False is not true : BMP file is not detected - +
TestEditAdditionalBooks - test_upload_metadata_cbr
@@ -597,7 +695,7 @@ AssertionError: False is not true : BMP file is not detected - +
TestEditAdditionalBooks - test_upload_metadata_cbt
@@ -606,19 +704,19 @@ AssertionError: False is not true : BMP file is not detected - +
TestEditAdditionalBooks - test_writeonly_calibre_database
- SKIP + SKIP
-