simoon
/
parallel-library
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge remote-tracking branch 'worker/db' into Develop

Browse Source 
# Conflicts:
#	cps/tasks/convert.py
pull/1539/merge
OzzieIsaacs 4 years ago
parent 4b68a6ff23 0480edce2a
commit 3b8e5ddfb3
4 changed files with 878 additions and 455 deletions
Show Stats Download Patch File Download Diff File

5
cps/__init__.py
Unescape Escape View File

@ -83,6 +83,8 @@ log = logger.create()
from . import services
db.CalibreDB.setup_db(config, cli.settingspath)
calibre_db = db.CalibreDB()
def create_app():
@ -91,7 +93,7 @@ def create_app():
if sys.version_info < (3, 0):
app.static_folder = app.static_folder.decode('utf-8')
app.root_path = app.root_path.decode('utf-8')
app.instance_path = app.instance_path .decode('utf-8')
app.instance_path = app.instance_path.decode('utf-8')
cache_buster.init_cache_busting(app)
@ -101,7 +103,6 @@ def create_app():
app.secret_key = os.getenv('SECRET_KEY', config_sql.get_flask_session_key(ub.session))
web_server.init_app(app, config)
calibre_db.setup_db(config, cli.settingspath)
babel.init_app(app)
_BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations())

162
cps/db.py
Unescape Escape View File

@ -42,48 +42,49 @@ from flask_babel import gettext as _
from . import logger, ub, isoLanguages
from .pagination import Pagination
from weakref import WeakSet
try:
import unidecode
use_unidecode = True
except ImportError:
use_unidecode = False
Session = None
cc_exceptions = ['datetime', 'comments', 'composite', 'series']
cc_classes = {}
Base = declarative_base()
books_authors_link = Table('books_authors_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('author', Integer, ForeignKey('authors.id'), primary_key=True)
)
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('author', Integer, ForeignKey('authors.id'), primary_key=True)
)
books_tags_link = Table('books_tags_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('tag', Integer, ForeignKey('tags.id'), primary_key=True)
)
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('tag', Integer, ForeignKey('tags.id'), primary_key=True)
)
books_series_link = Table('books_series_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('series', Integer, ForeignKey('series.id'), primary_key=True)
)
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('series', Integer, ForeignKey('series.id'), primary_key=True)
)
books_ratings_link = Table('books_ratings_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True)
)
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True)
)
books_languages_link = Table('books_languages_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True)
)
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True)
)
books_publishers_link = Table('books_publishers_link', Base.metadata,
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True)
)
Column('book', Integer, ForeignKey('books.id'), primary_key=True),
Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True)
)
class Identifiers(Base):
@ -99,7 +100,7 @@ class Identifiers(Base):
self.type = id_type
self.book = book
#def get(self):
# def get(self):
# return {self.type: self.val}
def formatType(self):
@ -290,7 +291,7 @@ class Publishers(Base):
class Data(Base):
__tablename__ = 'data'
__table_args__ = {'schema':'calibre'}
__table_args__ = {'schema': 'calibre'}
id = Column(Integer, primary_key=True)
book = Column(Integer, ForeignKey('books.id'), nullable=False)
@ -315,7 +316,7 @@ class Data(Base):
class Books(Base):
__tablename__ = 'books'
DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0) # ("0101-01-01 00:00:00+00:00")
DEFAULT_PUBDATE = datetime(101, 1, 1, 0, 0, 0, 0) # ("0101-01-01 00:00:00+00:00")
id = Column(Integer, primary_key=True, autoincrement=True)
title = Column(String(collation='NOCASE'), nullable=False, default='Unknown')
@ -354,7 +355,7 @@ class Books(Base):
self.path = path
self.has_cover = (has_cover != None)
#def as_dict(self):
# def as_dict(self):
# return {c.name: getattr(self, c.name) for c in self.__table__.columns}
def __repr__(self):
@ -366,6 +367,7 @@ class Books(Base):
def atom_timestamp(self):
return (self.timestamp.strftime('%Y-%m-%dT%H:%M:%S+00:00') or '')
class Custom_Columns(Base):
__tablename__ = 'custom_columns'
@ -385,6 +387,7 @@ class Custom_Columns(Base):
display_dict['enum_values'] = [x.decode('unicode_escape') for x in display_dict['enum_values']]
return display_dict
class AlchemyEncoder(json.JSONEncoder):
def default(self, obj):
@ -397,15 +400,15 @@ class AlchemyEncoder(json.JSONEncoder):
data = obj.__getattribute__(field)
try:
if isinstance(data, str):
data = data.replace("'","\'")
data = data.replace("'", "\'")
elif isinstance(data, InstrumentedList):
el =list()
el = list()
for ele in data:
if ele.get:
el.append(ele.get())
else:
el.append(json.dumps(ele, cls=AlchemyEncoder))
data =",".join(el)
data = ",".join(el)
if data == '[]':
data = ""
else:
@ -420,17 +423,32 @@ class AlchemyEncoder(json.JSONEncoder):
class CalibreDB():
_init = False
engine = None
config = None
session_factory = None
# This is a WeakSet so that references here don't keep other CalibreDB
# instances alive once they reach the end of their respective scopes
instances = WeakSet()
def __init__(self):
self.engine = None
""" Initialize a new CalibreDB session
"""
self.session = None
self.log = None
self.config = None
if self._init:
self.initSession()
self.instances.add(self)
def setup_db(self, config, app_db_path):
self.config = config
self.dispose()
global Session
def initSession(self):
self.session = self.session_factory()
self.update_title_sort(self.config)
@classmethod
def setup_db(cls, config, app_db_path):
cls.config = config
cls.dispose()
if not config.config_calibre_dir:
config.invalidate()
@ -442,22 +460,21 @@ class CalibreDB():
return False
try:
self.engine = create_engine('sqlite://',
echo=False,
isolation_level="SERIALIZABLE",
connect_args={'check_same_thread': False},
poolclass=StaticPool)
self.engine.execute("attach database '{}' as calibre;".format(dbpath))
self.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
conn = self.engine.connect()
cls.engine = create_engine('sqlite://',
echo=False,
isolation_level="SERIALIZABLE",
connect_args={'check_same_thread': False},
poolclass=StaticPool)
cls.engine.execute("attach database '{}' as calibre;".format(dbpath))
cls.engine.execute("attach database '{}' as app_settings;".format(app_db_path))
conn = cls.engine.connect()
# conn.text_factory = lambda b: b.decode(errors = 'ignore') possible fix for #1302
except Exception as e:
config.invalidate(e)
return False
config.db_configured = True
self.update_title_sort(config, conn.connection)
if not cc_classes:
cc = conn.execute("SELECT id, datatype FROM custom_columns")
@ -472,12 +489,12 @@ class CalibreDB():
'book': Column(Integer, ForeignKey('books.id'),
primary_key=True),
'map_value': Column('value', Integer,
ForeignKey('custom_column_' +
str(row.id) + '.id'),
primary_key=True),
ForeignKey('custom_column_' +
str(row.id) + '.id'),
primary_key=True),
'extra': Column(Float),
'asoc' : relationship('custom_column_' + str(row.id), uselist=False),
'value' : association_proxy('asoc', 'value')
'asoc': relationship('custom_column_' + str(row.id), uselist=False),
'value': association_proxy('asoc', 'value')
}
books_custom_column_links[row.id] = type(str('books_custom_column_' + str(row.id) + '_link'),
(Base,), dicttable)
@ -513,7 +530,7 @@ class CalibreDB():
'custom_column_' + str(cc_id[0]),
relationship(cc_classes[cc_id[0]],
primaryjoin=(
Books.id == cc_classes[cc_id[0]].book),
Books.id == cc_classes[cc_id[0]].book),
backref='books'))
elif (cc_id[1] == 'series'):
setattr(Books,
@ -527,17 +544,20 @@ class CalibreDB():
secondary=books_custom_column_links[cc_id[0]],
backref='books'))
Session = scoped_session(sessionmaker(autocommit=False,
autoflush=True,
bind=self.engine))
self.session = Session()
cls.session_factory = scoped_session(sessionmaker(autocommit=False,
autoflush=True,
bind=cls.engine))
for inst in cls.instances:
inst.initSession()
cls._init = True
return True
def get_book(self, book_id):
return self.session.query(Books).filter(Books.id == book_id).first()
def get_filtered_book(self, book_id, allow_show_archived=False):
return self.session.query(Books).filter(Books.id == book_id).\
return self.session.query(Books).filter(Books.id == book_id). \
filter(self.common_filters(allow_show_archived)).first()
def get_book_by_uuid(self, book_uuid):
@ -587,7 +607,8 @@ class CalibreDB():
def fill_indexpage(self, page, pagesize, database, db_filter, order, *join):
return self.fill_indexpage_with_archived_books(page, pagesize, database, db_filter, order, False, *join)
def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived, *join):
def fill_indexpage_with_archived_books(self, page, pagesize, database, db_filter, order, allow_show_archived,
*join):
pagesize = pagesize or self.config.config_books_per_page
if current_user.show_detail_random():
randm = self.session.query(Books) \
@ -642,7 +663,7 @@ class CalibreDB():
for authorterm in authorterms:
q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%")))
return self.session.query(Books)\
return self.session.query(Books) \
.filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first()
# read search results from calibre-database and return it (function is used for feed and simple search
@ -702,17 +723,23 @@ class CalibreDB():
conn = conn or self.session.connection().connection.connection
conn.create_function("title_sort", 1, _title_sort)
def dispose(self):
@classmethod
def dispose(cls):
# global session
old_session = self.session
self.session = None
if old_session:
try: old_session.close()
except: pass
if old_session.bind:
try: old_session.bind.dispose()
except Exception: pass
for inst in cls.instances:
old_session = inst.session
inst.session = None
if old_session:
try:
old_session.close()
except:
pass
if old_session.bind:
try:
old_session.bind.dispose()
except Exception:
pass
for attr in list(Books.__dict__.keys()):
if attr.startswith("custom_column_"):
@ -729,10 +756,11 @@ class CalibreDB():
Base.metadata.remove(table)
def reconnect_db(self, config, app_db_path):
self.session.close()
self.dispose()
self.engine.dispose()
self.setup_db(config, app_db_path)
def lcase(s):
try:
return unidecode.unidecode(s.lower())

3
cps/tasks/convert.py
Unescape Escape View File

@ -70,7 +70,8 @@ class TaskConvert(CalibreTask):
return self._handleError(str(e))
def _convert_ebook_format(self):
local_session = db.Session()
error_message = None
local_session = db.CalibreDB()
file_path = self.file_path
book_id = self.bookid
format_old_ext = u'.' + self.settings['old_book_format'].lower()

1163
test/Calibre-Web TestSummary_Linux.html
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save