simoon
/
parallel-library
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'Develop'

Browse Source 
# Conflicts:
#	cps/__init__.py
#	cps/about.py
#	cps/admin.py
#	cps/cli.py
#	cps/config_sql.py
#	cps/constants.py
#	cps/converter.py
#	cps/db.py
#	cps/editbooks.py
#	cps/gdriveutils.py
#	cps/helper.py
#	cps/logger.py
#	cps/oauth.py
#	cps/server.py
#	cps/services/simpleldap.py
#	cps/ub.py
#	cps/web.py
#	cps/worker.py
#	optional-requirements.txt
#	setup.cfg
#	setup.py
pull/977/head
Ozzieisaacs 6 years ago
parent 61a8eccf18 929f32335f
commit 26a7d9ef30
18 changed files with 208 additions and 242 deletions
Show Stats Download Patch File Download Diff File

1
.gitignore vendored
Unescape Escape View File

@ -14,6 +14,7 @@ build/
*.egg-info/ *.egg-info/
.installed.cfg .installed.cfg
*.egg *.egg
.pylint.d
# calibre-web # calibre-web
*.db *.db

16
cps/__init__.py
Unescape Escape View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
@ -34,8 +33,9 @@ from flask_login import LoginManager
from flask_babel import Babel from flask_babel import Babel
from flask_principal import Principal from flask_principal import Principal
from . import logger, cache_buster, cli, config_sql, ub from . import logger, cache_buster, cli, config_sql, ub, db, services
from .reverseproxy import ReverseProxied from .reverseproxy import ReverseProxied
from .server import WebServer
mimetypes.init() mimetypes.init()
@ -64,16 +64,10 @@ lm.anonymous_user = ub.Anonymous
ub.init_db(cli.settingspath) ub.init_db(cli.settingspath)
# pylint: disable=no-member
config = config_sql.load_configuration(ub.session) config = config_sql.load_configuration(ub.session)
from . import db, services
searched_ids = {} searched_ids = {}
feature_support = []
from .worker import WorkerThread
global_WorkerThread = WorkerThread()
from .server import WebServer
web_server = WebServer() web_server = WebServer()
babel = Babel() babel = Babel()
@ -109,7 +103,6 @@ def create_app():
if services.goodreads: if services.goodreads:
services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads) services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads)
global_WorkerThread.start()
return app return app
@babel.localeselector @babel.localeselector
@ -140,8 +133,7 @@ def get_locale():
@babel.timezoneselector @babel.timezoneselector
def get_timezone(): def get_timezone():
user = getattr(g, 'user', None) user = getattr(g, 'user', None)
if user is not None: return user.timezone if user else None
return user.timezone
from .updater import Updater from .updater import Updater
updater_thread = Updater() updater_thread = Updater()

67
cps/about.py
Unescape Escape View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
@ -23,56 +22,46 @@
from __future__ import division, print_function, unicode_literals from __future__ import division, print_function, unicode_literals
import sys import sys
import requests import sqlite3
from collections import OrderedDict
from flask import Blueprint import babel, pytz, requests, sqlalchemy
from flask import __version__ as flaskVersion import werkzeug, flask, flask_login, flask_principal, jinja2
from flask_babel import gettext as _ from flask_babel import gettext as _
from flask_principal import __version__ as flask_principalVersion
from flask_login import login_required
try:
from flask_login import __version__ as flask_loginVersion
except ImportError:
from flask_login.__about__ import __version__ as flask_loginVersion
from werkzeug import __version__ as werkzeugVersion
from babel import __version__ as babelVersion from . import db, converter, uploader, server, isoLanguages
from jinja2 import __version__ as jinja2Version
from pytz import __version__ as pytzVersion
from sqlalchemy import __version__ as sqlalchemyVersion
from . import db, converter, uploader
from .isoLanguages import __version__ as iso639Version
from .server import VERSION as serverVersion
from .web import render_title_template from .web import render_title_template
about = Blueprint('about', __name__) about = flask.Blueprint('about', __name__)
_VERSIONS = OrderedDict(
Python=sys.version,
WebServer=server.VERSION,
Flask=flask.__version__,
Flask_Login=flask_login.__version__,
Flask_Principal=flask_principal.__version__,
Werkzeug=werkzeug.__version__,
Babel=babel.__version__,
Jinja2=jinja2.__version__,
Requests=requests.__version__,
SqlAlchemy=sqlalchemy.__version__,
pySqlite=sqlite3.version,
SQLite=sqlite3.sqlite_version,
iso639=isoLanguages.__version__,
pytz=pytz.__version__,
)
_VERSIONS.update(uploader.get_versions())
@about.route("/stats") @about.route("/stats")
@login_required @flask_login.login_required
def stats(): def stats():
counter = db.session.query(db.Books).count() counter = db.session.query(db.Books).count()
authors = db.session.query(db.Authors).count() authors = db.session.query(db.Authors).count()
categorys = db.session.query(db.Tags).count() categorys = db.session.query(db.Tags).count()
series = db.session.query(db.Series).count() series = db.session.query(db.Series).count()
versions = uploader.get_versions() _VERSIONS['ebook converter'] = _(converter.get_version())
versions['Babel'] = 'v' + babelVersion return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=_VERSIONS,
versions['Sqlalchemy'] = 'v' + sqlalchemyVersion
versions['Werkzeug'] = 'v' + werkzeugVersion
versions['Jinja2'] = 'v' + jinja2Version
versions['Flask'] = 'v' + flaskVersion
versions['Flask Login'] = 'v' + flask_loginVersion
versions['Flask Principal'] = 'v' + flask_principalVersion
versions['Iso 639'] = 'v' + iso639Version
versions['pytz'] = 'v' + pytzVersion
versions['Requests'] = 'v' + requests.__version__
versions['pySqlite'] = 'v' + db.session.bind.dialect.dbapi.version
versions['Sqlite'] = 'v' + db.session.bind.dialect.dbapi.sqlite_version
versions.update(converter.versioncheck())
versions.update(serverVersion)
versions['Python'] = sys.version
return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=versions,
categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat") categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat")

4
cps/admin.py
Unescape Escape View File

@ -27,10 +27,6 @@ import base64
import json import json
import time import time
from datetime import datetime, timedelta from datetime import datetime, timedelta
# try:
# from imp import reload
# except ImportError:
# pass
from babel import Locale as LC from babel import Locale as LC
from babel.dates import format_datetime from babel.dates import format_datetime

60
cps/cli.py
Unescape Escape View File

@ -1,7 +1,5 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
# Copyright (C) 2018 OzzieIsaacs # Copyright (C) 2018 OzzieIsaacs
# #
@ -22,52 +20,19 @@ from __future__ import division, print_function, unicode_literals
import sys import sys
import os import os
import argparse import argparse
import socket
from .constants import CONFIG_DIR as _CONFIG_DIR from .constants import CONFIG_DIR as _CONFIG_DIR
from .constants import STABLE_VERSION as _STABLE_VERSION from .constants import STABLE_VERSION as _STABLE_VERSION
from .constants import NIGHTLY_VERSION as _NIGHTLY_VERSION from .constants import NIGHTLY_VERSION as _NIGHTLY_VERSION
VALID_CHARACTERS = 'ABCDEFabcdef:0123456789'
ipv6 = False
def version_info(): def version_info():
if _NIGHTLY_VERSION[1].startswith('$Format'): if _NIGHTLY_VERSION[1].startswith('$Format'):
return "Calibre-Web version: %s - unkown git-clone" % _STABLE_VERSION['version'] return "Calibre-Web version: %s - unkown git-clone" % _STABLE_VERSION['version']
else:
return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1]) return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1])
def validate_ip4(address):
address_list = address.split('.')
if len(address_list) != 4:
return False
for val in address_list:
if not val.isdigit():
return False
i = int(val)
if i < 0 or i > 255:
return False
return True
def validate_ip6(address):
address_list = address.split(':')
return (
len(address_list) == 8
and all(len(current) <= 4 for current in address_list)
and all(current in VALID_CHARACTERS for current in address)
)
def validate_ip(address):
if validate_ip4(address) or ipv6:
return address
print("IP address is invalid. Exiting")
sys.exit(1)
parser = argparse.ArgumentParser(description='Calibre Web is a web app' parser = argparse.ArgumentParser(description='Calibre Web is a web app'
' providing a interface for browsing, reading and downloading eBooks\n', prog='cps.py') ' providing a interface for browsing, reading and downloading eBooks\n', prog='cps.py')
parser.add_argument('-p', metavar='path', help='path and name to settings db, e.g. /opt/cw.db') parser.add_argument('-p', metavar='path', help='path and name to settings db, e.g. /opt/cw.db')
@ -108,7 +73,7 @@ if args.c:
print("Certfilepath is invalid. Exiting...") print("Certfilepath is invalid. Exiting...")
sys.exit(1) sys.exit(1)
if args.c is "": if args.c == "":
certfilepath = "" certfilepath = ""
if args.k: if args.k:
@ -122,15 +87,26 @@ if (args.k and not args.c) or (not args.k and args.c):
print("Certfile and Keyfile have to be used together. Exiting...") print("Certfile and Keyfile have to be used together. Exiting...")
sys.exit(1) sys.exit(1)
if args.k is "": if args.k == "":
keyfilepath = "" keyfilepath = ""
# handle and check ipadress argument # handle and check ipadress argument
if args.i: ipadress = args.i or None
ipv6 = validate_ip6(args.i) if ipadress:
ipadress = validate_ip(args.i) try:
# try to parse the given ip address with socket
if hasattr(socket, 'inet_pton'):
if ':' in ipadress:
socket.inet_pton(socket.AF_INET6, ipadress)
else: else:
ipadress = None socket.inet_pton(socket.AF_INET, ipadress)
else:
# on windows python < 3.4, inet_pton is not available
# inet_atom only handles IPv4 addresses
socket.inet_aton(ipadress)
except socket.error as err:
print(ipadress, ':', err)
sys.exit(1)
# handle and check user password argument # handle and check user password argument
user_password = args.s or None user_password = args.s or None

14
cps/config_sql.py
Unescape Escape View File

@ -147,9 +147,6 @@ class _ConfigSQL(object):
def get_config_ipaddress(self): def get_config_ipaddress(self):
return cli.ipadress or "" return cli.ipadress or ""
def get_ipaddress_type(self):
return cli.ipv6
def _has_role(self, role_flag): def _has_role(self, role_flag):
return constants.has_flag(self.config_default_role, role_flag) return constants.has_flag(self.config_default_role, role_flag)
@ -236,8 +233,14 @@ class _ConfigSQL(object):
if self.config_google_drive_watch_changes_response: if self.config_google_drive_watch_changes_response:
self.config_google_drive_watch_changes_response = json.loads(self.config_google_drive_watch_changes_response) self.config_google_drive_watch_changes_response = json.loads(self.config_google_drive_watch_changes_response)
self.db_configured = (self.config_calibre_dir and
(not self.config_use_google_drive or os.path.exists(self.config_calibre_dir + '/metadata.db'))) have_metadata_db = bool(self.config_calibre_dir)
if have_metadata_db:
if not self.config_use_google_drive:
db_file = os.path.join(self.config_calibre_dir, 'metadata.db')
have_metadata_db = os.path.isfile(db_file)
self.db_configured = have_metadata_db
logger.setup(self.config_logfile, self.config_log_level) logger.setup(self.config_logfile, self.config_log_level)
def save(self): def save(self):
@ -274,6 +277,7 @@ def _migrate_table(session, orm_class):
log.debug("%s: %s", column_name, err) log.debug("%s: %s", column_name, err)
column_default = "" if column.default is None else ("DEFAULT %r" % column.default.arg) column_default = "" if column.default is None else ("DEFAULT %r" % column.default.arg)
alter_table = "ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__, column_name, column.type, column_default) alter_table = "ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__, column_name, column.type, column_default)
log.debug(alter_table)
session.execute(alter_table) session.execute(alter_table)
changed = True changed = True

2
cps/constants.py
Unescape Escape View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
@ -136,4 +135,3 @@ NIGHTLY_VERSION[1] = '$Format:%cI$'
# clean-up the module namespace # clean-up the module namespace
del sys, os, namedtuple del sys, os, namedtuple

52
cps/converter.py
Unescape Escape View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
@ -21,41 +20,36 @@ from __future__ import division, print_function, unicode_literals
import os import os
import re import re
from flask_babel import gettext as _ from . import config, logger
from . import config
from .subproc_wrapper import process_wait from .subproc_wrapper import process_wait
def versionKindle(): log = logger.create()
versions = _(u'not installed')
if os.path.exists(config.config_converterpath):
try:
for lines in process_wait(config.config_converterpath):
if re.search('Amazon kindlegen\(', lines):
versions = lines
except Exception:
versions = _(u'Excecution permissions missing')
return {'kindlegen' : versions}
_NOT_CONFIGURED = 'not configured'
_NOT_INSTALLED = 'not installed'
_EXECUTION_ERROR = 'Execution permissions missing'
def versionCalibre():
versions = _(u'not installed') def _get_command_version(path, pattern, argument=None):
if os.path.exists(config.config_converterpath): if os.path.exists(path):
command = [path]
if argument:
command.append(argument)
try: try:
for lines in process_wait([config.config_converterpath, '--version']): for line in process_wait(command):
if re.search('ebook-convert.*\(calibre', lines): if re.search(pattern, line):
versions = lines return line
except Exception: except Exception as ex:
versions = _(u'Excecution permissions missing') log.warning("%s: %s", path, ex)
return {'Calibre converter' : versions} return _EXECUTION_ERROR
return _NOT_INSTALLED
def versioncheck(): def get_version():
version = None
if config.config_ebookconverter == 1: if config.config_ebookconverter == 1:
return versionKindle() version = _get_command_version(config.config_converterpath, r'Amazon kindlegen\(')
elif config.config_ebookconverter == 2: elif config.config_ebookconverter == 2:
return versionCalibre() version = _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version')
else: return version or _NOT_CONFIGURED
return {'ebook_converter':_(u'not configured')}

17
cps/db.py
Unescape Escape View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
@ -407,20 +406,18 @@ def setup_db(config):
def dispose(): def dispose():
global session global session
engine = None old_session = session
if session:
engine = session.bind
try: session.close()
except: pass
session = None session = None
if old_session:
if engine: try: old_session.close()
try: engine.dispose() except: pass
if old_session.bind:
try: old_session.bind.dispose()
except: pass except: pass
for attr in list(Books.__dict__.keys()): for attr in list(Books.__dict__.keys()):
if attr.startswith("custom_column_"): if attr.startswith("custom_column_"):
delattr(Books, attr) setattr(Books, attr, None)
for db_class in cc_classes.values(): for db_class in cc_classes.values():
Base.metadata.remove(db_class.__table__) Base.metadata.remove(db_class.__table__)

13
cps/editbooks.py
Unescape Escape View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
@ -30,12 +29,12 @@ from uuid import uuid4
from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response
from flask_babel import gettext as _ from flask_babel import gettext as _
from flask_login import current_user from flask_login import current_user, login_required
from . import constants, logger, isoLanguages, gdriveutils, uploader, helper from . import constants, logger, isoLanguages, gdriveutils, uploader, helper
from . import config, get_locale, db, ub, global_WorkerThread from . import config, get_locale, db, ub, worker
from .helper import order_authors, common_filters from .helper import order_authors, common_filters
from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required, login_required from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required
editbook = Blueprint('editbook', __name__) editbook = Blueprint('editbook', __name__)
@ -301,7 +300,7 @@ def edit_cc_data(book_id, book, to_save):
# remove old cc_val # remove old cc_val
del_cc = getattr(book, cc_string)[0] del_cc = getattr(book, cc_string)[0]
getattr(book, cc_string).remove(del_cc) getattr(book, cc_string).remove(del_cc)
if len(del_cc.books) == 0: if not del_cc.books or len(del_cc.books) == 0:
db.session.delete(del_cc) db.session.delete(del_cc)
else: else:
input_tags = to_save[cc_string].split(',') input_tags = to_save[cc_string].split(',')
@ -358,7 +357,7 @@ def upload_single_file(request, book, book_id):
# Queue uploader info # Queue uploader info
uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title) uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title)
global_WorkerThread.add_upload(current_user.nickname, worker.add_upload(current_user.nickname,
"<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>") "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>")
@ -667,7 +666,7 @@ def upload():
if error: if error:
flash(error, category="error") flash(error, category="error")
uploadText=_(u"File %(file)s uploaded", file=book.title) uploadText=_(u"File %(file)s uploaded", file=book.title)
global_WorkerThread.add_upload(current_user.nickname, worker.add_upload(current_user.nickname,
"<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>") "<a href=\"" + url_for('web.show_book', book_id=book.id) + "\">" + uploadText + "</a>")
# create data for displaying display Full language name instead of iso639.part3language # create data for displaying display Full language name instead of iso639.part3language

30
cps/helper.py
Unescape Escape View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
@ -60,7 +59,7 @@ try:
except ImportError: except ImportError:
use_PIL = False use_PIL = False
from . import logger, config, global_WorkerThread, get_locale, db, ub, isoLanguages from . import logger, config, get_locale, db, ub, isoLanguages, worker
from . import gdriveutils as gd from . import gdriveutils as gd
from .constants import STATIC_DIR as _STATIC_DIR from .constants import STATIC_DIR as _STATIC_DIR
from .pagination import Pagination from .pagination import Pagination
@ -112,7 +111,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
text = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title)) text = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title))
settings['old_book_format'] = old_book_format settings['old_book_format'] = old_book_format
settings['new_book_format'] = new_book_format settings['new_book_format'] = new_book_format
global_WorkerThread.add_convert(file_path, book.id, user_id, text, settings, kindle_mail) worker.add_convert(file_path, book.id, user_id, text, settings, kindle_mail)
return None return None
else: else:
error_message = _(u"%(format)s not found: %(fn)s", error_message = _(u"%(format)s not found: %(fn)s",
@ -121,9 +120,9 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format,
def send_test_mail(kindle_mail, user_name): def send_test_mail(kindle_mail, user_name):
global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, config.get_mail_settings(), worker.add_email(_(u'Calibre-Web test e-mail'), None, None,
kindle_mail, user_name, _(u"Test e-mail"), config.get_mail_settings(), kindle_mail, user_name,
_(u'This e-mail has been sent via Calibre-Web.')) _(u"Test e-mail"), _(u'This e-mail has been sent via Calibre-Web.'))
return return
@ -138,8 +137,9 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False):
text += "Don't forget to change your password after first login.\r\n" text += "Don't forget to change your password after first login.\r\n"
text += "Sincerely\r\n\r\n" text += "Sincerely\r\n\r\n"
text += "Your Calibre-Web team" text += "Your Calibre-Web team"
global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, config.get_mail_settings(), worker.add_email(_(u'Get Started with Calibre-Web'), None, None,
e_mail, None, _(u"Registration e-mail for user: %(name)s", name=user_name), text) config.get_mail_settings(), e_mail, None,
_(u"Registration e-mail for user: %(name)s", name=user_name), text)
return return
@ -207,13 +207,13 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id):
if convert: if convert:
# returns None if success, otherwise errormessage # returns None if success, otherwise errormessage
return convert_book_format(book_id, calibrepath, u'epub', book_format.lower(), user_id, kindle_mail) return convert_book_format(book_id, calibrepath, u'epub', book_format.lower(), user_id, kindle_mail)
else:
for entry in iter(book.data): for entry in iter(book.data):
if entry.format.upper() == book_format.upper(): if entry.format.upper() == book_format.upper():
result = entry.name + '.' + book_format.lower() converted_file_name = entry.name + '.' + book_format.lower()
global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, config.get_mail_settings(), worker.add_email(_(u"Send to Kindle"), book.path, converted_file_name,
kindle_mail, user_id, _(u"E-mail: %(book)s", book=book.title), config.get_mail_settings(), kindle_mail, user_id,
_(u'This e-mail has been sent via Calibre-Web.')) _(u"E-mail: %(book)s", book=book.title), _(u'This e-mail has been sent via Calibre-Web.'))
return return
return _(u"The requested file could not be read. Maybe wrong permissions?") return _(u"The requested file could not be read. Maybe wrong permissions?")
@ -232,7 +232,7 @@ def get_valid_filename(value, replace_whitespace=True):
value = value.replace(u'§', u'SS') value = value.replace(u'§', u'SS')
value = value.replace(u'ß', u'ss') value = value.replace(u'ß', u'ss')
value = unicodedata.normalize('NFKD', value) value = unicodedata.normalize('NFKD', value)
re_slugify = re.compile('[\W\s-]', re.UNICODE) re_slugify = re.compile(r'[\W\s-]', re.UNICODE)
if isinstance(value, str): # Python3 str, Python2 unicode if isinstance(value, str): # Python3 str, Python2 unicode
value = re_slugify.sub('', value).strip() value = re_slugify.sub('', value).strip()
else: else:
@ -254,7 +254,7 @@ def get_valid_filename(value, replace_whitespace=True):
def get_sorted_author(value): def get_sorted_author(value):
try: try:
if ',' not in value: if ',' not in value:
regexes = ["^(JR|SR)\.?$", "^I{1,3}\.?$", "^IV\.?$"] regexes = [r"^(JR|SR)\.?$", r"^I{1,3}\.?$", r"^IV\.?$"]
combined = "(" + ")|(".join(regexes) + ")" combined = "(" + ")|(".join(regexes) + ")"
value = value.split(" ") value = value.split(" ")
if re.match(combined, value[-1].upper()): if re.match(combined, value[-1].upper()):

11
cps/logger.py
Unescape Escape View File

@ -102,15 +102,20 @@ def setup(log_file, log_level=None):
log_file = _absolute_log_file(log_file, DEFAULT_LOG_FILE) log_file = _absolute_log_file(log_file, DEFAULT_LOG_FILE)
log_level = log_level or DEFAULT_LOG_LEVEL
logging.getLogger(__package__).setLevel(log_level)
r = logging.root r = logging.root
r.setLevel(log_level or DEFAULT_LOG_LEVEL) if log_level >= logging.INFO or os.environ.get('FLASK_DEBUG'):
# avoid spamming the log with debug messages from libraries
r.setLevel(log_level)
previous_handler = r.handlers[0] if r.handlers else None previous_handler = r.handlers[0] if r.handlers else None
if previous_handler: if previous_handler:
# if the log_file has not changed, don't create a new handler # if the log_file has not changed, don't create a new handler
if getattr(previous_handler, 'baseFilename', None) == log_file: if getattr(previous_handler, 'baseFilename', None) == log_file:
return return
r.debug("logging to %s level %s", log_file, r.level) logging.debug("logging to %s level %s", log_file, r.level)
if log_file == LOG_TO_STDERR: if log_file == LOG_TO_STDERR:
file_handler = StreamHandler() file_handler = StreamHandler()
@ -162,5 +167,3 @@ class StderrLogger(object):
self.buffer += message self.buffer += message
except Exception: except Exception:
self.log.debug("Logging Error") self.log.debug("Logging Error")

55
cps/server.py
Unescape Escape View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
@ -28,23 +27,30 @@ try:
from gevent.pywsgi import WSGIServer from gevent.pywsgi import WSGIServer
from gevent.pool import Pool from gevent.pool import Pool
from gevent import __version__ as _version from gevent import __version__ as _version
VERSION = {'Gevent': 'v' + _version} VERSION = 'Gevent ' + _version
_GEVENT = True _GEVENT = True
except ImportError: except ImportError:
from tornado.wsgi import WSGIContainer from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop from tornado.ioloop import IOLoop
from tornado import version as _version from tornado import version as _version
VERSION = {'Tornado': 'v' + _version} VERSION = 'Tornado ' + _version
_GEVENT = False _GEVENT = False
from . import logger, global_WorkerThread from . import logger
log = logger.create() log = logger.create()
class WebServer:
def _readable_listen_address(address, port):
if ':' in address:
address = "[" + address + "]"
return '%s:%s' % (address, port)
class WebServer(object):
def __init__(self): def __init__(self):
signal.signal(signal.SIGINT, self._killServer) signal.signal(signal.SIGINT, self._killServer)
@ -56,14 +62,12 @@ class WebServer:
self.app = None self.app = None
self.listen_address = None self.listen_address = None
self.listen_port = None self.listen_port = None
self.IPV6 = False
self.unix_socket_file = None self.unix_socket_file = None
self.ssl_args = None self.ssl_args = None
def init_app(self, application, config): def init_app(self, application, config):
self.app = application self.app = application
self.listen_address = config.get_config_ipaddress() self.listen_address = config.get_config_ipaddress()
self.IPV6 = config.get_ipaddress_type()
self.listen_port = config.config_port self.listen_port = config.config_port
if config.config_access_log: if config.config_access_log:
@ -78,8 +82,7 @@ class WebServer:
keyfile_path = config.get_config_keyfile() keyfile_path = config.get_config_keyfile()
if certfile_path and keyfile_path: if certfile_path and keyfile_path:
if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path): if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path):
self.ssl_args = {"certfile": certfile_path, self.ssl_args = dict(certfile=certfile_path, keyfile=keyfile_path)
"keyfile": keyfile_path}
else: else:
log.warning('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl.') log.warning('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl.')
log.warning('Cert path: %s', certfile_path) log.warning('Cert path: %s', certfile_path)
@ -107,32 +110,33 @@ class WebServer:
if os.name != 'nt': if os.name != 'nt':
unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET") unix_socket_file = os.environ.get("CALIBRE_UNIX_SOCKET")
if unix_socket_file: if unix_socket_file:
output = "socket:" + unix_socket_file + ":" + str(self.listen_port) return self._make_gevent_unix_socket(unix_socket_file), "unix:" + unix_socket_file
return self._make_gevent_unix_socket(unix_socket_file), output
if self.listen_address: if self.listen_address:
return (self.listen_address, self.listen_port), self._get_readable_listen_address() return (self.listen_address, self.listen_port), None
if os.name == 'nt': if os.name == 'nt':
self.listen_address = '0.0.0.0' self.listen_address = '0.0.0.0'
return (self.listen_address, self.listen_port), self._get_readable_listen_address() return (self.listen_address, self.listen_port), None
address = ('', self.listen_port)
try: try:
address = ('::', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET6) sock = WSGIServer.get_listener(address, family=socket.AF_INET6)
output = self._get_readable_listen_address(True)
except socket.error as ex: except socket.error as ex:
log.error('%s', ex) log.error('%s', ex)
log.warning('Unable to listen on "", trying on IPv4 only...') log.warning('Unable to listen on "", trying on IPv4 only...')
output = self._get_readable_listen_address(False) address = ('', self.listen_port)
sock = WSGIServer.get_listener(address, family=socket.AF_INET) sock = WSGIServer.get_listener(address, family=socket.AF_INET)
return sock, output
return sock, _readable_listen_address(*address)
def _start_gevent(self): def _start_gevent(self):
ssl_args = self.ssl_args or {} ssl_args = self.ssl_args or {}
try: try:
sock, output = self._make_gevent_socket() sock, output = self._make_gevent_socket()
if output is None:
output = _readable_listen_address(self.listen_address, self.listen_port)
log.info('Starting Gevent server on %s', output) log.info('Starting Gevent server on %s', output)
self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, spawn=Pool(), **ssl_args) self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, spawn=Pool(), **ssl_args)
self.wsgiserver.serve_forever() self.wsgiserver.serve_forever()
@ -142,7 +146,7 @@ class WebServer:
self.unix_socket_file = None self.unix_socket_file = None
def _start_tornado(self): def _start_tornado(self):
log.info('Starting Tornado server on %s', self._get_readable_listen_address()) log.info('Starting Tornado server on %s', _readable_listen_address(self.listen_address, self.listen_port))
# Max Buffersize set to 200MB ) # Max Buffersize set to 200MB )
http_server = HTTPServer(WSGIContainer(self.app), http_server = HTTPServer(WSGIContainer(self.app),
@ -154,18 +158,6 @@ class WebServer:
# wait for stop signal # wait for stop signal
self.wsgiserver.close(True) self.wsgiserver.close(True)
def _get_readable_listen_address(self, ipV6=False):
if self.listen_address == "":
listen_string = '""'
else:
ipV6 = self.IPV6
listen_string = self.listen_address
if ipV6:
adress = "[" + listen_string + "]"
else:
adress = listen_string
return adress + ":" + str(self.listen_port)
def start(self): def start(self):
try: try:
if _GEVENT: if _GEVENT:
@ -179,7 +171,6 @@ class WebServer:
return False return False
finally: finally:
self.wsgiserver = None self.wsgiserver = None
global_WorkerThread.stop()
if not self.restart: if not self.restart:
log.info("Performing shutdown of Calibre-Web") log.info("Performing shutdown of Calibre-Web")
@ -193,7 +184,7 @@ class WebServer:
os.execv(sys.executable, arguments) os.execv(sys.executable, arguments)
return True return True
def _killServer(self, signum, frame): def _killServer(self, ignored_signum, ignored_frame):
self.stop() self.stop()
def stop(self, restart=False): def stop(self, restart=False):

3
cps/services/simpleldap.py
Unescape Escape View File

@ -29,10 +29,7 @@ _ldap = LDAP()
def init_app(app, config): def init_app(app, config):
global _ldap
if config.config_login_type != constants.LOGIN_LDAP: if config.config_login_type != constants.LOGIN_LDAP:
_ldap = None
return return
app.config['LDAP_HOST'] = config.config_ldap_provider_url app.config['LDAP_HOST'] = config.config_ldap_provider_url

14
cps/ub.py
Unescape Escape View File

@ -478,13 +478,11 @@ def init_db(app_db_path):
def dispose(): def dispose():
global session global session
engine = None old_session = session
if session:
engine = session.bind
try: session.close()
except: pass
session = None session = None
if old_session:
if engine: try: old_session.close()
try: engine.dispose() except: pass
if old_session.bind:
try: old_session.bind.dispose()
except: pass except: pass

8
cps/web.py
Unescape Escape View File

@ -41,8 +41,8 @@ from werkzeug.exceptions import default_exceptions
from werkzeug.datastructures import Headers from werkzeug.datastructures import Headers
from werkzeug.security import generate_password_hash, check_password_hash from werkzeug.security import generate_password_hash, check_password_hash
from . import constants, logger, isoLanguages, services from . import constants, logger, isoLanguages, services, worker
from . import global_WorkerThread, searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app from . import searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app
from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
from .helper import common_filters, get_search_results, fill_indexpage, speaking_language, check_valid_domain, \ from .helper import common_filters, get_search_results, fill_indexpage, speaking_language, check_valid_domain, \
order_authors, get_typeahead, render_task_status, json_serial, get_cc_columns, \ order_authors, get_typeahead, render_task_status, json_serial, get_cc_columns, \
@ -245,7 +245,7 @@ def before_request():
@web.route("/ajax/emailstat") @web.route("/ajax/emailstat")
@login_required @login_required
def get_email_status_json(): def get_email_status_json():
tasks = global_WorkerThread.get_taskstatus() tasks = worker.get_taskstatus()
answer = render_task_status(tasks) answer = render_task_status(tasks)
js = json.dumps(answer, default=json_serial) js = json.dumps(answer, default=json_serial)
response = make_response(js) response = make_response(js)
@ -760,7 +760,7 @@ def category_list():
@login_required @login_required
def get_tasks_status(): def get_tasks_status():
# if current user admin, show all email, otherwise only own emails # if current user admin, show all email, otherwise only own emails
tasks = global_WorkerThread.get_taskstatus() tasks = worker.get_taskstatus()
answer = render_task_status(tasks) answer = render_task_status(tasks)
return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks") return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks")

42
cps/worker.py
Unescape Escape View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web)
@ -25,7 +24,7 @@ import smtplib
import socket import socket
import time import time
import threading import threading
from datetime import datetime, timedelta from datetime import datetime
try: try:
from StringIO import StringIO from StringIO import StringIO
@ -66,6 +65,13 @@ RET_FAIL = 0
RET_SUCCESS = 1 RET_SUCCESS = 1
def _get_main_thread():
for t in threading.enumerate():
if t.__class__.__name__ == '_MainThread':
return t
raise Exception("main thread not found?!")
# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append # For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
# it in MIME Base64 encoded to # it in MIME Base64 encoded to
def get_attachment(bookpath, filename): def get_attachment(bookpath, filename):
@ -173,7 +179,6 @@ class email_SSL(emailbase, smtplib.SMTP_SSL):
class WorkerThread(threading.Thread): class WorkerThread(threading.Thread):
def __init__(self): def __init__(self):
self._stopevent = threading.Event()
threading.Thread.__init__(self) threading.Thread.__init__(self)
self.status = 0 self.status = 0
self.current = 0 self.current = 0
@ -185,7 +190,8 @@ class WorkerThread(threading.Thread):
# Main thread loop starting the different tasks # Main thread loop starting the different tasks
def run(self): def run(self):
while not self._stopevent.isSet(): main_thread = _get_main_thread()
while main_thread.is_alive():
doLock = threading.Lock() doLock = threading.Lock()
doLock.acquire() doLock.acquire()
if self.current != self.last: if self.current != self.last:
@ -200,11 +206,9 @@ class WorkerThread(threading.Thread):
self.current += 1 self.current += 1
else: else:
doLock.release() doLock.release()
if main_thread.is_alive():
time.sleep(1) time.sleep(1)
def stop(self):
self._stopevent.set()
def get_send_status(self): def get_send_status(self):
if self.asyncSMTP: if self.asyncSMTP:
return self.asyncSMTP.getTransferStatus() return self.asyncSMTP.getTransferStatus()
@ -317,7 +321,7 @@ class WorkerThread(threading.Thread):
nextline = p.communicate()[0] nextline = p.communicate()[0]
# Format of error message (kindlegen translates its output texts): # Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting. # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) conv_error = re.search(r".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE)
# If error occoures, store error message for logfile # If error occoures, store error message for logfile
if conv_error: if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s", error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
@ -332,7 +336,7 @@ class WorkerThread(threading.Thread):
nextline = nextline.decode('utf-8') nextline = nextline.decode('utf-8')
log.debug(nextline.strip('\r\n')) log.debug(nextline.strip('\r\n'))
# parse progress string from calibre-converter # parse progress string from calibre-converter
progress = re.search("(\d+)%\s.*", nextline) progress = re.search(r"(\d+)%\s.*", nextline)
if progress: if progress:
self.UIqueue[self.current]['progress'] = progress.group(1) + ' %' self.UIqueue[self.current]['progress'] = progress.group(1) + ' %'
@ -511,3 +515,23 @@ class WorkerThread(threading.Thread):
self.UIqueue[self.current]['stat'] = STAT_FINISH_SUCCESS self.UIqueue[self.current]['stat'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['progress'] = "100 %" self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime'] self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime']
_worker = WorkerThread()
_worker.start()
def get_taskstatus():
return _worker.get_taskstatus()
def add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text):
return _worker.add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text)
def add_upload(user_name, taskMessage):
return _worker.add_upload(user_name, taskMessage)
def add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail=None):
return _worker.add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail)

7
optional-requirements.txt
Unescape Escape View File

@ -11,13 +11,20 @@ PyDrive==1.3.1
PyYAML==3.12 PyYAML==3.12
rsa==3.4.2 rsa==3.4.2
six==1.10.0 six==1.10.0
# goodreads # goodreads
goodreads>=0.3.2 goodreads>=0.3.2
python-Levenshtein>=0.12.0 python-Levenshtein>=0.12.0
# ldap login
python_ldap>=3.0.0
flask-simpleldap
# extracting metadata # extracting metadata
lxml>=3.8.0 lxml>=3.8.0
Pillow>=4.0.0 Pillow>=4.0.0
rarfile>=2.7 rarfile>=2.7
# other # other
natsort>=2.2.0 natsort>=2.2.0
git+https://github.com/OzzieIsaacs/comicapi.git@5346716578b2843f54d522f44d01bc8d25001d24#egg=comicapi git+https://github.com/OzzieIsaacs/comicapi.git@5346716578b2843f54d522f44d01bc8d25001d24#egg=comicapi

Write Preview
Loading…
Cancel
Save