From 006e596c7221729c9a37b79037fa72bb88506492 Mon Sep 17 00:00:00 2001 From: Daniel Pavel Date: Sun, 7 Jul 2019 15:53:38 +0300 Subject: [PATCH] Moved config class into separate file. Moved Goodreads and LDAP services into separate package. --- cps/__init__.py | 63 +-- cps/admin.py | 744 +++++++++++++-------------------- cps/config_sql.py | 287 +++++++++++++ cps/constants.py | 5 +- cps/converter.py | 16 +- cps/db.py | 100 +++-- cps/editbooks.py | 33 +- cps/gdrive.py | 20 +- cps/gdriveutils.py | 49 +++ cps/helper.py | 88 ++-- cps/isoLanguages.py | 47 ++- cps/jinjia.py | 6 +- cps/ldap_login.py | 67 --- cps/logger.py | 5 + cps/oauth_bb.py | 2 +- cps/opds.py | 7 +- cps/server.py | 1 + cps/services/__init__.py | 36 ++ cps/services/goodreads.py | 106 +++++ cps/services/simpleldap.py | 77 ++++ cps/subproc_wrapper.py | 10 + cps/templates/admin.html | 2 +- cps/templates/config_edit.html | 8 +- cps/ub.py | 484 ++------------------- cps/updater.py | 17 +- cps/web.py | 98 ++--- 26 files changed, 1138 insertions(+), 1240 deletions(-) create mode 100644 cps/config_sql.py delete mode 100644 cps/ldap_login.py create mode 100644 cps/services/__init__.py create mode 100644 cps/services/goodreads.py create mode 100644 cps/services/simpleldap.py diff --git a/cps/__init__.py b/cps/__init__.py index 6b8815e9..5808f8ae 100755 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -25,10 +25,6 @@ from __future__ import division, print_function, unicode_literals import sys import os import mimetypes -try: - import cPickle -except ImportError: - import pickle as cPickle from babel import Locale as LC from babel import negotiate_locale @@ -38,8 +34,7 @@ from flask_login import LoginManager from flask_babel import Babel from flask_principal import Principal -from . import logger, cache_buster, ub -from .constants import TRANSLATIONS_DIR as _TRANSLATIONS_DIR +from . import logger, cache_buster, cli, config_sql, ub from .reverseproxy import ReverseProxied @@ -68,16 +63,9 @@ lm.login_view = 'web.login' lm.anonymous_user = ub.Anonymous -ub.init_db() -config = ub.Config() -from . import db - -try: - with open(os.path.join(_TRANSLATIONS_DIR, 'iso639.pickle'), 'rb') as f: - language_table = cPickle.load(f) -except cPickle.UnpicklingError as error: - print("Can't read file cps/translations/iso639.pickle: %s" % error) - sys.exit(1) +ub.init_db(cli.settingspath) +config = config_sql.load_configuration(ub.session) +from . import db, services searched_ids = {} @@ -87,10 +75,8 @@ global_WorkerThread = WorkerThread() from .server import WebServer web_server = WebServer() -from .ldap_login import Ldap -ldap1 = Ldap() - babel = Babel() +_BABEL_TRANSLATIONS = set() log = logger.create() @@ -109,30 +95,45 @@ def create_app(): Principal(app) lm.init_app(app) app.secret_key = os.getenv('SECRET_KEY', 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT') + web_server.init_app(app, config) - db.setup_db() + db.setup_db(config) + babel.init_app(app) - ldap1.init_app(app) + _BABEL_TRANSLATIONS.update(str(item) for item in babel.list_translations()) + _BABEL_TRANSLATIONS.add('en') + + if services.ldap: + services.ldap.init_app(app, config) + if services.goodreads: + services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads) + global_WorkerThread.start() return app @babel.localeselector -def get_locale(): +def negociate_locale(): # if a user is logged in, use the locale from the user settings user = getattr(g, 'user', None) # user = None if user is not None and hasattr(user, "locale"): if user.nickname != 'Guest': # if the account is the guest account bypass the config lang settings return user.locale - translations = [str(item) for item in babel.list_translations()] + ['en'] - preferred = list() - for x in request.accept_languages.values(): - try: - preferred.append(str(LC.parse(x.replace('-', '_')))) - except (UnknownLocaleError, ValueError) as e: - log.warning('Could not parse locale "%s": %s', x, e) - preferred.append('en') - return negotiate_locale(preferred, translations) + + preferred = set() + if request.accept_languages: + for x in request.accept_languages.values(): + try: + preferred.add(str(LC.parse(x.replace('-', '_')))) + except (UnknownLocaleError, ValueError) as e: + log.warning('Could not parse locale "%s": %s', x, e) + # preferred.append('en') + + return negotiate_locale(preferred or ['en'], _BABEL_TRANSLATIONS) + + +def get_locale(): + return request._locale @babel.timezoneselector diff --git a/cps/admin.py b/cps/admin.py index 38b4ad95..175b612d 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -23,13 +23,14 @@ from __future__ import division, print_function, unicode_literals import os +import base64 import json import time from datetime import datetime, timedelta -try: - from imp import reload -except ImportError: - pass +# try: +# from imp import reload +# except ImportError: +# pass from babel import Locale as LC from babel.dates import format_datetime @@ -40,21 +41,16 @@ from sqlalchemy import and_ from sqlalchemy.exc import IntegrityError from werkzeug.security import generate_password_hash -from . import constants, logger, ldap1 +from . import constants, logger, helper, services from . import db, ub, web_server, get_locale, config, updater_thread, babel, gdriveutils -from .helper import speaking_language, check_valid_domain, check_unrar, send_test_mail, generate_random_password, \ - send_registration_mail -from .gdriveutils import is_gdrive_ready, gdrive_support, downloadFile, deleteDatabaseOnChange, listRootFolders -from .web import admin_required, render_title_template, before_request, unconfigured, login_required_if_no_ano +from .helper import speaking_language, check_valid_domain, send_test_mail, generate_random_password, send_registration_mail +from .gdriveutils import is_gdrive_ready, gdrive_support +from .web import admin_required, render_title_template, before_request, unconfigured, login_required_if_no_ano -feature_support = dict() -feature_support['ldap'] = ldap1.ldap_supported() - -try: - from goodreads.client import GoodreadsClient - feature_support['goodreads'] = True -except ImportError: - feature_support['goodreads'] = False +feature_support = { + 'ldap': bool(services.ldap), + 'goodreads': bool(services.goodreads) + } # try: # import rarfile @@ -63,7 +59,7 @@ except ImportError: # feature_support['rar'] = False try: - from oauth_bb import oauth_check + from .oauth_bb import oauth_check feature_support['oauth'] = True except ImportError: feature_support['oauth'] = False @@ -86,12 +82,10 @@ def admin_forbidden(): @admin_required def shutdown(): task = int(request.args.get("parameter").strip()) - if task == 1 or task == 0: # valid commandos received + if task in (0, 1): # valid commandos received # close all database connections - db.session.close() - db.engine.dispose() - ub.session.close() - ub.engine.dispose() + db.dispose() + ub.dispose() showtext = {} if task == 0: @@ -101,13 +95,13 @@ def shutdown(): # stop gevent/tornado server web_server.stop(task == 0) return json.dumps(showtext) - else: - if task == 2: - db.session.close() - db.engine.dispose() - db.setup_db() - return json.dumps({}) - abort(404) + + if task == 2: + log.warning("reconnecting to calibre database") + db.setup_db(config) + return '{}' + + abort(404) @admi.route("/admin/view") @@ -133,8 +127,8 @@ def admin(): commit = version['version'] allUser = ub.session.query(ub.User).all() - settings = ub.session.query(ub.Settings).first() - return render_title_template("admin.html", allUser=allUser, email=settings, config=config, commit=commit, + email_settings = config.get_mail_settings() + return render_title_template("admin.html", allUser=allUser, email=email_settings, config=config, commit=commit, title=_(u"Admin page"), page="admin") @@ -142,84 +136,60 @@ def admin(): @login_required @admin_required def configuration(): - return configuration_helper(0) + if request.method == "POST": + return _configuration_update_helper() + return _configuration_result() -@admi.route("/admin/viewconfig", methods=["GET", "POST"]) +@admi.route("/admin/viewconfig") @login_required @admin_required def view_configuration(): - reboot_required = False - if request.method == "POST": - to_save = request.form.to_dict() - content = ub.session.query(ub.Settings).first() - if "config_calibre_web_title" in to_save: - content.config_calibre_web_title = to_save["config_calibre_web_title"] - if "config_columns_to_ignore" in to_save: - content.config_columns_to_ignore = to_save["config_columns_to_ignore"] - if "config_read_column" in to_save: - content.config_read_column = int(to_save["config_read_column"]) - if "config_theme" in to_save: - content.config_theme = int(to_save["config_theme"]) - if "config_title_regex" in to_save: - if content.config_title_regex != to_save["config_title_regex"]: - content.config_title_regex = to_save["config_title_regex"] - reboot_required = True - if "config_random_books" in to_save: - content.config_random_books = int(to_save["config_random_books"]) - if "config_books_per_page" in to_save: - content.config_books_per_page = int(to_save["config_books_per_page"]) - # Mature Content configuration - if "config_mature_content_tags" in to_save: - content.config_mature_content_tags = to_save["config_mature_content_tags"].strip() - if "Show_mature_content" in to_save: - content.config_default_show |= constants.MATURE_CONTENT - - if "config_authors_max" in to_save: - content.config_authors_max = int(to_save["config_authors_max"]) - - # Default user configuration - content.config_default_role = 0 - if "admin_role" in to_save: - content.config_default_role |= constants.ROLE_ADMIN - if "download_role" in to_save: - content.config_default_role |= constants.ROLE_DOWNLOAD - if "viewer_role" in to_save: - content.config_default_role |= constants.ROLE_VIEWER - if "upload_role" in to_save: - content.config_default_role |= constants.ROLE_UPLOAD - if "edit_role" in to_save: - content.config_default_role |= constants.ROLE_EDIT - if "delete_role" in to_save: - content.config_default_role |= constants.ROLE_DELETE_BOOKS - if "passwd_role" in to_save: - content.config_default_role |= constants.ROLE_PASSWD - if "edit_shelf_role" in to_save: - content.config_default_role |= constants.ROLE_EDIT_SHELFS - - val = 0 - for key, __ in to_save.items(): - if key.startswith('show'): - val |= int(key[5:]) - content.config_default_show = val - - ub.session.commit() - flash(_(u"Calibre-Web configuration updated"), category="success") - config.loadSettings() - before_request() - if reboot_required: - # db.engine.dispose() # ToDo verify correct - # ub.session.close() - # ub.engine.dispose() - # stop Server - web_server.stop(True) - log.info('Reboot required, restarting') readColumn = db.session.query(db.Custom_Columns)\ .filter(and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all() return render_title_template("config_view_edit.html", conf=config, readColumns=readColumn, title=_(u"UI Configuration"), page="uiconfig") +@admi.route("/admin/viewconfig", methods=["POST"]) +@login_required +@admin_required +def update_view_configuration(): + reboot_required = False + to_save = request.form.to_dict() + + _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y) + _config_int = lambda x: config.set_from_dictionary(to_save, x, int) + + _config_string("config_calibre_web_title") + _config_string("config_columns_to_ignore") + _config_string("config_mature_content_tags") + reboot_required |= _config_string("config_title_regex") + + _config_int("config_read_column") + _config_int("config_theme") + _config_int("config_random_books") + _config_int("config_books_per_page") + _config_int("config_authors_max") + + config.config_default_role = constants.selected_roles(to_save) + config.config_default_role &= ~constants.ROLE_ANONYMOUS + + config.config_default_show = sum(int(k[5:]) for k in to_save if k.startswith('show_')) + if "Show_mature_content" in to_save: + config.config_default_show |= constants.MATURE_CONTENT + + config.save() + flash(_(u"Calibre-Web configuration updated"), category="success") + before_request() + if reboot_required: + db.dispose() + ub.dispose() + web_server.stop(True) + + return view_configuration() + + @admi.route("/ajax/editdomain", methods=['POST']) @login_required @admin_required @@ -280,281 +250,172 @@ def list_domain(): @unconfigured def basic_configuration(): logout_user() - return configuration_helper(1) + if request.method == "POST": + return _configuration_update_helper() + return _configuration_result() -def configuration_helper(origin): +def _configuration_update_helper(): reboot_required = False - gdriveError = None db_change = False - success = False - filedata = None - if not feature_support['gdrive']: - gdriveError = _('Import of optional Google Drive requirements missing') + to_save = request.form.to_dict() + + _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y) + _config_int = lambda x: config.set_from_dictionary(to_save, x, int) + _config_checkbox = lambda x: config.set_from_dictionary(to_save, x, lambda y: y == "on", False) + _config_checkbox_int = lambda x: config.set_from_dictionary(to_save, x, lambda y: 1 if (y == "on") else 0, 0) + + db_change |= _config_string("config_calibre_dir") + + # Google drive setup + if not os.path.isfile(gdriveutils.SETTINGS_YAML): + config.config_use_google_drive = False + + gdrive_secrets = {} + gdriveError = gdriveutils.get_error_text(gdrive_secrets) + if "config_use_google_drive" in to_save and not config.config_use_google_drive and not gdriveError: + if not gdrive_secrets: + return _configuration_result('client_secrets.json is not configured for web application') + gdriveutils.update_settings( + gdrive_secrets['client_id'], + gdrive_secrets['client_secret'], + gdrive_secrets['redirect_uris'][0] + ) + + # always show google drive settings, but in case of error deny support + config.config_use_google_drive = (not gdriveError) and ("config_use_google_drive" in to_save) + if _config_string("config_google_drive_folder"): + gdriveutils.deleteDatabaseOnChange() + + reboot_required |= _config_int("config_port") + + reboot_required |= _config_string("config_keyfile") + if config.config_keyfile and not os.path.isfile(config.config_keyfile): + return _configuration_result('Keyfile location is not valid, please enter correct path', gdriveError) + + reboot_required |= _config_string("config_certfile") + if config.config_certfile and not os.path.isfile(config.config_certfile): + return _configuration_result('Certfile location is not valid, please enter correct path', gdriveError) + + _config_checkbox_int("config_uploading") + _config_checkbox_int("config_anonbrowse") + _config_checkbox_int("config_public_reg") + + _config_int("config_ebookconverter") + _config_string("config_calibre") + _config_string("config_converterpath") + + if _config_int("config_login_type"): + reboot_required |= config.config_login_type != constants.LOGIN_STANDARD + + #LDAP configurator, + if config.config_login_type == constants.LOGIN_LDAP: + _config_string("config_ldap_provider_url") + _config_int("config_ldap_port") + _config_string("config_ldap_schema") + _config_string("config_ldap_dn") + _config_string("config_ldap_user_object") + if not config.config_ldap_provider_url or not config.config_ldap_port or not config.config_ldap_dn or not config.config_ldap_user_object: + return _configuration_result('Please enter a LDAP provider, port, DN and user object identifier', gdriveError) + + _config_string("config_ldap_serv_username") + if not config.config_ldap_serv_username or "config_ldap_serv_password" not in to_save: + return _configuration_result('Please enter a LDAP service account and password', gdriveError) + config.set_from_dictionary(to_save, "config_ldap_serv_password", base64.b64encode) + + _config_checkbox("config_ldap_use_ssl") + _config_checkbox("config_ldap_use_tls") + _config_checkbox("config_ldap_openldap") + _config_checkbox("config_ldap_require_cert") + _config_string("config_ldap_cert_path") + if config.config_ldap_cert_path and not os.path.isfile(config.config_ldap_cert_path): + return _configuration_result('LDAP Certfile location is not valid, please enter correct path', gdriveError) + + # Remote login configuration + _config_checkbox("config_remote_login") + if not config.config_remote_login: + ub.session.query(ub.RemoteAuthToken).delete() + + # Goodreads configuration + _config_checkbox("config_use_goodreads") + _config_string("config_goodreads_api_key") + _config_string("config_goodreads_api_secret") + if services.goodreads: + services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads) + + _config_int("config_updatechannel") + + # GitHub OAuth configuration + if config.config_login_type == constants.LOGIN_OAUTH_GITHUB: + _config_string("config_github_oauth_client_id") + _config_string("config_github_oauth_client_secret") + if not config.config_github_oauth_client_id or not config.config_github_oauth_client_secret: + return _configuration_result('Please enter Github oauth credentials', gdriveError) + + # Google OAuth configuration + if config.config_login_type == constants.LOGIN_OAUTH_GOOGLE: + _config_string("config_google_oauth_client_id") + _config_string("config_google_oauth_client_secret") + if not config.config_google_oauth_client_id or not config.config_google_oauth_client_secret: + return _configuration_result('Please enter Google oauth credentials', gdriveError) + + _config_int("config_log_level") + _config_string("config_logfile") + if not logger.is_valid_logfile(config.config_logfile): + return _configuration_result('Logfile location is not valid, please enter correct path', gdriveError) + + reboot_required |= _config_checkbox_int("config_access_log") + reboot_required |= _config_string("config_access_logfile") + if not logger.is_valid_logfile(config.config_access_logfile): + return _configuration_result('Access Logfile location is not valid, please enter correct path', gdriveError) + + # Rarfile Content configuration + _config_string("config_rarfile_location") + unrar_status = helper.check_unrar(config.config_rarfile_location) + if unrar_status: + return _configuration_result(unrar_status, gdriveError) + + try: + metadata_db = os.path.join(config.config_calibre_dir, "metadata.db") + if config.config_use_google_drive and is_gdrive_ready() and not os.path.exists(metadata_db): + gdriveutils.downloadFile(None, "metadata.db", metadata_db) + db_change = True + except Exception as e: + return _configuration_result('%s' % e, gdriveError) + + if db_change: + # reload(db) + if not db.setup_db(config): + return _configuration_result('DB location is not valid, please enter correct path', gdriveError) + + config.save() + flash(_(u"Calibre-Web configuration updated"), category="success") + if reboot_required: + web_server.stop(True) + + return _configuration_result(None, gdriveError) + + +def _configuration_result(error_flash=None, gdriveError=None): + gdrive_authenticate = not is_gdrive_ready() + gdrivefolders = [] + if gdriveError is None: + gdriveError = gdriveutils.get_error_text() + if gdriveError: + gdriveError = _(gdriveError) else: - if not os.path.isfile(gdriveutils.CLIENT_SECRETS): - gdriveError = _('client_secrets.json is missing or not readable') - else: - with open(gdriveutils.CLIENT_SECRETS, 'r') as settings: - filedata = json.load(settings) - if 'web' not in filedata: - gdriveError = _('client_secrets.json is not configured for web application') - if request.method == "POST": - to_save = request.form.to_dict() - content = ub.session.query(ub.Settings).first() # type: ub.Settings - if "config_calibre_dir" in to_save: - if content.config_calibre_dir != to_save["config_calibre_dir"]: - content.config_calibre_dir = to_save["config_calibre_dir"] - db_change = True - # Google drive setup - if not os.path.isfile(gdriveutils.SETTINGS_YAML): - content.config_use_google_drive = False - if "config_use_google_drive" in to_save and not content.config_use_google_drive and not gdriveError: - if filedata: - if filedata['web']['redirect_uris'][0].endswith('/'): - filedata['web']['redirect_uris'][0] = filedata['web']['redirect_uris'][0][:-1] - with open(gdriveutils.SETTINGS_YAML, 'w') as f: - yaml = "client_config_backend: settings\nclient_config_file: %(client_file)s\n" \ - "client_config:\n" \ - " client_id: %(client_id)s\n client_secret: %(client_secret)s\n" \ - " redirect_uri: %(redirect_uri)s\n\nsave_credentials: True\n" \ - "save_credentials_backend: file\nsave_credentials_file: %(credential)s\n\n" \ - "get_refresh_token: True\n\noauth_scope:\n" \ - " - https://www.googleapis.com/auth/drive\n" - f.write(yaml % {'client_file': gdriveutils.CLIENT_SECRETS, - 'client_id': filedata['web']['client_id'], - 'client_secret': filedata['web']['client_secret'], - 'redirect_uri': filedata['web']['redirect_uris'][0], - 'credential': gdriveutils.CREDENTIALS}) - else: - flash(_(u'client_secrets.json is not configured for web application'), category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - gdriveError=gdriveError, - gfeature_support=feature_support, title=_(u"Basic Configuration"), - page="config") - # always show google drive settings, but in case of error deny support - if "config_use_google_drive" in to_save and not gdriveError: - content.config_use_google_drive = "config_use_google_drive" in to_save - else: - content.config_use_google_drive = 0 - if "config_google_drive_folder" in to_save: - if content.config_google_drive_folder != to_save["config_google_drive_folder"]: - content.config_google_drive_folder = to_save["config_google_drive_folder"] - deleteDatabaseOnChange() - - if "config_port" in to_save: - if content.config_port != int(to_save["config_port"]): - content.config_port = int(to_save["config_port"]) - reboot_required = True - if "config_keyfile" in to_save: - if content.config_keyfile != to_save["config_keyfile"]: - if os.path.isfile(to_save["config_keyfile"]) or to_save["config_keyfile"] is u"": - content.config_keyfile = to_save["config_keyfile"] - reboot_required = True - else: - ub.session.commit() - flash(_(u'Keyfile location is not valid, please enter correct path'), category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - gdriveError=gdriveError, - feature_support=feature_support, title=_(u"Basic Configuration"), - page="config") - if "config_certfile" in to_save: - if content.config_certfile != to_save["config_certfile"]: - if os.path.isfile(to_save["config_certfile"]) or to_save["config_certfile"] is u"": - content.config_certfile = to_save["config_certfile"] - reboot_required = True - else: - ub.session.commit() - flash(_(u'Certfile location is not valid, please enter correct path'), category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - gdriveError=gdriveError, feature_support=feature_support, - title=_(u"Basic Configuration"), page="config") - content.config_uploading = 0 - content.config_anonbrowse = 0 - content.config_public_reg = 0 - if "config_uploading" in to_save and to_save["config_uploading"] == "on": - content.config_uploading = 1 - if "config_anonbrowse" in to_save and to_save["config_anonbrowse"] == "on": - content.config_anonbrowse = 1 - if "config_public_reg" in to_save and to_save["config_public_reg"] == "on": - content.config_public_reg = 1 - - if "config_converterpath" in to_save: - content.config_converterpath = to_save["config_converterpath"].strip() - if "config_calibre" in to_save: - content.config_calibre = to_save["config_calibre"].strip() - if "config_ebookconverter" in to_save: - content.config_ebookconverter = int(to_save["config_ebookconverter"]) - - #LDAP configurator, - if "config_login_type" in to_save and to_save["config_login_type"] == "1": - if not to_save["config_ldap_provider_url"] or not to_save["config_ldap_port"] or not to_save["config_ldap_dn"] or not to_save["config_ldap_user_object"]: - ub.session.commit() - flash(_(u'Please enter a LDAP provider, port, DN and user object identifier'), category="error") - return render_title_template("config_edit.html", content=config, origin=origin, - gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError, - feature_support=feature_support, title=_(u"Basic Configuration"), - page="config") - elif not to_save["config_ldap_serv_username"] or not to_save["config_ldap_serv_password"]: - ub.session.commit() - flash(_(u'Please enter a LDAP service account and password'), category="error") - return render_title_template("config_edit.html", content=config, origin=origin, - gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError, - feature_support=feature_support, title=_(u"Basic Configuration"), - page="config") - else: - content.config_use_ldap = 1 - content.config_ldap_provider_url = to_save["config_ldap_provider_url"] - content.config_ldap_port = to_save["config_ldap_port"] - content.config_ldap_schema = to_save["config_ldap_schema"] - content.config_ldap_serv_username = to_save["config_ldap_serv_username"] - content.config_ldap_serv_password = base64.b64encode(to_save["config_ldap_serv_password"]) - content.config_ldap_dn = to_save["config_ldap_dn"] - content.config_ldap_user_object = to_save["config_ldap_user_object"] - reboot_required = True - content.config_ldap_use_ssl = 0 - content.config_ldap_use_tls = 0 - content.config_ldap_require_cert = 0 - content.config_ldap_openldap = 0 - if "config_ldap_use_ssl" in to_save and to_save["config_ldap_use_ssl"] == "on": - content.config_ldap_use_ssl = 1 - if "config_ldap_use_tls" in to_save and to_save["config_ldap_use_tls"] == "on": - content.config_ldap_use_tls = 1 - if "config_ldap_require_cert" in to_save and to_save["config_ldap_require_cert"] == "on": - content.config_ldap_require_cert = 1 - if "config_ldap_openldap" in to_save and to_save["config_ldap_openldap"] == "on": - content.config_ldap_openldap = 1 - if "config_ldap_cert_path " in to_save: - if content.config_ldap_cert_path != to_save["config_ldap_cert_path "]: - if os.path.isfile(to_save["config_ldap_cert_path "]) or to_save["config_ldap_cert_path "] is u"": - content.config_certfile = to_save["config_ldap_cert_path "] - else: - ub.session.commit() - flash(_(u'Certfile location is not valid, please enter correct path'), category="error") - return render_title_template("config_edit.html", content=config, origin=origin, - gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError, - feature_support=feature_support, title=_(u"Basic Configuration"), - page="config") - - # Remote login configuration - content.config_remote_login = ("config_remote_login" in to_save and to_save["config_remote_login"] == "on") - if not content.config_remote_login: - ub.session.query(ub.RemoteAuthToken).delete() - - # Goodreads configuration - content.config_use_goodreads = ("config_use_goodreads" in to_save and to_save["config_use_goodreads"] == "on") - if "config_goodreads_api_key" in to_save: - content.config_goodreads_api_key = to_save["config_goodreads_api_key"] - if "config_goodreads_api_secret" in to_save: - content.config_goodreads_api_secret = to_save["config_goodreads_api_secret"] - if "config_updater" in to_save: - content.config_updatechannel = int(to_save["config_updater"]) - - # GitHub OAuth configuration - if "config_login_type" in to_save and to_save["config_login_type"] == "2": - if to_save["config_github_oauth_client_id"] == u'' or to_save["config_github_oauth_client_secret"] == u'': - ub.session.commit() - flash(_(u'Please enter Github oauth credentials'), category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - gdriveError=gdriveError, feature_support=feature_support, - title=_(u"Basic Configuration"), page="config") - else: - content.config_login_type = constants.LOGIN_OAUTH_GITHUB - content.config_github_oauth_client_id = to_save["config_github_oauth_client_id"] - content.config_github_oauth_client_secret = to_save["config_github_oauth_client_secret"] - reboot_required = True - - # Google OAuth configuration - if "config_login_type" in to_save and to_save["config_login_type"] == "3": - if to_save["config_google_oauth_client_id"] == u'' or to_save["config_google_oauth_client_secret"] == u'': - ub.session.commit() - flash(_(u'Please enter Google oauth credentials'), category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - gdriveError=gdriveError, feature_support=feature_support, - title=_(u"Basic Configuration"), page="config") - else: - content.config_login_type = constants.LOGIN_OAUTH_GOOGLE - content.config_google_oauth_client_id = to_save["config_google_oauth_client_id"] - content.config_google_oauth_client_secret = to_save["config_google_oauth_client_secret"] - reboot_required = True - - if "config_login_type" in to_save and to_save["config_login_type"] == "0": - content.config_login_type = constants.LOGIN_STANDARD - - if "config_log_level" in to_save: - content.config_log_level = int(to_save["config_log_level"]) - if content.config_logfile != to_save["config_logfile"]: - # check valid path, only path or file - if not logger.is_valid_logfile(to_save["config_logfile"]): - ub.session.commit() - flash(_(u'Logfile location is not valid, please enter correct path'), category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - gdriveError=gdriveError, feature_support=feature_support, - title=_(u"Basic Configuration"), page="config") - content.config_logfile = to_save["config_logfile"] - - content.config_access_log = 0 - if "config_access_log" in to_save and to_save["config_access_log"] == "on": - content.config_access_log = 1 - reboot_required = True - if "config_access_log" not in to_save and config.config_access_log: - reboot_required = True - - if content.config_access_logfile != to_save["config_access_logfile"]: - # check valid path, only path or file - if not logger.is_valid_logfile(to_save["config_access_logfile"]): - ub.session.commit() - flash(_(u'Access Logfile location is not valid, please enter correct path'), category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - gdriveError=gdriveError, feature_support=feature_support, - title=_(u"Basic Configuration"), page="config") - content.config_access_logfile = to_save["config_access_logfile"] - reboot_required = True - - # Rarfile Content configuration - if "config_rarfile_location" in to_save and to_save['config_rarfile_location'] is not u"": - check = check_unrar(to_save["config_rarfile_location"].strip()) - if not check[0] : - content.config_rarfile_location = to_save["config_rarfile_location"].strip() - else: - flash(check[1], category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - feature_support=feature_support, title=_(u"Basic Configuration")) - try: - if content.config_use_google_drive and is_gdrive_ready() and not \ - os.path.exists(os.path.join(content.config_calibre_dir, "metadata.db")): - downloadFile(None, "metadata.db", config.config_calibre_dir + "/metadata.db") - if db_change: - if config.db_configured: - db.session.close() - db.engine.dispose() - ub.session.commit() - flash(_(u"Calibre-Web configuration updated"), category="success") - config.loadSettings() - except Exception as e: - flash(e, category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - gdriveError=gdriveError, feature_support=feature_support, - title=_(u"Basic Configuration"), page="config") - if db_change: - reload(db) - if not db.setup_db(): - flash(_(u'DB location is not valid, please enter correct path'), category="error") - return render_title_template("config_edit.html", config=config, origin=origin, - gdriveError=gdriveError, feature_support=feature_support, - title=_(u"Basic Configuration"), page="config") - if reboot_required: - # stop Server - web_server.stop(True) - log.info('Reboot required, restarting') - if origin: - success = True - if is_gdrive_ready() and feature_support['gdrive'] is True and config.config_use_google_drive == True: - gdrivefolders = listRootFolders() - else: - gdrivefolders = list() - return render_title_template("config_edit.html", origin=origin, success=success, config=config, - show_authenticate_google_drive=not is_gdrive_ready(), + gdrivefolders = gdriveutils.listRootFolders() + + show_back_button = current_user.is_authenticated + show_login_button = config.db_configured and not current_user.is_authenticated + if error_flash: + config.load() + flash(_(error_flash), category="error") + show_login_button = False + + return render_title_template("config_edit.html", config=config, + show_back_button=show_back_button, show_login_button=show_login_button, + show_authenticate_google_drive=gdrive_authenticate, gdriveError=gdriveError, gdrivefolders=gdrivefolders, feature_support=feature_support, title=_(u"Basic Configuration"), page="config") @@ -570,34 +431,14 @@ def new_user(): to_save = request.form.to_dict() content.default_language = to_save["default_language"] content.mature_content = "Show_mature_content" in to_save - if "locale" in to_save: - content.locale = to_save["locale"] - - val = 0 - for key, __ in to_save.items(): - if key.startswith('show'): - val += int(key[5:]) - content.sidebar_view = val - + content.locale = to_save.get("locale", content.locale) + content.sidebar_view = sum(int(key[5:]) for key in to_save if key.startswith('show_')) if "show_detail_random" in to_save: content.sidebar_view |= constants.DETAIL_RANDOM - content.role = 0 - if "admin_role" in to_save: - content.role |= constants.ROLE_ADMIN - if "download_role" in to_save: - content.role |= constants.ROLE_DOWNLOAD - if "upload_role" in to_save: - content.role |= constants.ROLE_UPLOAD - if "edit_role" in to_save: - content.role |= constants.ROLE_EDIT - if "delete_role" in to_save: - content.role |= constants.ROLE_DELETE_BOOKS - if "passwd_role" in to_save: - content.role |= constants.ROLE_PASSWD - if "edit_shelf_role" in to_save: - content.role |= constants.ROLE_EDIT_SHELFS + content.role = constants.selected_roles(to_save) + if not to_save["nickname"] or not to_save["email"] or not to_save["password"]: flash(_(u"Please fill out all fields!"), category="error") return render_title_template("user_edit.html", new_user=1, content=content, translations=translations, @@ -637,39 +478,50 @@ def new_user(): registered_oauth=oauth_check) -@admi.route("/admin/mailsettings", methods=["GET", "POST"]) +@admi.route("/admin/mailsettings") @login_required @admin_required def edit_mailsettings(): - content = ub.session.query(ub.Settings).first() - if request.method == "POST": - to_save = request.form.to_dict() - content.mail_server = to_save["mail_server"] - content.mail_port = int(to_save["mail_port"]) - content.mail_login = to_save["mail_login"] - content.mail_password = to_save["mail_password"] - content.mail_from = to_save["mail_from"] - content.mail_use_ssl = int(to_save["mail_use_ssl"]) - try: - ub.session.commit() - except Exception as e: - flash(e, category="error") - if "test" in to_save and to_save["test"]: - if current_user.kindle_mail: - result = send_test_mail(current_user.kindle_mail, current_user.nickname) - if result is None: - flash(_(u"Test e-mail successfully send to %(kindlemail)s", kindlemail=current_user.kindle_mail), - category="success") - else: - flash(_(u"There was an error sending the Test e-mail: %(res)s", res=result), category="error") - else: - flash(_(u"Please configure your kindle e-mail address first..."), category="error") - else: - flash(_(u"E-mail server settings updated"), category="success") + content = config.get_mail_settings() + # log.debug("edit_mailsettings %r", content) return render_title_template("email_edit.html", content=content, title=_(u"Edit e-mail server settings"), page="mailset") +@admi.route("/admin/mailsettings", methods=["POST"]) +@login_required +@admin_required +def update_mailsettings(): + to_save = request.form.to_dict() + log.debug("update_mailsettings %r", to_save) + + _config_string = lambda x: config.set_from_dictionary(to_save, x, lambda y: y.strip() if y else y) + _config_int = lambda x: config.set_from_dictionary(to_save, x, int) + + _config_string("mail_server") + _config_int("mail_port") + _config_int("mail_use_ssl") + _config_string("mail_login") + _config_string("mail_password") + _config_string("mail_from") + config.save() + + if to_save.get("test"): + if current_user.kindle_mail: + result = send_test_mail(current_user.kindle_mail, current_user.nickname) + if result is None: + flash(_(u"Test e-mail successfully send to %(kindlemail)s", kindlemail=current_user.kindle_mail), + category="success") + else: + flash(_(u"There was an error sending the Test e-mail: %(res)s", res=result), category="error") + else: + flash(_(u"Please configure your kindle e-mail address first..."), category="error") + else: + flash(_(u"E-mail server settings updated"), category="success") + + return edit_mailsettings() + + @admi.route("/admin/user/", methods=["GET", "POST"]) @login_required @admin_required @@ -703,53 +555,21 @@ def edit_user(user_id): if "password" in to_save and to_save["password"]: content.password = generate_password_hash(to_save["password"]) - if "admin_role" in to_save: - content.role |= constants.ROLE_ADMIN - else: - content.role &= ~constants.ROLE_ADMIN - - if "download_role" in to_save: - content.role |= constants.ROLE_DOWNLOAD - else: - content.role &= ~constants.ROLE_DOWNLOAD - - if "viewer_role" in to_save: - content.role |= constants.ROLE_VIEWER - else: - content.role &= ~constants.ROLE_VIEWER - - if "upload_role" in to_save: - content.role |= constants.ROLE_UPLOAD - else: - content.role &= ~constants.ROLE_UPLOAD - - if "edit_role" in to_save: - content.role |= constants.ROLE_EDIT - else: - content.role &= ~constants.ROLE_EDIT - - if "delete_role" in to_save: - content.role |= constants.ROLE_DELETE_BOOKS - else: - content.role &= ~constants.ROLE_DELETE_BOOKS - - if "passwd_role" in to_save: - content.role |= constants.ROLE_PASSWD - else: - content.role &= ~constants.ROLE_PASSWD - - if "edit_shelf_role" in to_save: - content.role |= constants.ROLE_EDIT_SHELFS + anonymous = content.is_anonymous + content.role = constants.selected_roles(to_save) + if anonymous: + content.role |= constants.ROLE_ANONYMOUS else: - content.role &= ~constants.ROLE_EDIT_SHELFS + content.role &= ~constants.ROLE_ANONYMOUS - val = [int(k[5:]) for k, __ in to_save.items() if k.startswith('show_')] + val = [int(k[5:]) for k in to_save if k.startswith('show_')] sidebar = ub.get_sidebar_config() for element in sidebar: - if element['visibility'] in val and not content.check_visibility(element['visibility']): - content.sidebar_view |= element['visibility'] - elif not element['visibility'] in val and content.check_visibility(element['visibility']): - content.sidebar_view &= ~element['visibility'] + value = element['visibility'] + if value in val and not content.check_visibility(value): + content.sidebar_view |= value + elif not value in val and content.check_visibility(value): + content.sidebar_view &= ~value if "Show_detail_random" in to_save: content.sidebar_view |= constants.DETAIL_RANDOM diff --git a/cps/config_sql.py b/cps/config_sql.py new file mode 100644 index 00000000..37ea77e5 --- /dev/null +++ b/cps/config_sql.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2019 OzzieIsaacs, pwr +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from __future__ import division, print_function, unicode_literals +import os +import json + +from sqlalchemy import exc, Column, String, Integer, SmallInteger, Boolean +from sqlalchemy.ext.declarative import declarative_base + +from . import constants, cli, logger + + +log = logger.create() +_Base = declarative_base() + + +# Baseclass for representing settings in app.db with email server settings and Calibre database settings +# (application settings) +class _Settings(_Base): + __tablename__ = 'settings' + + id = Column(Integer, primary_key=True) + mail_server = Column(String, default='mail.example.org') + mail_port = Column(Integer, default=25) + mail_use_ssl = Column(SmallInteger, default=0) + mail_login = Column(String, default='mail@example.com') + mail_password = Column(String, default='mypassword') + mail_from = Column(String, default='automailer ') + config_calibre_dir = Column(String) + config_port = Column(Integer, default=constants.DEFAULT_PORT) + config_certfile = Column(String) + config_keyfile = Column(String) + config_calibre_web_title = Column(String, default=u'Calibre-Web') + config_books_per_page = Column(Integer, default=60) + config_random_books = Column(Integer, default=4) + config_authors_max = Column(Integer, default=0) + config_read_column = Column(Integer, default=0) + config_title_regex = Column(String, default=u'^(A|The|An|Der|Die|Das|Den|Ein|Eine|Einen|Dem|Des|Einem|Eines)\s+') + config_log_level = Column(SmallInteger, default=logger.DEFAULT_LOG_LEVEL) + config_access_log = Column(SmallInteger, default=0) + config_uploading = Column(SmallInteger, default=0) + config_anonbrowse = Column(SmallInteger, default=0) + config_public_reg = Column(SmallInteger, default=0) + config_default_role = Column(SmallInteger, default=0) + config_default_show = Column(SmallInteger, default=6143) + config_columns_to_ignore = Column(String) + config_use_google_drive = Column(Boolean, default=False) + config_google_drive_folder = Column(String) + config_google_drive_watch_changes_response = Column(String) + config_remote_login = Column(Boolean, default=False) + config_use_goodreads = Column(Boolean, default=False) + config_goodreads_api_key = Column(String) + config_goodreads_api_secret = Column(String) + config_login_type = Column(Integer, default=0) + # config_use_ldap = Column(Boolean) + config_ldap_provider_url = Column(String) + config_ldap_dn = Column(String) + # config_use_github_oauth = Column(Boolean) + config_github_oauth_client_id = Column(String) + config_github_oauth_client_secret = Column(String) + # config_use_google_oauth = Column(Boolean) + config_google_oauth_client_id = Column(String) + config_google_oauth_client_secret = Column(String) + config_ldap_provider_url = Column(String, default='localhost') + config_ldap_port = Column(SmallInteger, default=389) + config_ldap_schema = Column(String, default='ldap') + config_ldap_serv_username = Column(String) + config_ldap_serv_password = Column(String) + config_ldap_use_ssl = Column(Boolean, default=False) + config_ldap_use_tls = Column(Boolean, default=False) + config_ldap_require_cert = Column(Boolean, default=False) + config_ldap_cert_path = Column(String) + config_ldap_dn = Column(String) + config_ldap_user_object = Column(String) + config_ldap_openldap = Column(Boolean, default=False) + config_mature_content_tags = Column(String, default='') + config_logfile = Column(String) + config_access_logfile = Column(String) + config_ebookconverter = Column(Integer, default=0) + config_converterpath = Column(String) + config_calibre = Column(String) + config_rarfile_location = Column(String) + config_theme = Column(Integer, default=0) + config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) + + def __repr__(self): + return self.__class__.__name__ + + +# Class holds all application specific settings in calibre-web +class _ConfigSQL(object): + # pylint: disable=no-member + def __init__(self, session): + self._session = session + self._settings = None + self.db_configured = None + self.config_calibre_dir = None + self.load() + + def _read_from_storage(self): + if self._settings is None: + log.debug("_ConfigSQL._read_from_storage") + self._settings = self._session.query(_Settings).first() + return self._settings + + def get_config_certfile(self): + if cli.certfilepath: + return cli.certfilepath + if cli.certfilepath == "": + return None + return self.config_certfile + + def get_config_keyfile(self): + if cli.keyfilepath: + return cli.keyfilepath + if cli.certfilepath == "": + return None + return self.config_keyfile + + def get_config_ipaddress(self): + return cli.ipadress or "" + + def get_ipaddress_type(self): + return cli.ipv6 + + def _has_role(self, role_flag): + return constants.has_flag(self.config_default_role, role_flag) + + def role_admin(self): + return self._has_role(constants.ROLE_ADMIN) + + def role_download(self): + return self._has_role(constants.ROLE_DOWNLOAD) + + def role_viewer(self): + return self._has_role(constants.ROLE_VIEWER) + + def role_upload(self): + return self._has_role(constants.ROLE_UPLOAD) + + def role_edit(self): + return self._has_role(constants.ROLE_EDIT) + + def role_passwd(self): + return self._has_role(constants.ROLE_PASSWD) + + def role_edit_shelfs(self): + return self._has_role(constants.ROLE_EDIT_SHELFS) + + def role_delete_books(self): + return self._has_role(constants.ROLE_DELETE_BOOKS) + + def show_element_new_user(self, value): + return constants.has_flag(self.config_default_show, value) + + def show_detail_random(self): + return self.show_element_new_user(constants.DETAIL_RANDOM) + + def show_mature_content(self): + return self.show_element_new_user(constants.MATURE_CONTENT) + + def mature_content_tags(self): + mct = self.config_mature_content_tags.split(",") + return [t.strip() for t in mct] + + def get_log_level(self): + return logger.get_level_name(self.config_log_level) + + def get_mail_settings(self): + return {k:v for k, v in self.__dict__.items() if k.startswith('mail_')} + + def set_from_dictionary(self, dictionary, field, convertor=None, default=None): + '''Possibly updates a field of this object. + The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor. + + :returns: `True` if the field has changed value + ''' + new_value = dictionary.get(field, default) + if new_value is None: + # log.debug("_ConfigSQL set_from_dictionary field '%s' not found", field) + return False + + if field not in self.__dict__: + log.warning("_ConfigSQL trying to set unknown field '%s' = %r", field, new_value) + return False + + if convertor is not None: + new_value = convertor(new_value) + + current_value = self.__dict__.get(field) + if current_value == new_value: + return False + + # log.debug("_ConfigSQL set_from_dictionary '%s' = %r (was %r)", field, new_value, current_value) + setattr(self, field, new_value) + return True + + def load(self): + '''Load all configuration values from the underlying storage.''' + s = self._read_from_storage() # type: _Settings + for k, v in s.__dict__.items(): + if k[0] != '_': + if v is None: + # if the storage column has no value, apply the (possible) default + column = s.__class__.__dict__.get(k) + if column.default is not None: + v = column.default.arg + setattr(self, k, v) + + if self.config_google_drive_watch_changes_response: + self.config_google_drive_watch_changes_response = json.loads(self.config_google_drive_watch_changes_response) + self.db_configured = (self.config_calibre_dir and + (not self.config_use_google_drive or os.path.exists(self.config_calibre_dir + '/metadata.db'))) + logger.setup(self.config_logfile, self.config_log_level) + + def save(self): + '''Apply all configuration values to the underlying storage.''' + s = self._read_from_storage() # type: _Settings + + for k, v in self.__dict__.items(): + if k[0] == '_': + continue + if hasattr(s, k): # and getattr(s, k, None) != v: + # log.debug("_Settings save '%s' = %r", k, v) + setattr(s, k, v) + + log.debug("_ConfigSQL updating storage") + self._session.merge(s) + self._session.commit() + self.load() + + def invalidate(self): + log.warning("invalidating configuration") + self.db_configured = False + self.config_calibre_dir = None + self.save() + + +def _migrate_table(session, orm_class): + changed = False + + for column_name, column in orm_class.__dict__.items(): + if column_name[0] != '_': + try: + session.query(column).first() + except exc.OperationalError as err: + log.debug("%s: %s", column_name, err) + column_default = "" if column.default is None else ("DEFAULT %r" % column.default.arg) + alter_table = "ALTER TABLE %s ADD COLUMN `%s` %s %s" % (orm_class.__tablename__, column_name, column.type, column_default) + session.execute(alter_table) + changed = True + + if changed: + session.commit() + + +def _migrate_database(session): + # make sure the table is created, if it does not exist + _Base.metadata.create_all(session.bind) + _migrate_table(session, _Settings) + + +def load_configuration(session): + _migrate_database(session) + + if not session.query(_Settings).count(): + session.add(_Settings()) + session.commit() + + return _ConfigSQL(session) diff --git a/cps/constants.py b/cps/constants.py index fce91312..8d0002f1 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -74,7 +74,7 @@ SIDEBAR_PUBLISHER = 1 << 12 SIDEBAR_RATING = 1 << 13 SIDEBAR_FORMAT = 1 << 14 -ADMIN_USER_ROLES = (ROLE_VIEWER << 1) - 1 - (ROLE_ANONYMOUS | ROLE_EDIT_SHELFS) +ADMIN_USER_ROLES = sum(r for r in ALL_ROLES.values()) & ~ROLE_EDIT_SHELFS & ~ROLE_ANONYMOUS ADMIN_USER_SIDEBAR = (SIDEBAR_FORMAT << 1) - 1 UPDATE_STABLE = 0 << 0 @@ -109,6 +109,9 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'mobi', 'azw', 'azw3', 'cbr', 'cbz' def has_flag(value, bit_flag): return bit_flag == (bit_flag & (value or 0)) +def selected_roles(dictionary): + return sum(v for k, v in ALL_ROLES.items() if k in dictionary) + # :rtype: BookMeta BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, ' diff --git a/cps/converter.py b/cps/converter.py index a2eb572d..6dc44383 100644 --- a/cps/converter.py +++ b/cps/converter.py @@ -24,19 +24,14 @@ import re from flask_babel import gettext as _ from . import config -from .subproc_wrapper import process_open +from .subproc_wrapper import process_wait def versionKindle(): versions = _(u'not installed') if os.path.exists(config.config_converterpath): try: - p = process_open(config.config_converterpath) - # p = subprocess.Popen(ub.config.config_converterpath, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - p.wait() - for lines in p.stdout.readlines(): - if isinstance(lines, bytes): - lines = lines.decode('utf-8') + for lines in process_wait(config.config_converterpath): if re.search('Amazon kindlegen\(', lines): versions = lines except Exception: @@ -48,12 +43,7 @@ def versionCalibre(): versions = _(u'not installed') if os.path.exists(config.config_converterpath): try: - p = process_open([config.config_converterpath, '--version']) - # p = subprocess.Popen([ub.config.config_converterpath, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - p.wait() - for lines in p.stdout.readlines(): - if isinstance(lines, bytes): - lines = lines.decode('utf-8') + for lines in process_wait([config.config_converterpath, '--version']): if re.search('ebook-convert.*\(calibre', lines): versions = lines except Exception: diff --git a/cps/db.py b/cps/db.py index 5429e93c..edcdef63 100755 --- a/cps/db.py +++ b/cps/db.py @@ -30,24 +30,10 @@ from sqlalchemy import String, Integer, Boolean from sqlalchemy.orm import relationship, sessionmaker, scoped_session from sqlalchemy.ext.declarative import declarative_base -from . import config, ub - session = None cc_exceptions = ['datetime', 'comments', 'float', 'composite', 'series'] -cc_classes = None -engine = None - - -# user defined sort function for calibre databases (Series, etc.) -def title_sort(title): - # calibre sort stuff - title_pat = re.compile(config.config_title_regex, re.IGNORECASE) - match = title_pat.search(title) - if match: - prep = match.group(1) - title = title.replace(prep, '') + ', ' + prep - return title.strip() +cc_classes = {} Base = declarative_base() @@ -325,40 +311,45 @@ class Custom_Columns(Base): return display_dict -def setup_db(): - global engine - global session - global cc_classes - - if config.config_calibre_dir is None or config.config_calibre_dir == u'': - content = ub.session.query(ub.Settings).first() - content.config_calibre_dir = None - content.db_configured = False - ub.session.commit() - config.loadSettings() +def update_title_sort(config, conn=None): + # user defined sort function for calibre databases (Series, etc.) + def _title_sort(title): + # calibre sort stuff + title_pat = re.compile(config.config_title_regex, re.IGNORECASE) + match = title_pat.search(title) + if match: + prep = match.group(1) + title = title.replace(prep, '') + ', ' + prep + return title.strip() + + conn = conn or session.connection().connection.connection + conn.create_function("title_sort", 1, _title_sort) + + +def setup_db(config): + dispose() + + if not config.config_calibre_dir: + config.invalidate() return False dbpath = os.path.join(config.config_calibre_dir, "metadata.db") + if not os.path.exists(dbpath): + config.invalidate() + return False + try: - if not os.path.exists(dbpath): - raise engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False, isolation_level="SERIALIZABLE", connect_args={'check_same_thread': False}) conn = engine.connect() - except Exception: - content = ub.session.query(ub.Settings).first() - content.config_calibre_dir = None - content.db_configured = False - ub.session.commit() - config.loadSettings() + except: + config.invalidate() return False - content = ub.session.query(ub.Settings).first() - content.db_configured = True - ub.session.commit() - config.loadSettings() - conn.connection.create_function('title_sort', 1, title_sort) + + config.db_configured = True + update_title_sort(config, conn.connection) # conn.connection.create_function('lower', 1, lcase) # conn.connection.create_function('upper', 1, ucase) @@ -367,7 +358,6 @@ def setup_db(): cc_ids = [] books_custom_column_links = {} - cc_classes = {} for row in cc: if row.datatype not in cc_exceptions: books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata, @@ -406,8 +396,38 @@ def setup_db(): backref='books')) + global session Session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine)) session = Session() return True + + +def dispose(): + global session + + engine = None + if session: + engine = session.bind + try: session.close() + except: pass + session = None + + if engine: + try: engine.dispose() + except: pass + + for attr in list(Books.__dict__.keys()): + if attr.startswith("custom_column_"): + delattr(Books, attr) + + for db_class in cc_classes.values(): + Base.metadata.remove(db_class.__table__) + cc_classes.clear() + + for table in reversed(Base.metadata.sorted_tables): + name = table.key + if name.startswith("custom_column_") or name.startswith("books_custom_column_"): + if table is not None: + Base.metadata.remove(table) diff --git a/cps/editbooks.py b/cps/editbooks.py index e15011fb..7f850254 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -33,7 +33,7 @@ from flask_babel import gettext as _ from flask_login import current_user from . import constants, logger, isoLanguages, gdriveutils, uploader, helper -from . import config, get_locale, db, ub, global_WorkerThread, language_table +from . import config, get_locale, db, ub, global_WorkerThread from .helper import order_authors, common_filters from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required, login_required @@ -206,7 +206,7 @@ def delete_book(book_id, book_format): def render_edit_book(book_id): - db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort) + db.update_title_sort(config) cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() book = db.session.query(db.Books)\ .filter(db.Books.id == book_id).filter(common_filters()).first() @@ -215,8 +215,8 @@ def render_edit_book(book_id): flash(_(u"Error opening eBook. File does not exist or file is not accessible"), category="error") return redirect(url_for("web.index")) - for indx in range(0, len(book.languages)): - book.languages[indx].language_name = language_table[get_locale()][book.languages[indx].lang_code] + for lang in book.languages: + lang.language_name = isoLanguages.get_language_name(get_locale(), lang.lang_code) book = order_authors(book) @@ -354,7 +354,7 @@ def upload_single_file(request, book, book_id): db_format = db.Data(book_id, file_ext.upper(), file_size, file_name) db.session.add(db_format) db.session.commit() - db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort) + db.update_title_sort(config) # Queue uploader info uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title) @@ -385,7 +385,7 @@ def edit_book(book_id): return render_edit_book(book_id) # create the function for sorting... - db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort) + db.update_title_sort(config) book = db.session.query(db.Books)\ .filter(db.Books.id == book_id).filter(common_filters()).first() @@ -484,17 +484,12 @@ def edit_book(book_id): # handle book languages input_languages = to_save["languages"].split(',') - input_languages = [x.strip().lower() for x in input_languages if x != ''] - input_l = [] - invers_lang_table = [x.lower() for x in language_table[get_locale()].values()] - for lang in input_languages: - try: - res = list(language_table[get_locale()].keys())[invers_lang_table.index(lang)] - input_l.append(res) - except ValueError: - log.error('%s is not a valid language', lang) - flash(_(u"%(langname)s is not a valid language", langname=lang), category="error") - modify_database_object(input_l, book.languages, db.Languages, db.session, 'languages') + unknown_languages = [] + input_l = isoLanguages.get_language_codes(get_locale(), input_languages, unknown_languages) + for l in unknown_languages: + log.error('%s is not a valid language', l) + flash(_(u"%(langname)s is not a valid language", langname=l), category="error") + modify_database_object(list(input_l), book.languages, db.Languages, db.session, 'languages') # handle book ratings if to_save["rating"].strip(): @@ -546,7 +541,7 @@ def upload(): if request.method == 'POST' and 'btn-upload' in request.files: for requested_file in request.files.getlist("btn-upload"): # create the function for sorting... - db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort) + db.update_title_sort(config) db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4())) # check if file extension is correct @@ -659,7 +654,7 @@ def upload(): # save data to database, reread data db.session.commit() - db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort) + db.update_title_sort(config) book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first() # upload book to gdrive if nesseccary and add "(bookid)" to folder name diff --git a/cps/gdrive.py b/cps/gdrive.py index 196b9dac..263c829b 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -39,7 +39,7 @@ try: except ImportError: pass -from . import logger, gdriveutils, config, ub, db +from . import logger, gdriveutils, config, db from .web import admin_required @@ -94,12 +94,9 @@ def watch_gdrive(): try: result = gdriveutils.watchChange(gdriveutils.Gdrive.Instance().drive, notification_id, 'web_hook', address, gdrive_watch_callback_token, current_milli_time() + 604800*1000) - settings = ub.session.query(ub.Settings).first() - settings.config_google_drive_watch_changes_response = json.dumps(result) - ub.session.merge(settings) - ub.session.commit() - settings = ub.session.query(ub.Settings).first() - config.loadSettings() + config.config_google_drive_watch_changes_response = json.dumps(result) + # after save(), config_google_drive_watch_changes_response will be a json object, not string + config.save() except HttpError as e: reason=json.loads(e.content)['error']['errors'][0] if reason['reason'] == u'push.webhookUrlUnauthorized': @@ -121,11 +118,8 @@ def revoke_watch_gdrive(): last_watch_response['resourceId']) except HttpError: pass - settings = ub.session.query(ub.Settings).first() - settings.config_google_drive_watch_changes_response = None - ub.session.merge(settings) - ub.session.commit() - config.loadSettings() + config.config_google_drive_watch_changes_response = None + config.save() return redirect(url_for('admin.configuration')) @@ -157,7 +151,7 @@ def on_received_watch_confirmation(): log.info('Setting up new DB') # prevent error on windows, as os.rename does on exisiting files move(os.path.join(tmpDir, "tmp_metadata.db"), dbpath) - db.setup_db() + db.setup_db(config) except Exception as e: log.exception(e) updateMetaData() diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 705966b2..2333a626 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -19,6 +19,7 @@ from __future__ import division, print_function, unicode_literals import os +import json import shutil from flask import Response, stream_with_context @@ -534,3 +535,51 @@ def do_gdrive_download(df, headers): log.warning('An error occurred: %s', resp) return return Response(stream_with_context(stream()), headers=headers) + + +_SETTINGS_YAML_TEMPLATE = """ +client_config_backend: settings +client_config_file: %(client_file)s +client_config: + client_id: %(client_id)s + client_secret: %(client_secret)s + redirect_uri: %(redirect_uri)s + +save_credentials: True +save_credentials_backend: file +save_credentials_file: %(credential)s + +get_refresh_token: True + +oauth_scope: + - https://www.googleapis.com/auth/drive +""" + +def update_settings(client_id, client_secret, redirect_uri): + if redirect_uri.endswith('/'): + redirect_uri = redirect_uri[:-1] + config_params = { + 'client_file': CLIENT_SECRETS, + 'client_id': client_id, + 'client_secret': client_secret, + 'redirect_uri': redirect_uri, + 'credential': CREDENTIALS + } + + with open(SETTINGS_YAML, 'w') as f: + f.write(_SETTINGS_YAML_TEMPLATE % config_params) + + +def get_error_text(client_secrets=None): + if not gdrive_support: + return 'Import of optional Google Drive requirements missing' + + if not os.path.isfile(CLIENT_SECRETS): + return 'client_secrets.json is missing or not readable' + + with open(CLIENT_SECRETS, 'r') as settings: + filedata = json.load(settings) + if 'web' not in filedata: + return 'client_secrets.json is not configured for web application' + if client_secrets: + client_secrets.update(filedata['web']) diff --git a/cps/helper.py b/cps/helper.py index 5520b6af..1ea5ef71 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -26,17 +26,16 @@ import json import mimetypes import random import re -import requests import shutil import time import unicodedata from datetime import datetime, timedelta -from functools import reduce from tempfile import gettempdir +import requests from babel import Locale as LC from babel.core import UnknownLocaleError -from babel.dates import format_datetime, format_timedelta +from babel.dates import format_datetime from babel.units import format_unit from flask import send_from_directory, make_response, redirect, abort from flask_babel import gettext as _ @@ -55,12 +54,6 @@ try: except ImportError: use_unidecode = False -try: - import Levenshtein - use_levenshtein = True -except ImportError: - use_levenshtein = False - try: from PIL import Image use_PIL = True @@ -71,7 +64,7 @@ from . import logger, config, global_WorkerThread, get_locale, db, ub, isoLangua from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR from .pagination import Pagination -from .subproc_wrapper import process_open +from .subproc_wrapper import process_wait from .worker import STAT_WAITING, STAT_FAIL, STAT_STARTED, STAT_FINISH_SUCCESS from .worker import TASK_EMAIL, TASK_CONVERT, TASK_UPLOAD, TASK_CONVERT_ANY @@ -110,7 +103,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, if os.path.exists(file_path + "." + old_book_format.lower()): # read settings and append converter task to queue if kindle_mail: - settings = ub.get_mail_settings() + settings = config.get_mail_settings() settings['subject'] = _('Send to Kindle') # pretranslate Subject for e-mail settings['body'] = _(u'This e-mail has been sent via Calibre-Web.') # text = _(u"%(format)s: %(book)s", format=new_book_format, book=book.title) @@ -128,7 +121,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, def send_test_mail(kindle_mail, user_name): - global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, ub.get_mail_settings(), + global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, config.get_mail_settings(), kindle_mail, user_name, _(u"Test e-mail"), _(u'This e-mail has been sent via Calibre-Web.')) return @@ -145,7 +138,7 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False): text += "Don't forget to change your password after first login.\r\n" text += "Sincerely\r\n\r\n" text += "Your Calibre-Web team" - global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, ub.get_mail_settings(), + global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, config.get_mail_settings(), e_mail, None, _(u"Registration e-mail for user: %(name)s", name=user_name), text) return @@ -218,7 +211,7 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id): for entry in iter(book.data): if entry.format.upper() == book_format.upper(): result = entry.name + '.' + book_format.lower() - global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, ub.get_mail_settings(), + global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, config.get_mail_settings(), kindle_mail, user_id, _(u"E-mail: %(book)s", book=book.title), _(u'This e-mail has been sent via Calibre-Web.')) return @@ -561,27 +554,23 @@ def do_download_file(book, book_format, data, headers): def check_unrar(unrarLocation): - error = False - if os.path.exists(unrarLocation): - try: - if sys.version_info < (3, 0): - unrarLocation = unrarLocation.encode(sys.getfilesystemencoding()) - p = process_open(unrarLocation) - p.wait() - for lines in p.stdout.readlines(): - if isinstance(lines, bytes): - lines = lines.decode('utf-8') - value=re.search('UNRAR (.*) freeware', lines) - if value: - version = value.group(1) - except OSError as e: - error = True - log.exception(e) - version =_(u'Error excecuting UnRar') - else: - version = _(u'Unrar binary file not found') - error=True - return (error, version) + if not unrarLocation: + return + + if not os.path.exists(unrarLocation): + return 'Unrar binary file not found' + + try: + if sys.version_info < (3, 0): + unrarLocation = unrarLocation.encode(sys.getfilesystemencoding()) + for lines in process_wait(unrarLocation): + value = re.search('UNRAR (.*) freeware', lines) + if value: + version = value.group(1) + log.debug("unrar version %s", version) + except OSError as err: + log.exception(err) + return 'Error excecuting UnRar' @@ -605,7 +594,7 @@ def json_serial(obj): def format_runtime(runtime): retVal = "" if runtime.days: - retVal = format_unit(runtime.days, 'duration-day', length="long", locale=web.get_locale()) + ', ' + retVal = format_unit(runtime.days, 'duration-day', length="long", locale=get_locale()) + ', ' mins, seconds = divmod(runtime.seconds, 60) hours, minutes = divmod(mins, 60) # ToDo: locale.number_symbols._data['timeSeparator'] -> localize time separator ? @@ -754,28 +743,6 @@ def get_search_results(term): func.lower(db.Books.title).ilike("%" + term + "%") )).all() -def get_unique_other_books(library_books, author_books): - # Get all identifiers (ISBN, Goodreads, etc) and filter author's books by that list so we show fewer duplicates - # Note: Not all images will be shown, even though they're available on Goodreads.com. - # See https://www.goodreads.com/topic/show/18213769-goodreads-book-images - identifiers = reduce(lambda acc, book: acc + map(lambda identifier: identifier.val, book.identifiers), - library_books, []) - other_books = filter(lambda book: book.isbn not in identifiers and book.gid["#text"] not in identifiers, - author_books) - - # Fuzzy match book titles - if use_levenshtein: - library_titles = reduce(lambda acc, book: acc + [book.title], library_books, []) - other_books = filter(lambda author_book: not filter( - lambda library_book: - # Remove items in parentheses before comparing - Levenshtein.ratio(re.sub(r"\(.*\)", "", author_book.title), library_book) > 0.7, - library_titles - ), other_books) - - return other_books - - def get_cc_columns(): tmpcc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() if config.config_columns_to_ignore: @@ -802,10 +769,7 @@ def get_download_link(book_id, book_format): file_name = book.authors[0].name + '_' + file_name file_name = get_valid_filename(file_name) headers = Headers() - try: - headers["Content-Type"] = mimetypes.types_map['.' + book_format] - except KeyError: - headers["Content-Type"] = "application/octet-stream" + headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream") headers["Content-Disposition"] = "attachment; filename*=UTF-8''%s.%s" % (quote(file_name.encode('utf-8')), book_format) return do_download_file(book, book_format, data, headers) diff --git a/cps/isoLanguages.py b/cps/isoLanguages.py index ab112270..808d3761 100644 --- a/cps/isoLanguages.py +++ b/cps/isoLanguages.py @@ -18,6 +18,14 @@ # along with this program. If not, see . from __future__ import division, print_function, unicode_literals +import sys +import os +try: + import cPickle +except ImportError: + import pickle as cPickle + +from .constants import TRANSLATIONS_DIR as _TRANSLATIONS_DIR try: @@ -33,14 +41,43 @@ except ImportError: __version__ = "? (PyCountry)" def _copy_fields(l): - l.part1 = l.alpha_2 - l.part3 = l.alpha_3 + l.part1 = getattr(l, 'alpha_2', None) + l.part3 = getattr(l, 'alpha_3', None) return l def get(name=None, part1=None, part3=None): - if (part3 is not None): + if part3 is not None: return _copy_fields(pyc_languages.get(alpha_3=part3)) - if (part1 is not None): + if part1 is not None: return _copy_fields(pyc_languages.get(alpha_2=part1)) - if (name is not None): + if name is not None: return _copy_fields(pyc_languages.get(name=name)) + + +try: + with open(os.path.join(_TRANSLATIONS_DIR, 'iso639.pickle'), 'rb') as f: + _LANGUAGES = cPickle.load(f) +except cPickle.UnpicklingError as error: + print("Can't read file cps/translations/iso639.pickle: %s" % error) + sys.exit(1) + + +def get_language_names(locale): + return _LANGUAGES.get(locale) + + +def get_language_name(locale, lang_code): + return get_language_names(locale)[lang_code] + + +def get_language_codes(locale, language_names, remainder=None): + language_names = set(x.strip().lower() for x in language_names if x) + + for k, v in get_language_names(locale).items(): + v = v.lower() + if v in language_names: + language_names.remove(v) + yield k + + if remainder is not None: + remainder.extend(language_names) diff --git a/cps/jinjia.py b/cps/jinjia.py index 37f9ce30..ffd6832c 100644 --- a/cps/jinjia.py +++ b/cps/jinjia.py @@ -71,11 +71,7 @@ def shortentitle_filter(s, nchar=20): @jinjia.app_template_filter('mimetype') def mimetype_filter(val): - try: - s = mimetypes.types_map['.' + val] - except Exception: - s = 'application/octet-stream' - return s + return mimetypes.types_map.get('.' + val, 'application/octet-stream') @jinjia.app_template_filter('formatdate') diff --git a/cps/ldap_login.py b/cps/ldap_login.py deleted file mode 100644 index dcded6f6..00000000 --- a/cps/ldap_login.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- - -# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) -# Copyright (C) 2019 Krakinou -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from __future__ import division, print_function, unicode_literals -import base64 - -try: - from flask_simpleldap import LDAP # , LDAPException - ldap_support = True -except ImportError: - ldap_support = False - -from . import config, logger - -log = logger.create() - -class Ldap(): - - def __init__(self): - self.ldap = None - return - - def init_app(self, app): - if ldap_support and config.config_login_type == 1: - app.config['LDAP_HOST'] = config.config_ldap_provider_url - app.config['LDAP_PORT'] = config.config_ldap_port - app.config['LDAP_SCHEMA'] = config.config_ldap_schema - app.config['LDAP_USERNAME'] = config.config_ldap_user_object.replace('%s', config.config_ldap_serv_username)\ - + ',' + config.config_ldap_dn - app.config['LDAP_PASSWORD'] = base64.b64decode(config.config_ldap_serv_password) - if config.config_ldap_use_ssl: - app.config['LDAP_USE_SSL'] = True - if config.config_ldap_use_tls: - app.config['LDAP_USE_TLS'] = True - app.config['LDAP_REQUIRE_CERT'] = config.config_ldap_require_cert - if config.config_ldap_require_cert: - app.config['LDAP_CERT_PATH'] = config.config_ldap_cert_path - app.config['LDAP_BASE_DN'] = config.config_ldap_dn - app.config['LDAP_USER_OBJECT_FILTER'] = config.config_ldap_user_object - if config.config_ldap_openldap: - app.config['LDAP_OPENLDAP'] = True - - # app.config['LDAP_BASE_DN'] = 'ou=users,dc=yunohost,dc=org' - # app.config['LDAP_USER_OBJECT_FILTER'] = '(uid=%s)' - self.ldap = LDAP(app) - - elif config.config_login_type == 1 and not ldap_support: - log.error('Cannot activate ldap support, did you run \'pip install --target vendor -r optional-requirements.txt\'?') - - @classmethod - def ldap_supported(cls): - return ldap_support diff --git a/cps/logger.py b/cps/logger.py index 6b13f50d..3a540683 100644 --- a/cps/logger.py +++ b/cps/logger.py @@ -157,3 +157,8 @@ class StderrLogger(object): self.buffer += message except Exception: self.log.debug("Logging Error") + + +# if debugging, start logging to stderr immediately +if os.environ.get('FLASK_DEBUG', None): + setup(LOG_TO_STDERR, logging.DEBUG) diff --git a/cps/oauth_bb.py b/cps/oauth_bb.py index f258e706..39777911 100644 --- a/cps/oauth_bb.py +++ b/cps/oauth_bb.py @@ -24,7 +24,6 @@ from __future__ import division, print_function, unicode_literals import json from functools import wraps -from oauth import OAuthBackend from flask import session, request, make_response, abort from flask import Blueprint, flash, redirect, url_for @@ -37,6 +36,7 @@ from sqlalchemy.orm.exc import NoResultFound from . import constants, logger, config, app, ub from .web import login_required +from .oauth import OAuthBackend # from .web import github_oauth_required diff --git a/cps/opds.py b/cps/opds.py index 48e2b968..657b3861 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -31,7 +31,7 @@ from flask_login import current_user from sqlalchemy.sql.expression import func, text, or_, and_ from werkzeug.security import check_password_hash -from . import logger, config, db, ub, ldap1 +from . import constants, logger, config, db, ub, services from .helper import fill_indexpage, get_download_link, get_book_cover from .pagination import Pagination from .web import common_filters, get_search_results, render_read_books, download_required @@ -40,7 +40,6 @@ from .web import common_filters, get_search_results, render_read_books, download opds = Blueprint('opds', __name__) log = logger.create() -ldap_support = ldap1.ldap_supported() def requires_basic_auth_if_no_ano(f): @@ -51,8 +50,8 @@ def requires_basic_auth_if_no_ano(f): if not auth or not check_auth(auth.username, auth.password): return authenticate() return f(*args, **kwargs) - if config.config_login_type == 1 and ldap_support: - return ldap1.ldap.basic_auth_required(f) + if config.config_login_type == constants.LOGIN_LDAP and services.ldap: + return services.ldap.basic_auth_required(f) return decorated diff --git a/cps/server.py b/cps/server.py index 5945b332..1d564824 100644 --- a/cps/server.py +++ b/cps/server.py @@ -197,6 +197,7 @@ class WebServer: self.stop() def stop(self, restart=False): + log.info("webserver stop (restart=%s)", restart) self.restart = restart if self.wsgiserver: if _GEVENT: diff --git a/cps/services/__init__.py b/cps/services/__init__.py new file mode 100644 index 00000000..90607160 --- /dev/null +++ b/cps/services/__init__.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2019 pwr +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals + +from .. import logger + + +log = logger.create() + + +try: from . import goodreads +except ImportError as err: + log.warning("goodreads: %s", err) + goodreads = None + + +try: from . import simpleldap as ldap +except ImportError as err: + log.warning("simpleldap: %s", err) + ldap = None diff --git a/cps/services/goodreads.py b/cps/services/goodreads.py new file mode 100644 index 00000000..55161c7a --- /dev/null +++ b/cps/services/goodreads.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018-2019 OzzieIsaacs, pwr +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import time +from functools import reduce + +from goodreads.client import GoodreadsClient + +try: import Levenshtein +except ImportError: Levenshtein = False + +from .. import logger + + +log = logger.create() +_client = None # type: GoodreadsClient + +# GoodReads TOS allows for 24h caching of data +_CACHE_TIMEOUT = 23 * 60 * 60 # 23 hours (in seconds) +_AUTHORS_CACHE = {} + + +def connect(key=None, secret=None, enabled=True): + global _client + + if not enabled or not key or not secret: + _client = None + return + + if _client: + # make sure the configuration has not changed since last we used the client + if _client.client_key != key or _client.client_secret != secret: + _client = None + + if not _client: + _client = GoodreadsClient(key, secret) + + +def get_author_info(author_name): + now = time.time() + author_info = _AUTHORS_CACHE.get(author_name, None) + if author_info: + if now < author_info._timestamp + _CACHE_TIMEOUT: + return author_info + # clear expired entries + del _AUTHORS_CACHE[author_name] + + if not _client: + log.warning("failed to get a Goodreads client") + return + + try: + author_info = _client.find_author(author_name=author_name) + except Exception as ex: + # Skip goodreads, if site is down/inaccessible + log.warning('Goodreads website is down/inaccessible? %s', ex) + return + + if author_info: + author_info._timestamp = now + _AUTHORS_CACHE[author_name] = author_info + return author_info + + +def get_other_books(author_info, library_books=None): + # Get all identifiers (ISBN, Goodreads, etc) and filter author's books by that list so we show fewer duplicates + # Note: Not all images will be shown, even though they're available on Goodreads.com. + # See https://www.goodreads.com/topic/show/18213769-goodreads-book-images + + if not author_info: + return + + identifiers = [] + library_titles = [] + if library_books: + identifiers = list(reduce(lambda acc, book: acc + [i.val for i in book.identifiers if i.val], library_books, [])) + library_titles = [book.title for book in library_books] + + for book in author_info.books: + if book.isbn in identifiers: + continue + if book.gid["#text"] in identifiers: + continue + + if Levenshtein and library_titles: + goodreads_title = book._book_dict['title_without_series'] + if any(Levenshtein.ratio(goodreads_title, title) > 0.7 for title in library_titles): + continue + + yield book diff --git a/cps/services/simpleldap.py b/cps/services/simpleldap.py new file mode 100644 index 00000000..cd0c16ce --- /dev/null +++ b/cps/services/simpleldap.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018-2019 OzzieIsaacs, pwr +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import division, print_function, unicode_literals +import base64 + +from flask_simpleldap import LDAP +from ldap import SERVER_DOWN, INVALID_CREDENTIALS + +from .. import constants, logger + + +log = logger.create() +_ldap = None + + +def init_app(app, config): + global _ldap + + if config.config_login_type != constants.LOGIN_LDAP: + _ldap = None + return + + app.config['LDAP_HOST'] = config.config_ldap_provider_url + app.config['LDAP_PORT'] = config.config_ldap_port + app.config['LDAP_SCHEMA'] = config.config_ldap_schema + app.config['LDAP_USERNAME'] = config.config_ldap_user_object.replace('%s', config.config_ldap_serv_username)\ + + ',' + config.config_ldap_dn + app.config['LDAP_PASSWORD'] = base64.b64decode(config.config_ldap_serv_password) + app.config['LDAP_REQUIRE_CERT'] = bool(config.config_ldap_require_cert) + if config.config_ldap_require_cert: + app.config['LDAP_CERT_PATH'] = config.config_ldap_cert_path + app.config['LDAP_BASE_DN'] = config.config_ldap_dn + app.config['LDAP_USER_OBJECT_FILTER'] = config.config_ldap_user_object + app.config['LDAP_USE_SSL'] = bool(config.config_ldap_use_ssl) + app.config['LDAP_USE_TLS'] = bool(config.config_ldap_use_tls) + app.config['LDAP_OPENLDAP'] = bool(config.config_ldap_openldap) + + # app.config['LDAP_BASE_DN'] = 'ou=users,dc=yunohost,dc=org' + # app.config['LDAP_USER_OBJECT_FILTER'] = '(uid=%s)' + _ldap = LDAP(app) + + +def basic_auth_required(func): + return _ldap.basic_auth_required(func) + + +def bind_user(username, password): + '''Attempts a LDAP login. + + :returns: True if login succeeded, False if login failed, None if server unavailable. + ''' + try: + result = _ldap.bind_user(username, password) + log.debug("LDAP login '%s': %r", username, result) + return result is not None + except SERVER_DOWN as ex: + log.warning('LDAP Server down: %s', ex) + return None + except INVALID_CREDENTIALS as ex: + log.info("LDAP login '%s' failed: %s", username, ex) + return False diff --git a/cps/subproc_wrapper.py b/cps/subproc_wrapper.py index 8dceca65..088cb3d5 100644 --- a/cps/subproc_wrapper.py +++ b/cps/subproc_wrapper.py @@ -43,3 +43,13 @@ def process_open(command, quotes=(), env=None, sout=subprocess.PIPE, serr=subpro exc_command = [x for x in command] return subprocess.Popen(exc_command, shell=False, stdout=sout, stderr=serr, universal_newlines=True, env=env) + + +def process_wait(command, serr=subprocess.PIPE): + '''Run command, wait for process to terminate, and return an iterator over lines of its output.''' + p = process_open(command, serr=serr) + p.wait() + for l in p.stdout.readlines(): + if isinstance(l, bytes): + l = l.decode('utf-8') + yield l diff --git a/cps/templates/admin.html b/cps/templates/admin.html index 6618a113..17b84f34 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -69,7 +69,7 @@
{{_('Log level')}}
-
{{config.get_Log_Level()}}
+
{{config.get_log_level()}}
{{_('Port')}}
diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 59800b98..de4063c9 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -94,8 +94,8 @@
- - @@ -327,10 +327,10 @@
- {% if not origin %} + {% if show_back_button %} {{_('Back')}} {% endif %} - {% if success %} + {% if show_login_button %} {{_('Login')}} {% endif %}
diff --git a/cps/ub.py b/cps/ub.py index 02623448..a2727e24 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -19,10 +19,8 @@ # along with this program. If not, see . from __future__ import division, print_function, unicode_literals -import sys import os import datetime -import json from binascii import hexlify from flask import g @@ -40,22 +38,18 @@ from sqlalchemy.orm import relationship, sessionmaker from sqlalchemy.ext.declarative import declarative_base from werkzeug.security import generate_password_hash -from . import constants, logger, cli +from . import constants session = None - -engine = create_engine(u'sqlite:///{0}'.format(cli.settingspath), echo=False) Base = declarative_base() def get_sidebar_config(kwargs=None): kwargs = kwargs or [] if 'content' in kwargs: - if not isinstance(kwargs['content'], Settings): - content = not kwargs['content'].role_anonymous() - else: - content = False + content = kwargs['content'] + content = isinstance(content, UserBase) and not content.role_anonymous() else: content = 'conf' in kwargs sidebar = list() @@ -148,7 +142,7 @@ class UserBase: @property def is_anonymous(self): - return False + return self.role_anonymous() def get_id(self): return str(self.id) @@ -200,7 +194,6 @@ class Anonymous(AnonymousUserMixin, UserBase): def loadSettings(self): data = session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() # type: User - settings = session.query(Settings).first() self.nickname = data.nickname self.role = data.role self.id=data.id @@ -208,9 +201,11 @@ class Anonymous(AnonymousUserMixin, UserBase): self.default_language = data.default_language self.locale = data.locale self.mature_content = data.mature_content - self.anon_browse = settings.config_anonbrowse self.kindle_mail = data.kindle_mail + # settings = session.query(Settings).first() + # self.anon_browse = settings.config_anonbrowse + def role_admin(self): return False @@ -220,7 +215,7 @@ class Anonymous(AnonymousUserMixin, UserBase): @property def is_anonymous(self): - return self.anon_browse + return True # self.anon_browse @property def is_authenticated(self): @@ -295,78 +290,6 @@ class Registration(Base): return u"".format(self.domain) -# Baseclass for representing settings in app.db with email server settings and Calibre database settings -# (application settings) -class Settings(Base): - __tablename__ = 'settings' - - id = Column(Integer, primary_key=True) - mail_server = Column(String) - mail_port = Column(Integer, default=25) - mail_use_ssl = Column(SmallInteger, default=0) - mail_login = Column(String) - mail_password = Column(String) - mail_from = Column(String) - config_calibre_dir = Column(String) - config_port = Column(Integer, default=constants.DEFAULT_PORT) - config_certfile = Column(String) - config_keyfile = Column(String) - config_calibre_web_title = Column(String, default=u'Calibre-Web') - config_books_per_page = Column(Integer, default=60) - config_random_books = Column(Integer, default=4) - config_authors_max = Column(Integer, default=0) - config_read_column = Column(Integer, default=0) - config_title_regex = Column(String, default=u'^(A|The|An|Der|Die|Das|Den|Ein|Eine|Einen|Dem|Des|Einem|Eines)\s+') - config_log_level = Column(SmallInteger, default=logger.DEFAULT_LOG_LEVEL) - config_access_log = Column(SmallInteger, default=0) - config_uploading = Column(SmallInteger, default=0) - config_anonbrowse = Column(SmallInteger, default=0) - config_public_reg = Column(SmallInteger, default=0) - config_default_role = Column(SmallInteger, default=0) - config_default_show = Column(SmallInteger, default=6143) - config_columns_to_ignore = Column(String) - config_use_google_drive = Column(Boolean) - config_google_drive_folder = Column(String) - config_google_drive_watch_changes_response = Column(String) - config_remote_login = Column(Boolean) - config_use_goodreads = Column(Boolean) - config_goodreads_api_key = Column(String) - config_goodreads_api_secret = Column(String) - config_login_type = Column(Integer, default=0) - # config_use_ldap = Column(Boolean) - config_ldap_provider_url = Column(String) - config_ldap_dn = Column(String) - # config_use_github_oauth = Column(Boolean) - config_github_oauth_client_id = Column(String) - config_github_oauth_client_secret = Column(String) - # config_use_google_oauth = Column(Boolean) - config_google_oauth_client_id = Column(String) - config_google_oauth_client_secret = Column(String) - config_ldap_provider_url = Column(String, default='localhost') - config_ldap_port = Column(SmallInteger, default=389) - config_ldap_schema = Column(String, default='ldap') - config_ldap_serv_username = Column(String) - config_ldap_serv_password = Column(String) - config_ldap_use_ssl = Column(Boolean, default=False) - config_ldap_use_tls = Column(Boolean, default=False) - config_ldap_require_cert = Column(Boolean, default=False) - config_ldap_cert_path = Column(String) - config_ldap_dn = Column(String) - config_ldap_user_object = Column(String) - config_ldap_openldap = Column(Boolean) - config_mature_content_tags = Column(String) - config_logfile = Column(String) - config_access_logfile = Column(String) - config_ebookconverter = Column(Integer, default=0) - config_converterpath = Column(String) - config_calibre = Column(String) - config_rarfile_location = Column(String) - config_theme = Column(Integer, default=0) - config_updatechannel = Column(Integer, default=0) - - def __repr__(self): - pass - class RemoteAuthToken(Base): __tablename__ = 'remote_auth_token' @@ -385,153 +308,11 @@ class RemoteAuthToken(Base): return '' % self.id -# Class holds all application specific settings in calibre-web -class Config: - def __init__(self): - self.db_configured = None - self.config_logfile = None - self.loadSettings() - - def loadSettings(self): - data = session.query(Settings).first() # type: Settings - - self.config_calibre_dir = data.config_calibre_dir - self.config_port = data.config_port - self.config_certfile = data.config_certfile - self.config_keyfile = data.config_keyfile - self.config_calibre_web_title = data.config_calibre_web_title - self.config_books_per_page = data.config_books_per_page - self.config_random_books = data.config_random_books - self.config_authors_max = data.config_authors_max - self.config_title_regex = data.config_title_regex - self.config_read_column = data.config_read_column - self.config_log_level = data.config_log_level - self.config_access_log = data.config_access_log - self.config_uploading = data.config_uploading - self.config_anonbrowse = data.config_anonbrowse - self.config_public_reg = data.config_public_reg - self.config_default_role = data.config_default_role - self.config_default_show = data.config_default_show - self.config_columns_to_ignore = data.config_columns_to_ignore - self.config_use_google_drive = data.config_use_google_drive - self.config_google_drive_folder = data.config_google_drive_folder - self.config_ebookconverter = data.config_ebookconverter - self.config_converterpath = data.config_converterpath - self.config_calibre = data.config_calibre - if data.config_google_drive_watch_changes_response: - self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response) - else: - self.config_google_drive_watch_changes_response=None - self.config_columns_to_ignore = data.config_columns_to_ignore - self.db_configured = bool(self.config_calibre_dir is not None and - (not self.config_use_google_drive or os.path.exists(self.config_calibre_dir + '/metadata.db'))) - self.config_remote_login = data.config_remote_login - self.config_use_goodreads = data.config_use_goodreads - self.config_goodreads_api_key = data.config_goodreads_api_key - self.config_goodreads_api_secret = data.config_goodreads_api_secret - self.config_login_type = data.config_login_type - # self.config_use_ldap = data.config_use_ldap - self.config_ldap_user_object = data.config_ldap_user_object - self.config_ldap_openldap = data.config_ldap_openldap - self.config_ldap_provider_url = data.config_ldap_provider_url - self.config_ldap_port = data.config_ldap_port - self.config_ldap_schema = data.config_ldap_schema - self.config_ldap_serv_username = data.config_ldap_serv_username - self.config_ldap_serv_password = data.config_ldap_serv_password - self.config_ldap_use_ssl = data.config_ldap_use_ssl - self.config_ldap_use_tls = data.config_ldap_use_ssl - self.config_ldap_require_cert = data.config_ldap_require_cert - self.config_ldap_cert_path = data.config_ldap_cert_path - self.config_ldap_dn = data.config_ldap_dn - # self.config_use_github_oauth = data.config_use_github_oauth - self.config_github_oauth_client_id = data.config_github_oauth_client_id - self.config_github_oauth_client_secret = data.config_github_oauth_client_secret - # self.config_use_google_oauth = data.config_use_google_oauth - self.config_google_oauth_client_id = data.config_google_oauth_client_id - self.config_google_oauth_client_secret = data.config_google_oauth_client_secret - self.config_mature_content_tags = data.config_mature_content_tags or u'' - self.config_logfile = data.config_logfile or u'' - self.config_access_logfile = data.config_access_logfile or u'' - self.config_rarfile_location = data.config_rarfile_location - self.config_theme = data.config_theme - self.config_updatechannel = data.config_updatechannel - logger.setup(self.config_logfile, self.config_log_level) - - @property - def get_update_channel(self): - return self.config_updatechannel - - def get_config_certfile(self): - if cli.certfilepath: - return cli.certfilepath - if cli.certfilepath is "": - return None - return self.config_certfile - - def get_config_keyfile(self): - if cli.keyfilepath: - return cli.keyfilepath - if cli.certfilepath is "": - return None - return self.config_keyfile - - def get_config_ipaddress(self): - return cli.ipadress or "" - - def get_ipaddress_type(self): - return cli.ipv6 - - def _has_role(self, role_flag): - return constants.has_flag(self.config_default_role, role_flag) - - def role_admin(self): - return self._has_role(constants.ROLE_ADMIN) - - def role_download(self): - return self._has_role(constants.ROLE_DOWNLOAD) - - def role_viewer(self): - return self._has_role(constants.ROLE_VIEWER) - - def role_upload(self): - return self._has_role(constants.ROLE_UPLOAD) - - def role_edit(self): - return self._has_role(constants.ROLE_EDIT) - - def role_passwd(self): - return self._has_role(constants.ROLE_PASSWD) - - def role_edit_shelfs(self): - return self._has_role(constants.ROLE_EDIT_SHELFS) - - def role_delete_books(self): - return self._has_role(constants.ROLE_DELETE_BOOKS) - - def show_element_new_user(self, value): - return constants.has_flag(self.config_default_show, value) - - def show_detail_random(self): - return self.show_element_new_user(constants.DETAIL_RANDOM) - - def show_mature_content(self): - return self.show_element_new_user(constants.MATURE_CONTENT) - - def mature_content_tags(self): - if sys.version_info > (3, 0): # Python3 str, Python2 unicode - lstrip = str.lstrip - else: - lstrip = unicode.lstrip - return list(map(lstrip, self.config_mature_content_tags.split(","))) - - def get_Log_Level(self): - return logger.get_level_name(self.config_log_level) - - # Migrate database to current version, has to be updated after every database change. Currently migration from # everywhere to curent should work. Migration is done by checking if relevant coloums are existing, and than adding # rows with SQL commands -def migrate_Database(): +def migrate_Database(session): + engine = session.bind if not engine.dialect.has_table(engine.connect(), "book_read_link"): ReadBook.__table__.create(bind=engine) if not engine.dialect.has_table(engine.connect(), "bookmark"): @@ -547,45 +328,12 @@ def migrate_Database(): conn = engine.connect() conn.execute("insert into registration (domain) values('%.%')") session.commit() - try: - session.query(exists().where(Settings.config_use_google_drive)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_use_google_drive` INTEGER DEFAULT 0") - conn.execute("ALTER TABLE Settings ADD column `config_google_drive_folder` String DEFAULT ''") - conn.execute("ALTER TABLE Settings ADD column `config_google_drive_watch_changes_response` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_columns_to_ignore)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_columns_to_ignore` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_default_role)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_default_role` SmallInteger DEFAULT 0") - session.commit() - try: - session.query(exists().where(Settings.config_authors_max)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_authors_max` INTEGER DEFAULT 0") - session.commit() try: session.query(exists().where(BookShelf.order)).scalar() except exc.OperationalError: # Database is not compatible, some rows are missing conn = engine.connect() conn.execute("ALTER TABLE book_shelf_link ADD column 'order' INTEGER DEFAULT 1") session.commit() - try: - session.query(exists().where(Settings.config_rarfile_location)).scalar() - session.commit() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_rarfile_location` String DEFAULT ''") - session.commit() try: create = False session.query(exists().where(User.sidebar_view)).scalar() @@ -618,145 +366,6 @@ def migrate_Database(): if session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() is None: create_anonymous_user() - try: - session.query(exists().where(Settings.config_remote_login)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_remote_login` INTEGER DEFAULT 0") - try: - session.query(exists().where(Settings.config_use_goodreads)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_use_goodreads` INTEGER DEFAULT 0") - conn.execute("ALTER TABLE Settings ADD column `config_goodreads_api_key` String DEFAULT ''") - conn.execute("ALTER TABLE Settings ADD column `config_goodreads_api_secret` String DEFAULT ''") - try: - session.query(exists().where(Settings.config_mature_content_tags)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_mature_content_tags` String DEFAULT ''") - try: - session.query(exists().where(Settings.config_default_show)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_default_show` SmallInteger DEFAULT 2047") - session.commit() - try: - session.query(exists().where(Settings.config_logfile)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_logfile` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_certfile)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_certfile` String DEFAULT ''") - conn.execute("ALTER TABLE Settings ADD column `config_keyfile` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_read_column)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_read_column` INTEGER DEFAULT 0") - session.commit() - try: - session.query(exists().where(Settings.config_ebookconverter)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ebookconverter` INTEGER DEFAULT 0") - conn.execute("ALTER TABLE Settings ADD column `config_converterpath` String DEFAULT ''") - conn.execute("ALTER TABLE Settings ADD column `config_calibre` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_login_type)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_login_type` INTEGER DEFAULT 0") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_provider_url)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_provider_url` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_port)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_port` INTEGER DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_schema)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_schema` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_serv_username)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_serv_username` String DEFAULT ''") - conn.execute("ALTER TABLE Settings ADD column `config_ldap_serv_password` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_use_ssl)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_use_ssl` INTEGER DEFAULT 0") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_use_tls)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_use_tls` INTEGER DEFAULT 0") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_require_cert)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_require_cert` INTEGER DEFAULT 0") - conn.execute("ALTER TABLE Settings ADD column `config_ldap_cert_path` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_dn)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_dn` String DEFAULT ''") - conn.execute("ALTER TABLE Settings ADD column `config_github_oauth_client_id` String DEFAULT ''") - conn.execute("ALTER TABLE Settings ADD column `config_github_oauth_client_secret` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_user_object)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_user_object` String DEFAULT ''") - session.commit() - try: - session.query(exists().where(Settings.config_ldap_openldap)).scalar() - except exc.OperationalError: - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_ldap_openldap` INTEGER DEFAULT 0") - session.commit() - try: - session.query(exists().where(Settings.config_theme)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_theme` INTEGER DEFAULT 0") - session.commit() - try: - session.query(exists().where(Settings.config_updatechannel)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_updatechannel` INTEGER DEFAULT 0") - session.commit() - try: - session.query(exists().where(Settings.config_access_log)).scalar() - except exc.OperationalError: # Database is not compatible, some rows are missing - conn = engine.connect() - conn.execute("ALTER TABLE Settings ADD column `config_access_log` INTEGER DEFAULT 0") - conn.execute("ALTER TABLE Settings ADD column `config_access_logfile` String DEFAULT ''") - session.commit() try: # check if one table with autoincrement is existing (should be user table) conn = engine.connect() @@ -792,42 +401,13 @@ def migrate_Database(): session.commit() -def clean_database(): +def clean_database(session): # Remove expired remote login tokens now = datetime.datetime.now() session.query(RemoteAuthToken).filter(now > RemoteAuthToken.expiration).delete() - - -def create_default_config(): - settings = Settings() - settings.mail_server = "mail.example.com" - settings.mail_port = 25 - settings.mail_use_ssl = 0 - settings.mail_login = "mail@example.com" - settings.mail_password = "mypassword" - settings.mail_from = "automailer " - - session.add(settings) session.commit() -def get_mail_settings(): - settings = session.query(Settings).first() - - if not settings: - return {} - - data = { - 'mail_server': settings.mail_server, - 'mail_port': settings.mail_port, - 'mail_use_ssl': settings.mail_use_ssl, - 'mail_login': settings.mail_login, - 'mail_password': settings.mail_password, - 'mail_from': settings.mail_from - } - - return data - # Save downloaded books per user in calibre-web's own database def update_download(book_id, user_id): check = session.query(Downloads).filter(Downloads.user_id == user_id).filter(Downloads.book_id == @@ -844,7 +424,7 @@ def delete_download(book_id): session.commit() # Generate user Guest (translated text), as anoymous user, no rights -def create_anonymous_user(): +def create_anonymous_user(session): user = User() user.nickname = "Guest" user.email = 'no@email' @@ -859,7 +439,7 @@ def create_anonymous_user(): # Generate User admin with admin123 password, and access to everything -def create_admin_user(): +def create_admin_user(session): user = User() user.nickname = "admin" user.role = constants.ADMIN_USER_ROLES @@ -873,23 +453,37 @@ def create_admin_user(): except Exception: session.rollback() -def init_db(): + +def init_db(app_db_path): # Open session for database connection global session + + engine = create_engine(u'sqlite:///{0}'.format(app_db_path), echo=False) + Session = sessionmaker() Session.configure(bind=engine) session = Session() - - if not os.path.exists(cli.settingspath): - try: - Base.metadata.create_all(engine) - create_default_config() - create_admin_user() - create_anonymous_user() - except Exception: - raise + if os.path.exists(app_db_path): + Base.metadata.create_all(engine) + migrate_Database(session) + clean_database(session) else: Base.metadata.create_all(engine) - migrate_Database() - clean_database() + create_admin_user(session) + create_anonymous_user(session) + + +def dispose(): + global session + + engine = None + if session: + engine = session.bind + try: session.close() + except: pass + session = None + + if engine: + try: engine.dispose() + except: pass diff --git a/cps/updater.py b/cps/updater.py index 33012db1..3dbd386f 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -22,7 +22,6 @@ import sys import os import datetime import json -import requests import shutil import threading import time @@ -30,6 +29,7 @@ import zipfile from io import BytesIO from tempfile import gettempdir +import requests from babel.dates import format_datetime from flask_babel import gettext as _ @@ -58,16 +58,14 @@ class Updater(threading.Thread): self.updateIndex = None def get_current_version_info(self): - if config.get_update_channel == constants.UPDATE_STABLE: + if config.config_updatechannel == constants.UPDATE_STABLE: return self._stable_version_info() - else: - return self._nightly_version_info() + return self._nightly_version_info() def get_available_updates(self, request_method): - if config.get_update_channel == constants.UPDATE_STABLE: + if config.config_updatechannel == constants.UPDATE_STABLE: return self._stable_available_updates(request_method) - else: - return self._nightly_available_updates(request_method) + return self._nightly_available_updates(request_method) def run(self): try: @@ -430,10 +428,9 @@ class Updater(threading.Thread): return json.dumps(status) def _get_request_path(self): - if config.get_update_channel == constants.UPDATE_STABLE: + if config.config_updatechannel == constants.UPDATE_STABLE: return self.updateFile - else: - return _REPOSITORY_API_URL + '/zipball/master' + return _REPOSITORY_API_URL + '/zipball/master' def _load_remote_data(self, repository_url): status = { diff --git a/cps/web.py b/cps/web.py index 491fbbf6..a3c9e1f4 100644 --- a/cps/web.py +++ b/cps/web.py @@ -41,18 +41,20 @@ from werkzeug.exceptions import default_exceptions from werkzeug.datastructures import Headers from werkzeug.security import generate_password_hash, check_password_hash -from . import constants, logger, isoLanguages, ldap1 -from . import global_WorkerThread, searched_ids, lm, babel, db, ub, config, get_locale, app, language_table +from . import constants, logger, isoLanguages, services +from . import global_WorkerThread, searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download from .helper import common_filters, get_search_results, fill_indexpage, speaking_language, check_valid_domain, \ - order_authors, get_typeahead, render_task_status, json_serial, get_unique_other_books, get_cc_columns, \ + order_authors, get_typeahead, render_task_status, json_serial, get_cc_columns, \ get_book_cover, get_download_link, send_mail, generate_random_password, send_registration_mail, \ check_send_to_kindle, check_read_formats, lcase from .pagination import Pagination from .redirect import redirect_back -feature_support = dict() -feature_support['ldap'] = ldap1.ldap_supported() +feature_support = { + 'ldap': bool(services.ldap), + 'goodreads': bool(services.goodreads) + } try: from .oauth_bb import oauth_check, register_user_with_oauth, logout_oauth_user, get_oauth_status @@ -61,12 +63,6 @@ except ImportError: feature_support['oauth'] = False oauth_check = {} -try: - from goodreads.client import GoodreadsClient - feature_support['goodreads'] = True -except ImportError: - feature_support['goodreads'] = False - try: from functools import wraps except ImportError: @@ -230,6 +226,8 @@ def render_title_template(*args, **kwargs): @web.before_app_request def before_request(): + # log.debug("before_request: %s %s %r", request.method, request.path, getattr(request, 'locale', None)) + request._locale = negociate_locale() g.user = current_user g.allow_registration = config.config_public_reg g.allow_upload = config.config_uploading @@ -292,7 +290,7 @@ def toggle_read(book_id): ub.session.commit() else: try: - db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort) + db.update_title_sort(config) book = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first() read_status = getattr(book, 'custom_column_' + str(config.config_read_column)) if len(read_status): @@ -396,10 +394,10 @@ def get_series_json(): def get_languages_json(): if request.method == "GET": query = request.args.get('q').lower() - languages = language_table[get_locale()] - entries_start = [s for key, s in languages.items() if s.lower().startswith(query.lower())] + language_names = isoLanguages.get_language_names(get_locale()) + entries_start = [s for key, s in language_names.items() if s.lower().startswith(query.lower())] if len(entries_start) < 5: - entries = [s for key, s in languages.items() if query in s.lower()] + entries = [s for key, s in language_names.items() if query in s.lower()] entries_start.extend(entries[0:(5-len(entries_start))]) entries_start = list(set(entries_start)) json_dumps = json.dumps([dict(name=r) for r in entries_start[0:5]]) @@ -534,29 +532,26 @@ def render_hot_books(page): abort(404) -def render_author_books(page, book_id, order): - entries, __, pagination = fill_indexpage(page, db.Books, db.Books.authors.any(db.Authors.id == book_id), + +def render_author_books(page, author_id, order): + entries, __, pagination = fill_indexpage(page, db.Books, db.Books.authors.any(db.Authors.id == author_id), [order[0], db.Series.name, db.Books.series_index], db.books_series_link, db.Series) if entries is None or not len(entries): flash(_(u"Error opening eBook. File does not exist or file is not accessible:"), category="error") return redirect(url_for("web.index")) - name = db.session.query(db.Authors).filter(db.Authors.id == book_id).first().name.replace('|', ',') + author = db.session.query(db.Authors).get(author_id) + author_name = author.name.replace('|', ',') author_info = None other_books = [] - if feature_support['goodreads'] and config.config_use_goodreads: - try: - gc = GoodreadsClient(config.config_goodreads_api_key, config.config_goodreads_api_secret) - author_info = gc.find_author(author_name=name) - other_books = get_unique_other_books(entries.all(), author_info.books) - except Exception: - # Skip goodreads, if site is down/inaccessible - logger.error('Goodreads website is down/inaccessible') - - return render_title_template('author.html', entries=entries, pagination=pagination, id=book_id, - title=_(u"Author: %(name)s", name=name), author=author_info, other_books=other_books, + if services.goodreads and config.config_use_goodreads: + author_info = services.goodreads.get_author_info(author_name) + other_books = services.goodreads.get_other_books(author_info, entries) + + return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id, + title=_(u"Author: %(name)s", name=author_name), author=author_info, other_books=other_books, page="author") @@ -985,10 +980,7 @@ def serve_book(book_id, book_format): log.info('Serving book: %s', data.name) if config.config_use_google_drive: headers = Headers() - try: - headers["Content-Type"] = mimetypes.types_map['.' + book_format] - except KeyError: - headers["Content-Type"] = "application/octet-stream" + headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream") df = getFileFromEbooksFolder(book.path, data.name + "." + book_format) return do_gdrive_download(df, headers) else: @@ -1007,7 +999,7 @@ def download_link(book_id, book_format, anyname): @login_required @download_required def send_to_kindle(book_id, book_format, convert): - settings = ub.get_mail_settings() + settings = config.get_mail_settings() if settings.get("mail_server", "mail.example.com") == "mail.example.com": flash(_(u"Please configure the SMTP mail settings first..."), category="error") elif current_user.kindle_mail: @@ -1085,41 +1077,33 @@ def login(): return redirect(url_for('admin.basic_configuration')) if current_user is not None and current_user.is_authenticated: return redirect(url_for('web.index')) - if config.config_login_type == 1 and not feature_support['ldap']: + if config.config_login_type == constants.LOGIN_LDAP and not services.ldap: flash(_(u"Cannot activate LDAP authentication"), category="error") if request.method == "POST": form = request.form.to_dict() user = ub.session.query(ub.User).filter(func.lower(ub.User.nickname) == form['username'].strip().lower())\ .first() - if config.config_login_type == 1 and user and feature_support['ldap']: - try: - if ldap1.ldap.bind_user(form['username'], form['password']) is not None: - login_user(user, remember=True) - flash(_(u"you are now logged in as: '%(nickname)s'", nickname=user.nickname), - category="success") - return redirect_back(url_for("web.index")) - except ldap1.ldap.INVALID_CREDENTIALS as e: - log.error('Login Error: ' + str(e)) + if config.config_login_type == constants.LOGIN_LDAP and services.ldap and user: + login_result = services.ldap.bind_user(form['username'], form['password']) + if login_result: + login_user(user, remember=True) + flash(_(u"you are now logged in as: '%(nickname)s'", nickname=user.nickname), + category="success") + return redirect_back(url_for("web.index")) + if login_result is None: + flash(_(u"Could not login. LDAP server down, please contact your administrator"), category="error") + else: ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr) log.info('LDAP Login failed for user "%s" IP-adress: %s', form['username'], ipAdress) flash(_(u"Wrong Username or Password"), category="error") - except ldap1.ldap.SERVER_DOWN: - log.info('LDAP Login failed, LDAP Server down') - flash(_(u"Could not login. LDAP server down, please contact your administrator"), category="error") - '''except LDAPException as exception: - app.logger.error('Login Error: ' + str(exception)) - ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr) - app.logger.info('LDAP Login failed for user "' + form['username'] + ', IP-address :' + ipAdress) - flash(_(u"Wrong Username or Password"), category="error")''' else: - if user and check_password_hash(user.password, form['password']) and user.nickname is not "Guest": + if user and check_password_hash(user.password, form['password']) and user.nickname != "Guest": login_user(user, remember=True) flash(_(u"You are now logged in as: '%(nickname)s'", nickname=user.nickname), category="success") return redirect_back(url_for("web.index")) - else: - ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr) - log.info('Login failed for user "%s" IP-adress: %s', form['username'], ipAdress) - flash(_(u"Wrong Username or Password"), category="error") + ipAdress = request.headers.get('X-Forwarded-For', request.remote_addr) + log.info('Login failed for user "%s" IP-adress: %s', form['username'], ipAdress) + flash(_(u"Wrong Username or Password"), category="error") next_url = url_for('web.index')