From a00d93a2d908ff121967caf8b7eb79dfd0be7e0d Mon Sep 17 00:00:00 2001 From: Ozzieisaacs Date: Wed, 6 Feb 2019 21:52:24 +0100 Subject: [PATCH] Working again (basically) --- cps.py | 15 +- cps/__init__.py | 83 +++++++ cps/db.py | 2 +- cps/gdriveutils.py | 2 +- cps/helper.py | 35 ++- cps/pagination.py | 75 ++++++ cps/server.py | 72 +++--- cps/templates/admin.html | 10 +- cps/templates/book_edit.html | 12 +- cps/templates/detail.html | 42 ++-- cps/templates/index.html | 16 +- cps/templates/layout.html | 58 ++--- cps/templates/search_form.html | 2 +- cps/templates/tasks.html | 4 +- cps/ub.py | 27 +- cps/updater.py | 3 +- cps/web.py | 442 ++++++++++++++------------------- cps/worker.py | 52 ++-- 18 files changed, 500 insertions(+), 452 deletions(-) create mode 100644 cps/pagination.py diff --git a/cps.py b/cps.py index 055c0ffe..d38a9f33 100755 --- a/cps.py +++ b/cps.py @@ -1,21 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import os import sys - -base_path = os.path.dirname(os.path.abspath(__file__)) -# Insert local directories into path -sys.path.append(base_path) -sys.path.append(os.path.join(base_path, 'cps')) -sys.path.append(os.path.join(base_path, 'vendor')) - -from cps.server import Server +from cps import create_app +from cps.web import web +from cps import Server if __name__ == '__main__': + app = create_app() + app.register_blueprint(web) Server.startServer() - diff --git a/cps/__init__.py b/cps/__init__.py index faa18be5..1170a85a 100755 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -1,2 +1,85 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- + +# import logging +# from logging.handlers import SMTPHandler, RotatingFileHandler +# import os + +from flask import Flask# , request, current_app +from flask_login import LoginManager +from flask_babel import Babel # , lazy_gettext as _l +import cache_buster +from reverseproxy import ReverseProxied +import logging +from logging.handlers import RotatingFileHandler +from flask_principal import Principal +# from flask_sqlalchemy import SQLAlchemy +import os +import ub +from ub import Config, Settings +import cPickle + + +# Normal +babel = Babel() +lm = LoginManager() +lm.login_view = 'web.login' +lm.anonymous_user = ub.Anonymous + + + +ub_session = ub.session +# ub_session.start() +config = Config() + + +import db + +with open(os.path.join(config.get_main_dir, 'cps/translations/iso639.pickle'), 'rb') as f: + language_table = cPickle.load(f) + +searched_ids = {} + + +from worker import WorkerThread + +global_WorkerThread = WorkerThread() + +from server import server +Server = server() + + +def create_app(): + app = Flask(__name__) + app.wsgi_app = ReverseProxied(app.wsgi_app) + cache_buster.init_cache_busting(app) + + formatter = logging.Formatter( + "[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s") + try: + file_handler = RotatingFileHandler(config.get_config_logfile(), maxBytes=50000, backupCount=2) + except IOError: + file_handler = RotatingFileHandler(os.path.join(config.get_main_dir, "calibre-web.log"), + maxBytes=50000, backupCount=2) + # ToDo: reset logfile value in config class + file_handler.setFormatter(formatter) + app.logger.addHandler(file_handler) + app.logger.setLevel(config.config_log_level) + + app.logger.info('Starting Calibre Web...') + logging.getLogger("book_formats").addHandler(file_handler) + logging.getLogger("book_formats").setLevel(config.config_log_level) + Principal(app) + lm.init_app(app) + babel.init_app(app) + app.secret_key = os.getenv('SECRET_KEY', 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT') + Server.init_app(app) + db.setup_db() + global_WorkerThread.start() + + # app.config.from_object(config_class) + # db.init_app(app) + # login.init_app(app) + + + return app diff --git a/cps/db.py b/cps/db.py index 225bcf4e..c9fecd37 100755 --- a/cps/db.py +++ b/cps/db.py @@ -24,7 +24,7 @@ from sqlalchemy.orm import * import os import re import ast -from ub import config +from cps import config import ub import sys diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index cb36a413..1f0b8b83 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -27,7 +27,7 @@ except ImportError: gdrive_support = False import os -from ub import config +from cps import config import cli import shutil from flask import Response, stream_with_context diff --git a/cps/helper.py b/cps/helper.py index 0f489942..ba323449 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -20,14 +20,13 @@ import db -import ub +from cps import config from flask import current_app as app from tempfile import gettempdir import sys import os import re import unicodedata -# from io import BytesIO import worker import time from flask import send_from_directory, make_response, redirect, abort @@ -41,9 +40,10 @@ try: import gdriveutils as gd except ImportError: pass -import web +# import web import random from subproc_wrapper import process_open +import ub try: import unidecode @@ -51,11 +51,6 @@ try: except ImportError: use_unidecode = False -# Global variables -# updater_thread = None -global_WorkerThread = worker.WorkerThread() -global_WorkerThread.start() - def update_download(book_id, user_id): check = ub.session.query(ub.Downloads).filter(ub.Downloads.user_id == user_id).filter(ub.Downloads.book_id == @@ -73,7 +68,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id) app.logger.error("convert_book_format: " + error_message) return error_message - if ub.config.config_use_google_drive: + if config.config_use_google_drive: df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()) if df: datafile = os.path.join(calibrepath, book.path, data.name + u"." + old_book_format.lower()) @@ -133,7 +128,7 @@ def check_send_to_kindle(entry): """ if len(entry.data): bookformats=list() - if ub.config.config_ebookconverter == 0: + if config.config_ebookconverter == 0: # no converter - only for mobi and pdf formats for ele in iter(entry.data): if 'MOBI' in ele.format: @@ -156,11 +151,11 @@ def check_send_to_kindle(entry): bookformats.append({'format': 'Azw3','convert':0,'text':_('Send %(format)s to Kindle',format='Azw3')}) if 'PDF' in formats: bookformats.append({'format': 'Pdf','convert':0,'text':_('Send %(format)s to Kindle',format='Pdf')}) - if ub.config.config_ebookconverter >= 1: + if config.config_ebookconverter >= 1: if 'EPUB' in formats and not 'MOBI' in formats: bookformats.append({'format': 'Mobi','convert':1, 'text':_('Convert %(orig)s to %(format)s and send to Kindle',orig='Epub',format='Mobi')}) - if ub.config.config_ebookconverter == 2: + if config.config_ebookconverter == 2: if 'EPUB' in formats and not 'AZW3' in formats: bookformats.append({'format': 'Azw3','convert':1, 'text':_('Convert %(orig)s to %(format)s and send to Kindle',orig='Epub',format='Azw3')}) @@ -407,21 +402,21 @@ def generate_random_password(): ################################## External interface def update_dir_stucture(book_id, calibrepath, first_author = None): - if ub.config.config_use_google_drive: + if config.config_use_google_drive: return update_dir_structure_gdrive(book_id, first_author) else: return update_dir_structure_file(book_id, calibrepath, first_author) def delete_book(book, calibrepath, book_format): - if ub.config.config_use_google_drive: + if config.config_use_google_drive: return delete_book_gdrive(book, book_format) else: return delete_book_file(book, calibrepath, book_format) def get_book_cover(cover_path): - if ub.config.config_use_google_drive: + if config.config_use_google_drive: try: if not web.is_gdrive_ready(): return send_from_directory(os.path.join(os.path.dirname(__file__), "static"), "generic_cover.jpg") @@ -437,7 +432,7 @@ def get_book_cover(cover_path): # traceback.print_exc() return send_from_directory(os.path.join(os.path.dirname(__file__), "static"),"generic_cover.jpg") else: - return send_from_directory(os.path.join(ub.config.config_calibre_dir, cover_path), "cover.jpg") + return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg") # saves book cover to gdrive or locally @@ -447,7 +442,7 @@ def save_cover(url, book_path): web.app.logger.error("Cover is no jpg file, can't save") return False - if ub.config.config_use_google_drive: + if config.config_use_google_drive: tmpDir = gettempdir() f = open(os.path.join(tmpDir, "uploaded_cover.jpg"), "wb") f.write(img.content) @@ -456,7 +451,7 @@ def save_cover(url, book_path): web.app.logger.info("Cover is saved on Google Drive") return True - f = open(os.path.join(ub.config.config_calibre_dir, book_path, "cover.jpg"), "wb") + f = open(os.path.join(config.config_calibre_dir, book_path, "cover.jpg"), "wb") f.write(img.content) f.close() web.app.logger.info("Cover is saved") @@ -464,7 +459,7 @@ def save_cover(url, book_path): def do_download_file(book, book_format, data, headers): - if ub.config.config_use_google_drive: + if config.config_use_google_drive: startTime = time.time() df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) web.app.logger.debug(time.time() - startTime) @@ -473,7 +468,7 @@ def do_download_file(book, book_format, data, headers): else: abort(404) else: - filename = os.path.join(ub.config.config_calibre_dir, book.path) + filename = os.path.join(config.config_calibre_dir, book.path) if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): # ToDo: improve error handling web.app.logger.error('File not found: %s' % os.path.join(filename, data.name + "." + book_format)) diff --git a/cps/pagination.py b/cps/pagination.py new file mode 100644 index 00000000..891d616d --- /dev/null +++ b/cps/pagination.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018-2019 OzzieIsaacs, cervinko, jkrehm, bodybybuddha, ok11, +# andy29485, idalin, Kyosfonica, wuqi, Kennyl, lemmsh, +# falgh1, grunjol, csitko, ytils, xybydy, trasba, vrabe, +# ruben-herold, marblepebble, JackED42, SiphonSquirrel, +# apetresc, nanu-c, mutschler +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from math import ceil + +# simple pagination for the feed +class Pagination(object): + def __init__(self, page, per_page, total_count): + self.page = int(page) + self.per_page = int(per_page) + self.total_count = int(total_count) + + @property + def next_offset(self): + return int(self.page * self.per_page) + + @property + def previous_offset(self): + return int((self.page - 2) * self.per_page) + + @property + def last_offset(self): + last = int(self.total_count) - int(self.per_page) + if last < 0: + last = 0 + return int(last) + + @property + def pages(self): + return int(ceil(self.total_count / float(self.per_page))) + + @property + def has_prev(self): + return self.page > 1 + + @property + def has_next(self): + return self.page < self.pages + + # right_edge: last right_edges count of all pages are shown as number, means, if 10 pages are paginated -> 9,10 shwn + # left_edge: first left_edges count of all pages are shown as number -> 1,2 shwn + # left_current: left_current count below current page are shown as number, means if current page 5 -> 3,4 shwn + # left_current: right_current count above current page are shown as number, means if current page 5 -> 6,7 shwn + def iter_pages(self, left_edge=2, left_current=2, + right_current=4, right_edge=2): + last = 0 + left_current = self.page - left_current - 1 + right_current = self.page + right_current + 1 + right_edge = self.pages - right_edge + for num in range(1, (self.pages + 1)): + if num <= left_edge or (left_current < num < right_current) or num > right_edge: + if last + 1 != num: + yield None + yield num + last = num diff --git a/cps/server.py b/cps/server.py index 0531a729..a2c122dc 100644 --- a/cps/server.py +++ b/cps/server.py @@ -22,7 +22,7 @@ from socket import error as SocketError import sys import os import signal -import web +from cps import config, global_WorkerThread try: from gevent.pywsgi import WSGIServer @@ -42,82 +42,81 @@ class server: wsgiserver = None restart= False + app = None def __init__(self): signal.signal(signal.SIGINT, self.killServer) signal.signal(signal.SIGTERM, self.killServer) + def init_app(self,application): + self.app = application + def start_gevent(self): try: ssl_args = dict() - certfile_path = web.ub.config.get_config_certfile() - keyfile_path = web.ub.config.get_config_keyfile() + certfile_path = config.get_config_certfile() + keyfile_path = config.get_config_keyfile() if certfile_path and keyfile_path: if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path): ssl_args = {"certfile": certfile_path, "keyfile": keyfile_path} else: - web.app.logger.info('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s' % (certfile_path, keyfile_path)) + self.app.logger.info('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s' % (certfile_path, keyfile_path)) if os.name == 'nt': - self.wsgiserver= WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args) + self.wsgiserver= WSGIServer(('0.0.0.0', config.config_port), self.app, spawn=Pool(), **ssl_args) else: - self.wsgiserver = WSGIServer(('', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args) - web.py3_gevent_link = self.wsgiserver + self.wsgiserver = WSGIServer(('', config.config_port), self.app, spawn=Pool(), **ssl_args) self.wsgiserver.serve_forever() except SocketError: try: - web.app.logger.info('Unable to listen on \'\', trying on IPv4 only...') - self.wsgiserver = WSGIServer(('0.0.0.0', web.ub.config.config_port), web.app, spawn=Pool(), **ssl_args) - web.py3_gevent_link = self.wsgiserver + self.app.logger.info('Unable to listen on \'\', trying on IPv4 only...') + self.wsgiserver = WSGIServer(('0.0.0.0', config.config_port), self.app, spawn=Pool(), **ssl_args) self.wsgiserver.serve_forever() except (OSError, SocketError) as e: - web.app.logger.info("Error starting server: %s" % e.strerror) + self.app.logger.info("Error starting server: %s" % e.strerror) print("Error starting server: %s" % e.strerror) - web.helper.global_WorkerThread.stop() + global_WorkerThread.stop() sys.exit(1) except Exception: - web.app.logger.info("Unknown error while starting gevent") + self.app.logger.info("Unknown error while starting gevent") def startServer(self): if gevent_present: - web.app.logger.info('Starting Gevent server') + self.app.logger.info('Starting Gevent server') # leave subprocess out to allow forking for fetchers and processors self.start_gevent() else: try: ssl = None - web.app.logger.info('Starting Tornado server') - certfile_path = web.ub.config.get_config_certfile() - keyfile_path = web.ub.config.get_config_keyfile() + self.app.logger.info('Starting Tornado server') + certfile_path = config.get_config_certfile() + keyfile_path = config.get_config_keyfile() if certfile_path and keyfile_path: if os.path.isfile(certfile_path) and os.path.isfile(keyfile_path): ssl = {"certfile": certfile_path, "keyfile": keyfile_path} else: - web.app.logger.info('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s' % (certfile_path, keyfile_path)) + self.app.logger.info('The specified paths for the ssl certificate file and/or key file seem to be broken. Ignoring ssl. Cert path: %s | Key path: %s' % (certfile_path, keyfile_path)) # Max Buffersize set to 200MB - http_server = HTTPServer(WSGIContainer(web.app), + http_server = HTTPServer(WSGIContainer(self.app), max_buffer_size = 209700000, ssl_options=ssl) - http_server.listen(web.ub.config.config_port) + http_server.listen(config.config_port) self.wsgiserver=IOLoop.instance() self.wsgiserver.start() # wait for stop signal self.wsgiserver.close(True) except SocketError as e: - web.app.logger.info("Error starting server: %s" % e.strerror) + self.app.logger.info("Error starting server: %s" % e.strerror) print("Error starting server: %s" % e.strerror) - web.helper.global_WorkerThread.stop() + global_WorkerThread.stop() sys.exit(1) - # ToDo: Somehow caused by circular import under python3 refactor - if sys.version_info > (3, 0): - self.restart = web.py3_restart_Typ if self.restart == True: - web.app.logger.info("Performing restart of Calibre-Web") - web.helper.global_WorkerThread.stop() + self.app.logger.info("Performing restart of Calibre-Web") + global_WorkerThread.stop() if os.name == 'nt': arguments = ["\"" + sys.executable + "\""] for e in sys.argv: @@ -126,26 +125,17 @@ class server: else: os.execl(sys.executable, sys.executable, *sys.argv) else: - web.app.logger.info("Performing shutdown of Calibre-Web") - web.helper.global_WorkerThread.stop() + self.app.logger.info("Performing shutdown of Calibre-Web") + global_WorkerThread.stop() sys.exit(0) def setRestartTyp(self,starttyp): self.restart = starttyp - # ToDo: Somehow caused by circular import under python3 refactor - web.py3_restart_Typ = starttyp def killServer(self, signum, frame): self.stopServer() def stopServer(self): - # ToDo: Somehow caused by circular import under python3 refactor - if sys.version_info > (3, 0): - if not self.wsgiserver: - if gevent_present: - self.wsgiserver = web.py3_gevent_link - else: - self.wsgiserver = IOLoop.instance() if self.wsgiserver: if gevent_present: self.wsgiserver.close() @@ -155,10 +145,6 @@ class server: @staticmethod def getNameVersion(): if gevent_present: - return {'Gevent':'v'+geventVersion} + return {'Gevent':'v' + geventVersion} else: return {'Tornado':'v'+tornadoVersion} - - -# Start Instance of Server -Server=server() diff --git a/cps/templates/admin.html b/cps/templates/admin.html index e43e3f6b..4063f23b 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -18,7 +18,7 @@ {% for user in content %} {% if not user.role_anonymous() or config.config_anonbrowse %} - {{user.nickname}} + {{user.nickname}} {{user.email}} {{user.kindle_mail}} {{user.downloads.count()}} @@ -30,7 +30,7 @@ {% endif %} {% endfor %} - + @@ -53,7 +53,7 @@ {{email.mail_from}} - + @@ -96,8 +96,8 @@
{% if config.config_remote_login %}{% else %}{% endif %}
-
{{_('Basic Configuration')}}
-
{{_('UI Configuration')}}
+
{{_('Basic Configuration')}}
+
{{_('UI Configuration')}}
diff --git a/cps/templates/book_edit.html b/cps/templates/book_edit.html index c9871f1a..449a2d57 100644 --- a/cps/templates/book_edit.html +++ b/cps/templates/book_edit.html @@ -6,7 +6,7 @@
{% if book.has_cover %} - {{ book.title }} + {{ book.title }} {% else %} {{ book.title }} {% endif %} @@ -19,7 +19,7 @@

{{_('Delete formats:')}}

{% for file in book.data %}
- {{_('Delete')}} - {{file.format}} + {{_('Delete')}} - {{file.format}}
{% endfor %}
@@ -28,7 +28,7 @@ {% if source_formats|length > 0 and conversion_formats|length > 0 %}

{{_('Convert book format:')}}

-
+
@@ -53,7 +53,7 @@ {% endif %}
- +
@@ -175,7 +175,7 @@
{{_('Get metadata')}} - {{_('Back')}} + {{_('Back')}}
@@ -196,7 +196,7 @@
- {{_('Delete')}} + {{_('Delete')}}
diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 4bb96eb6..e68bcf47 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -5,7 +5,7 @@
{% if entry.has_cover %} - {{ entry.title }} + {{ entry.title }} {% else %} {{ entry.title }} {% endif %} @@ -22,7 +22,7 @@ {{_('Download')}} : {% for format in entry.data %} - + {{format.format}} ({{ format.uncompressed_size|filesizeformat }}) {% endfor %} @@ -33,7 +33,7 @@ {% endif %} @@ -42,7 +42,7 @@ {% endif %} {% if g.user.kindle_mail and g.user.is_authenticated and kindle_list %} {% if kindle_list.__len__() == 1 %} - {{kindle_list[0]['text']}} + {{kindle_list[0]['text']}} {% else %}
@@ -66,14 +66,14 @@ @@ -86,7 +86,7 @@

{{entry.title|shortentitle(40)}}

{% for author in entry.authors %} - {{author.name.replace('|',',')}} + {{author.name.replace('|',',')}} {% if not loop.last %} & {% endif %} @@ -108,7 +108,7 @@ {% endif %} {% if entry.series|length > 0 %} -

{{_('Book')}} {{entry.series_index}} {{_('of')}} {{entry.series[0].name}}

+

{{_('Book')}} {{entry.series_index}} {{_('of')}} {{entry.series[0].name}}

{% endif %} {% if entry.languages.__len__() > 0 %} @@ -137,7 +137,7 @@ {% for tag in entry.tags %} - {{tag.name}} + {{tag.name}} {%endfor%}

@@ -148,7 +148,7 @@

{{_('Publisher')}}: - {{entry.publishers[0].name}} + {{entry.publishers[0].name}}

@@ -191,7 +191,7 @@

-

+