From 63634961d42af9219c4bee32d15062e5440392ee Mon Sep 17 00:00:00 2001 From: Daniel Pavel Date: Sun, 14 Jul 2019 20:28:32 +0300 Subject: [PATCH] cleaner worker api the worker thread now stops on its own --- cps/__init__.py | 10 ++-------- cps/editbooks.py | 10 +++++----- cps/helper.py | 37 +++++++++++++++++++------------------ cps/server.py | 3 +-- cps/web.py | 8 ++++---- cps/worker.py | 45 +++++++++++++++++++++++++++++++++++---------- 6 files changed, 66 insertions(+), 47 deletions(-) diff --git a/cps/__init__.py b/cps/__init__.py index 7dcea6ed..26750d1a 100755 --- a/cps/__init__.py +++ b/cps/__init__.py @@ -34,8 +34,9 @@ from flask_login import LoginManager from flask_babel import Babel from flask_principal import Principal -from . import logger, cache_buster, cli, config_sql, ub +from . import logger, cache_buster, cli, config_sql, ub, db, services from .reverseproxy import ReverseProxied +from .server import WebServer mimetypes.init() @@ -66,14 +67,8 @@ lm.anonymous_user = ub.Anonymous ub.init_db(cli.settingspath) # pylint: disable=no-member config = config_sql.load_configuration(ub.session) -from . import db, services searched_ids = {} - -from .worker import WorkerThread -global_WorkerThread = WorkerThread() - -from .server import WebServer web_server = WebServer() babel = Babel() @@ -109,7 +104,6 @@ def create_app(): if services.goodreads: services.goodreads.connect(config.config_goodreads_api_key, config.config_goodreads_api_secret, config.config_use_goodreads) - global_WorkerThread.start() return app @babel.localeselector diff --git a/cps/editbooks.py b/cps/editbooks.py index 7f850254..8851a0d8 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -30,12 +30,12 @@ from uuid import uuid4 from flask import Blueprint, request, flash, redirect, url_for, abort, Markup, Response from flask_babel import gettext as _ -from flask_login import current_user +from flask_login import current_user, login_required from . import constants, logger, isoLanguages, gdriveutils, uploader, helper -from . import config, get_locale, db, ub, global_WorkerThread +from . import config, get_locale, db, ub, worker from .helper import order_authors, common_filters -from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required, login_required +from .web import login_required_if_no_ano, render_title_template, edit_required, upload_required editbook = Blueprint('editbook', __name__) @@ -358,7 +358,7 @@ def upload_single_file(request, book, book_id): # Queue uploader info uploadText=_(u"File format %(ext)s added to %(book)s", ext=file_ext.upper(), book=book.title) - global_WorkerThread.add_upload(current_user.nickname, + worker.add_upload(current_user.nickname, "" + uploadText + "") @@ -667,7 +667,7 @@ def upload(): if error: flash(error, category="error") uploadText=_(u"File %(file)s uploaded", file=book.title) - global_WorkerThread.add_upload(current_user.nickname, + worker.add_upload(current_user.nickname, "" + uploadText + "") # create data for displaying display Full language name instead of iso639.part3language diff --git a/cps/helper.py b/cps/helper.py index 1ceeb0b8..d7308d04 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -60,7 +60,7 @@ try: except ImportError: use_PIL = False -from . import logger, config, global_WorkerThread, get_locale, db, ub, isoLanguages +from . import logger, config, get_locale, db, ub, isoLanguages, worker from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR from .pagination import Pagination @@ -112,7 +112,7 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, text = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title)) settings['old_book_format'] = old_book_format settings['new_book_format'] = new_book_format - global_WorkerThread.add_convert(file_path, book.id, user_id, text, settings, kindle_mail) + worker.add_convert(file_path, book.id, user_id, text, settings, kindle_mail) return None else: error_message = _(u"%(format)s not found: %(fn)s", @@ -121,9 +121,9 @@ def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, def send_test_mail(kindle_mail, user_name): - global_WorkerThread.add_email(_(u'Calibre-Web test e-mail'),None, None, config.get_mail_settings(), - kindle_mail, user_name, _(u"Test e-mail"), - _(u'This e-mail has been sent via Calibre-Web.')) + worker.add_email(_(u'Calibre-Web test e-mail'), None, None, + config.get_mail_settings(), kindle_mail, user_name, + _(u"Test e-mail"), _(u'This e-mail has been sent via Calibre-Web.')) return @@ -138,8 +138,9 @@ def send_registration_mail(e_mail, user_name, default_password, resend=False): text += "Don't forget to change your password after first login.\r\n" text += "Sincerely\r\n\r\n" text += "Your Calibre-Web team" - global_WorkerThread.add_email(_(u'Get Started with Calibre-Web'),None, None, config.get_mail_settings(), - e_mail, None, _(u"Registration e-mail for user: %(name)s", name=user_name), text) + worker.add_email(_(u'Get Started with Calibre-Web'), None, None, + config.get_mail_settings(), e_mail, None, + _(u"Registration e-mail for user: %(name)s", name=user_name), text) return @@ -207,15 +208,15 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id): if convert: # returns None if success, otherwise errormessage return convert_book_format(book_id, calibrepath, u'epub', book_format.lower(), user_id, kindle_mail) - else: - for entry in iter(book.data): - if entry.format.upper() == book_format.upper(): - result = entry.name + '.' + book_format.lower() - global_WorkerThread.add_email(_(u"Send to Kindle"), book.path, result, config.get_mail_settings(), - kindle_mail, user_id, _(u"E-mail: %(book)s", book=book.title), - _(u'This e-mail has been sent via Calibre-Web.')) - return - return _(u"The requested file could not be read. Maybe wrong permissions?") + + for entry in iter(book.data): + if entry.format.upper() == book_format.upper(): + converted_file_name = entry.name + '.' + book_format.lower() + worker.add_email(_(u"Send to Kindle"), book.path, converted_file_name, + config.get_mail_settings(), kindle_mail, user_id, + _(u"E-mail: %(book)s", book=book.title), _(u'This e-mail has been sent via Calibre-Web.')) + return + return _(u"The requested file could not be read. Maybe wrong permissions?") def get_valid_filename(value, replace_whitespace=True): @@ -232,7 +233,7 @@ def get_valid_filename(value, replace_whitespace=True): value = value.replace(u'§', u'SS') value = value.replace(u'ß', u'ss') value = unicodedata.normalize('NFKD', value) - re_slugify = re.compile('[\W\s-]', re.UNICODE) + re_slugify = re.compile(r'[\W\s-]', re.UNICODE) if isinstance(value, str): # Python3 str, Python2 unicode value = re_slugify.sub('', value).strip() else: @@ -254,7 +255,7 @@ def get_valid_filename(value, replace_whitespace=True): def get_sorted_author(value): try: if ',' not in value: - regexes = ["^(JR|SR)\.?$", "^I{1,3}\.?$", "^IV\.?$"] + regexes = [r"^(JR|SR)\.?$", r"^I{1,3}\.?$", r"^IV\.?$"] combined = "(" + ")|(".join(regexes) + ")" value = value.split(" ") if re.match(combined, value[-1].upper()): diff --git a/cps/server.py b/cps/server.py index 1d564824..ada9d156 100644 --- a/cps/server.py +++ b/cps/server.py @@ -38,7 +38,7 @@ except ImportError: VERSION = {'Tornado': 'v' + _version} _GEVENT = False -from . import logger, global_WorkerThread +from . import logger log = logger.create() @@ -179,7 +179,6 @@ class WebServer: return False finally: self.wsgiserver = None - global_WorkerThread.stop() if not self.restart: log.info("Performing shutdown of Calibre-Web") diff --git a/cps/web.py b/cps/web.py index d35c2d28..512a0930 100644 --- a/cps/web.py +++ b/cps/web.py @@ -41,8 +41,8 @@ from werkzeug.exceptions import default_exceptions from werkzeug.datastructures import Headers from werkzeug.security import generate_password_hash, check_password_hash -from . import constants, logger, isoLanguages, services -from . import global_WorkerThread, searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app +from . import constants, logger, isoLanguages, services, worker +from . import searched_ids, lm, babel, db, ub, config, negociate_locale, get_locale, app from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download from .helper import common_filters, get_search_results, fill_indexpage, speaking_language, check_valid_domain, \ order_authors, get_typeahead, render_task_status, json_serial, get_cc_columns, \ @@ -245,7 +245,7 @@ def before_request(): @web.route("/ajax/emailstat") @login_required def get_email_status_json(): - tasks = global_WorkerThread.get_taskstatus() + tasks = worker.get_taskstatus() answer = render_task_status(tasks) js = json.dumps(answer, default=json_serial) response = make_response(js) @@ -760,7 +760,7 @@ def category_list(): @login_required def get_tasks_status(): # if current user admin, show all email, otherwise only own emails - tasks = global_WorkerThread.get_taskstatus() + tasks = worker.get_taskstatus() answer = render_task_status(tasks) return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"), page="tasks") diff --git a/cps/worker.py b/cps/worker.py index 30ee0907..37212873 100644 --- a/cps/worker.py +++ b/cps/worker.py @@ -25,7 +25,7 @@ import smtplib import socket import time import threading -from datetime import datetime, timedelta +from datetime import datetime try: from StringIO import StringIO @@ -66,6 +66,13 @@ RET_FAIL = 0 RET_SUCCESS = 1 +def _get_main_thread(): + for t in threading.enumerate(): + if t.__class__.__name__ == '_MainThread': + return t + raise Exception("main thread not found?!") + + # For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append # it in MIME Base64 encoded to def get_attachment(bookpath, filename): @@ -173,19 +180,19 @@ class email_SSL(emailbase, smtplib.SMTP_SSL): class WorkerThread(threading.Thread): def __init__(self): - self._stopevent = threading.Event() threading.Thread.__init__(self) self.status = 0 self.current = 0 self.last = 0 self.queue = list() self.UIqueue = list() - self.asyncSMTP=None + self.asyncSMTP = None self.id = 0 # Main thread loop starting the different tasks def run(self): - while not self._stopevent.isSet(): + main_thread = _get_main_thread() + while main_thread.is_alive(): doLock = threading.Lock() doLock.acquire() if self.current != self.last: @@ -200,10 +207,8 @@ class WorkerThread(threading.Thread): self.current += 1 else: doLock.release() - time.sleep(1) - - def stop(self): - self._stopevent.set() + if main_thread.is_alive(): + time.sleep(1) def get_send_status(self): if self.asyncSMTP: @@ -317,7 +322,7 @@ class WorkerThread(threading.Thread): nextline = p.communicate()[0] # Format of error message (kindlegen translates its output texts): # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting. - conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) + conv_error = re.search(r".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) # If error occoures, store error message for logfile if conv_error: error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s", @@ -332,7 +337,7 @@ class WorkerThread(threading.Thread): nextline = nextline.decode('utf-8') log.debug(nextline.strip('\r\n')) # parse progress string from calibre-converter - progress = re.search("(\d+)%\s.*", nextline) + progress = re.search(r"(\d+)%\s.*", nextline) if progress: self.UIqueue[self.current]['progress'] = progress.group(1) + ' %' @@ -511,3 +516,23 @@ class WorkerThread(threading.Thread): self.UIqueue[self.current]['stat'] = STAT_FINISH_SUCCESS self.UIqueue[self.current]['progress'] = "100 %" self.UIqueue[self.current]['formRuntime'] = datetime.now() - self.queue[self.current]['starttime'] + + +_worker = WorkerThread() +_worker.start() + + +def get_taskstatus(): + return _worker.get_taskstatus() + + +def add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text): + return _worker.add_email(subject, filepath, attachment, settings, recipient, user_name, taskMessage, text) + + +def add_upload(user_name, taskMessage): + return _worker.add_upload(user_name, taskMessage) + + +def add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail=None): + return _worker.add_convert(file_path, bookid, user_name, taskMessage, settings, kindle_mail)