Added feature to send emails in a background-task

cps/asyncmail.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import smtplib
+import threading
+from datetime import datetime
+import logging
+import time
+import socket
+import sys
+from email.generator import Generator
+import web
+from flask_babel import gettext as _
+# from babel.dates import format_datetime
+import re
+
+try:
+    from StringIO import StringIO
+except ImportError as e:
+    from io import StringIO
+
+chunksize = 8192
+
+STAT_WAITING = 0
+STAT_FAIL = 1
+STAT_STARTED = 2
+STAT_FINISH_SUCCESS = 3
+
+
+class email(smtplib.SMTP):
+
+    transferSize = 0
+    progress = 0
+
+    def __init__(self, *args, **kwargs):
+        smtplib.SMTP.__init__(self, *args, **kwargs)
+
+    def data(self, msg):
+        self.transferSize = len(msg)
+        (code, resp) = smtplib.SMTP.data(self, msg)
+        self.progress = 0
+        return (code, resp)
+
+    def send(self, str):
+        """Send `str' to the server."""
+        if self.debuglevel > 0:
+            print>> sys.stderr, 'send:', repr(str)
+        if hasattr(self, 'sock') and self.sock:
+            try:
+                if self.transferSize:
+                    lock=threading.Lock()
+                    lock.acquire()
+                    self.transferSize = len(str)
+                    lock.release()
+                    for i in range(0, self.transferSize, chunksize):
+                        self.sock.send(str[i:i+chunksize])
+                        lock.acquire()
+                        self.progress = i
+                        lock.release()
+                else:
+                    self.sock.sendall(str)
+            except socket.error:
+                self.close()
+                raise smtplib.SMTPServerDisconnected('Server not connected')
+        else:
+            raise smtplib.SMTPServerDisconnected('please run connect() first')
+
+    def getTransferStatus(self):
+        if self.transferSize:
+            lock2 = threading.Lock()
+            lock2.acquire()
+            value = round(float(self.progress) / float(self.transferSize),2)*100
+            lock2.release()
+            return str(value) + ' %'
+        else:
+            return "100 %"
+
+class email_SSL(email):
+
+    def __init__(self, *args, **kwargs):
+        smtplib.SMTP_SSL.__init__(self, *args, **kwargs)
+
+
+class EMailThread(threading.Thread):
+
+    def __init__(self):
+        threading.Thread.__init__(self)
+        self.status = 0
+        self.current = 0
+        self.last = 0
+        self.queue=list()
+        self.UIqueue = list()
+        self.asyncSMTP=None
+
+    def run(self):
+        while 1:
+            doLock = threading.Lock()
+            doLock.acquire()
+            if self.current != self.last:
+                doLock.release()
+                self.send_raw_email()
+                self.current += 1
+
+            time.sleep(1)
+
+    def get_send_status(self):
+        if self.asyncSMTP:
+            return self.asyncSMTP.getTransferStatus()
+        else:
+            return "0 %"
+
+    def delete_completed_tasks(self):
+        # muss gelockt werden
+        for index, task in reversed(list(enumerate(self.UIqueue))):
+            if task['progress'] == "100 %":
+                # delete tasks
+                self.queue.pop(index)
+                self.UIqueue.pop(index)
+                # if we are deleting entries before the current index, adjust the index
+                # if self.current >= index:
+                self.current -= 1
+        self.last = len(self.queue)
+
+    def get_taskstatus(self):
+        if self.current  < len(self.queue):
+            if self.queue[self.current]['status'] == STAT_STARTED:
+                self.UIqueue[self.current]['progress'] = self.get_send_status()
+                self.UIqueue[self.current]['runtime'] = self._formatRuntime(
+                                                        datetime.now() - self.queue[self.current]['starttime'])
+
+        return self.UIqueue
+
+    def add_email(self, data, settings, recipient, user_name):
+        # if more than 50 entries in the list, clean the list
+        addLock = threading.Lock()
+        addLock.acquire()
+        if self.last >= 3:
+            self.delete_completed_tasks()
+        # progress, runtime, and status = 0
+        self.queue.append({'data':data, 'settings':settings, 'recipent':recipient, 'starttime': 0,
+                           'status': STAT_WAITING})
+        self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': 'E-Mail',
+                             'runtime': '0 s', 'status': _('Waiting') })
+        # access issue
+        self.last=len(self.queue)
+        addLock.release()
+
+    def send_raw_email(self):
+        obj=self.queue[self.current]
+        # settings = ub.get_mail_settings()
+
+        obj['data']['From'] = obj['settings']["mail_from"]
+        obj['data']['To'] = obj['recipent']
+
+        use_ssl = int(obj['settings'].get('mail_use_ssl', 0))
+
+        # convert MIME message to string
+        fp = StringIO()
+        gen = Generator(fp, mangle_from_=False)
+        gen.flatten(obj['data'])
+        obj['data'] = fp.getvalue()
+
+        # send email
+        try:
+            timeout = 600  # set timeout to 5mins
+
+            org_stderr = sys.stderr
+            #org_stderr2 = smtplib.stderr
+            sys.stderr = StderrLogger()
+            #smtplib.stderr = StderrLogger()
+
+            self.queue[self.current]['status'] = STAT_STARTED
+            self.UIqueue[self.current]['status'] = _('Started')
+            self.queue[self.current]['starttime'] = datetime.now()
+            self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
+
+
+            if use_ssl == 2:
+                self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
+            else:
+                self.asyncSMTP = email(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
+
+            # link to logginglevel
+            if web.ub.config.config_log_level != logging.DEBUG:
+                self.asyncSMTP.set_debuglevel(0)
+            else:
+                self.asyncSMTP.set_debuglevel(1)
+            if use_ssl == 1:
+                self.asyncSMTP.starttls()
+            if obj['settings']["mail_password"]:
+                self.asyncSMTP.login(str(obj['settings']["mail_login"]), str(obj['settings']["mail_password"]))
+            self.asyncSMTP.sendmail(obj['settings']["mail_from"], obj['recipent'], obj['data'])
+            self.asyncSMTP.quit()
+            self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
+            self.UIqueue[self.current]['status'] = _('Finished')
+            self.UIqueue[self.current]['progress'] = "100 %"
+            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
+                                                        datetime.now() - self.queue[self.current]['starttime'])
+
+            sys.stderr = org_stderr
+            #smtplib.stderr = org_stderr2
+
+        except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as e:
+            self.queue[self.current]['status'] = STAT_FAIL
+            self.UIqueue[self.current]['status'] = _('Failed')
+            self.UIqueue[self.current]['progress'] = "100 %"
+            self.UIqueue[self.current]['runtime'] = self._formatRuntime(
+                                                    datetime.now() - self.queue[self.current]['starttime'])
+            web.app.logger.error(e)
+        return None
+
+    def _formatRuntime(self, runtime):
+        val = re.split('\:|\.', str(runtime))[0:3]
+        erg = list()
+        for v in val:
+            if int(v) > 0:
+                erg.append(v)
+        retVal = (':'.join(erg)).lstrip('0') + ' s'
+        if retVal == ' s':
+            retVal = '0 s'
+        return retVal
+
+class StderrLogger(object):
+
+    buffer = ''
+
+    def __init__(self):
+        self.logger = web.app.logger
+
+    def write(self, message):
+        if message == '\n':
+            self.logger.debug(self.buffer)
+            print self.buffer
+            self.buffer = ''
+        else:
+            self.buffer += message

cps/helper.py

@@ -5,9 +5,7 @@ import db
 import ub
 from flask import current_app as app
 import logging
-import smtplib
 from tempfile import gettempdir
-import socket
 import sys
 import os
 import traceback
@@ -15,6 +13,7 @@ import re
 import unicodedata
 from io import BytesIO
 import converter
+import asyncmail
 
 try:
     from StringIO import StringIO
@@ -28,11 +27,9 @@ except ImportError as e:
     from email.mime.text import MIMEText
 
 from email import encoders
-from email.generator import Generator
 from email.utils import formatdate
 from email.utils import make_msgid
 from flask_babel import gettext as _
-import subprocess
 import threading
 import shutil
 import requests
@@ -52,11 +49,22 @@ except ImportError:
 
 # Global variables
 updater_thread = None
+global_eMailThread = asyncmail.EMailThread()
+global_eMailThread.start()
 
 RET_SUCCESS = 1
 RET_FAIL = 0
 
 
+def update_download(book_id, user_id):
+    check = ub.session.query(ub.Downloads).filter(ub.Downloads.user_id == user_id).filter(ub.Downloads.book_id ==
+                                                                                          book_id).first()
+
+    if not check:
+        new_download = ub.Downloads(user_id=user_id, book_id=book_id)
+        ub.session.add(new_download)
+        ub.session.commit()
+
 def make_mobi(book_id, calibrepath):
     book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
     data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first()
@@ -73,74 +81,16 @@ def make_mobi(book_id, calibrepath):
         return error_message, RET_FAIL
 
 
-class StderrLogger(object):
+def send_test_mail(kindle_mail, user_name):
-
-    buffer = ''
-
-    def __init__(self):
-        self.logger = logging.getLogger('cps.web')
-
-    def write(self, message):
-        if message == '\n':
-            self.logger.debug(self.buffer)
-            self.buffer = ''
-        else:
-            self.buffer += message
-
-
-def send_raw_email(kindle_mail, msg):
-    settings = ub.get_mail_settings()
-
-    msg['From'] = settings["mail_from"]
-    msg['To'] = kindle_mail
-
-    use_ssl = int(settings.get('mail_use_ssl', 0))
-
-    # convert MIME message to string
-    fp = StringIO()
-    gen = Generator(fp, mangle_from_=False)
-    gen.flatten(msg)
-    msg = fp.getvalue()
-
-    # send email
-    try:
-        timeout = 600     # set timeout to 5mins
-
-        org_stderr = sys.stderr
-        sys.stderr = StderrLogger()
-
-        if use_ssl == 2:
-            mailserver = smtplib.SMTP_SSL(settings["mail_server"], settings["mail_port"], timeout)
-        else:
-            mailserver = smtplib.SMTP(settings["mail_server"], settings["mail_port"], timeout)
-        mailserver.set_debuglevel(1)
-
-        if use_ssl == 1:
-            mailserver.starttls()
-
-        if settings["mail_password"]:
-            mailserver.login(str(settings["mail_login"]), str(settings["mail_password"]))
-        mailserver.sendmail(settings["mail_from"], kindle_mail, msg)
-        mailserver.quit()
-
-        smtplib.stderr = org_stderr
-
-    except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as ex:
-        app.logger.error(traceback.print_exc())
-        return _("Failed to send mail: %s" % str(ex))
-
-    return None
-
-
-def send_test_mail(kindle_mail):
     msg = MIMEMultipart()
     msg['Subject'] = _(u'Calibre-web test email')
     text = _(u'This email has been sent via calibre web.')
     msg.attach(MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8'))
-    return send_raw_email(kindle_mail, msg)
+    global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_name)
+    return # send_raw_email(kindle_mail, msg)
 
 
-def send_mail(book_id, kindle_mail, calibrepath):
+def send_mail(book_id, kindle_mail, calibrepath, user_id):
     """Send email with attachments"""
     # create MIME message
     msg = MIMEMultipart()
@@ -179,8 +129,8 @@ def send_mail(book_id, kindle_mail, calibrepath):
         msg.attach(get_attachment(formats['pdf']))
     else:
         return _("Could not find any formats suitable for sending by email")
-
+    global_eMailThread.add_email(msg,ub.get_mail_settings(),kindle_mail, user_id)
-    return send_raw_email(kindle_mail, msg)
+    return None # send_raw_email(kindle_mail, msg)
 
 
 def get_attachment(file_path):

cps/templates/layout.html

@@ -67,6 +67,9 @@
                   </li>
                 {% endif %}
               {% endif %}
+              {% if g.user.role_admin() %}
+                <li><a id="top_tasks" href="{{url_for('get_tasks_status')}}"><span class="glyphicon glyphicon-tasks"></span><span class="hidden-sm"> {{_('Tasks')}}</span></a></li>
+              {% endif %}
               {% if g.user.role_admin() %}
                 <li><a id="top_admin" href="{{url_for('admin')}}"><span class="glyphicon glyphicon-dashboard"></span><span class="hidden-sm"> {{_('Admin')}}</span></a></li>
               {% endif %}
@@ -222,6 +225,7 @@
     </div>
     {% block modal %}{% endblock %}
 
+
     <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
     <!--script src="https://code.jquery.com/jquery.js"></script-->
     <script src="{{ url_for('static', filename='js/libs/jquery.min.js') }}"></script>

cps/templates/tasks.html

@@ -0,0 +1,48 @@
+{% extends "layout.html" %}
+{% block header %}
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table.css">
+{% endblock %}
+{% block body %}
+<div class="discover">
+    <h2>{{_('Tasks list')}}</h2>
+    <table class="table table-no-bordered" id="table" data-url="{{'/ajax/emailstat'}}">
+      <thead>
+        <tr>
+            {% if g.user.role_admin() %}
+            <th data-field="user">{{_('User')}}</th>
+            {% endif %}
+            <th data-field="type">{{_('Task')}}</th>
+            <th data-field="status">{{_('Status')}}</th>
+            <th data-field="progress">{{_('Progress')}}</th>
+            <th data-field="runtime">{{_('Runtime')}}</th>
+            <th data-field="starttime">{{_('Starttime')}}</th>
+        </tr>
+      </thead>
+    </table>
+  <!--div class="btn btn-default" id="tasks_delete">{{_('Delete finished tasks')}}</div>
+  <div class="btn btn-default" id="tasks_hide">{{_('Hide all tasks')}}</div-->
+</div>
+{% endblock %}
+{% block js %}
+<script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-table/1.8.1/bootstrap-table.js"></script>
+        <script>
+            $('#table').bootstrapTable({
+               formatNoMatches: function () {
+                return '';
+                },
+                striped: true
+            });
+            setInterval(function() {
+                $.ajax({
+                    method:"get",
+                    url: "{{'/ajax/emailstat'}}",
+                    async: true,
+                    timeout: 900,
+                    success:function(data){
+                    $('#table').bootstrapTable("load", data);
+                }
+                });
+            }, 1000);
+        </script>
+
+{% endblock %}

cps/ub.py

@@ -10,7 +10,6 @@ import sys
 import os
 import logging
 from werkzeug.security import generate_password_hash
-from flask_babel import gettext as _
 import json
 import datetime
 from binascii import hexlify

cps/web.py

@@ -912,13 +912,41 @@ def get_metadata_calibre_companion(uuid):
     else:
         return ""
 
+@app.route("/ajax/emailstat")
+@login_required
+def get_email_status_json():
+    answer=list()
+    tasks=helper.global_eMailThread.get_taskstatus()
+    if not current_user.role_admin():
+        for task in tasks:
+            if task['user'] == current_user.nickname:
+                if task['formStarttime']:
+                    task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale())
+                    task['formStarttime'] = ""
+                else:
+                    if 'starttime' not in task:
+                        task['starttime'] = ""
+                answer.append(task)
+    else:
+        for task in tasks:
+            if task['formStarttime']:
+                task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale())
+                task['formStarttime'] = ""
+            else:
+                if 'starttime' not in  task:
+                    task['starttime'] = ""
+        answer = tasks
+    js=json.dumps(answer)
+    response = make_response(js)
+    response.headers["Content-Type"] = "application/json; charset=utf-8"
+    return response
+
 
 @app.route("/get_authors_json", methods=['GET', 'POST'])
 @login_required_if_no_ano
 def get_authors_json():
     if request.method == "GET":
         query = request.args.get('q')
-        # entries = db.session.execute("select name from authors where name like '%" + query + "%'")
         entries = db.session.query(db.Authors).filter(db.Authors.name.ilike("%" + query + "%")).all()
         json_dumps = json.dumps([dict(name=r.name) for r in entries])
         return json_dumps
@@ -929,10 +957,7 @@ def get_authors_json():
 def get_tags_json():
     if request.method == "GET":
         query = request.args.get('q')
-        # entries = db.session.execute("select name from tags where name like '%" + query + "%'")
         entries = db.session.query(db.Tags).filter(db.Tags.name.ilike("%" + query + "%")).all()
-        # for x in entries:
-        #    alfa = dict(name=x.name)
         json_dumps = json.dumps([dict(name=r.name) for r in entries])
         return json_dumps
 
@@ -1421,6 +1446,35 @@ def bookmark(book_id, book_format):
     ub.session.commit()
     return "", 201
 
+@app.route("/tasks")
+@login_required
+def get_tasks_status():
+    # if current user admin, show all email, otherwise only own emails
+    answer=list()
+    tasks=helper.global_eMailThread.get_taskstatus()
+    if not current_user.role_admin():
+        for task in tasks:
+            if task['user'] == current_user.nickname:
+                if task['formStarttime']:
+                    task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale())
+                    task['formStarttime'] = ""
+                else:
+                    if 'starttime' not in task:
+                        task['starttime'] = ""
+                answer.append(task)
+    else:
+        for task in tasks:
+            if task['formStarttime']:
+                task['starttime'] = format_datetime(task['formStarttime'], format='short', locale=get_locale())
+                task['formStarttime'] = ""
+            else:
+                if 'starttime' not in  task:
+                    task['starttime'] = ""
+        answer = tasks
+
+    # foreach row format row
+    return render_title_template('tasks.html', entries=answer, title=_(u"Tasks"))
+
 
 @app.route("/admin")
 @login_required
@@ -2147,9 +2201,9 @@ def send_to_kindle(book_id):
     if settings.get("mail_server", "mail.example.com") == "mail.example.com":
         flash(_(u"Please configure the SMTP mail settings first..."), category="error")
     elif current_user.kindle_mail:
-        result = helper.send_mail(book_id, current_user.kindle_mail, config.config_calibre_dir)
+        result = helper.send_mail(book_id, current_user.kindle_mail, config.config_calibre_dir, current_user.nickname)
         if result is None:
-            flash(_(u"Book successfully send to %(kindlemail)s", kindlemail=current_user.kindle_mail),
+            flash(_(u"Book successfully queued for sending to %(kindlemail)s", kindlemail=current_user.kindle_mail),
                   category="success")
             ub.update_download(book_id, int(current_user.id))
         else:
@@ -2851,7 +2905,7 @@ def edit_mailsettings():
             flash(e, category="error")
         if "test" in to_save and to_save["test"]:
             if current_user.kindle_mail:
-                result = helper.send_test_mail(current_user.kindle_mail)
+                result = helper.send_test_mail(current_user.kindle_mail, current_user.nickname)
                 if result is None:
                     flash(_(u"Test E-Mail successfully send to %(kindlemail)s", kindlemail=current_user.kindle_mail),
                           category="success")