Bugfix Get gdrive instances (#554, #525)

Metadata.db download works again
removed DEVELOPMENT constant
removed db logging in debug mode (too, noisy, to less information)
code refactoring url_for_other_page
feed languge set to en-EN
update status shos local time instead of UTC
Error handling (back to index page) in case of gdrive authenticate aborted
Mistyping page register fixed
Mistyping bokk fixed
Added uploaded books to tasklist (#442)
Error handling for failed file update added
Code refactoring worker thread
Tasks now never show any decimal values
Converter function unified
removed shell from subprocess call
preparation for limiting domain for registering emails
Book series can now increased in 0.1 steps (#562)
Accordion panels in config are now usable on touch devices like iPad (#545)
Gdrive authenticate button only visible after logged in (#525)
Fixed misstyping in german translation
pull/591/head
OzzieIsaacs 6 years ago
parent f8132f4d02
commit cdb1b52652

@ -3,8 +3,9 @@ try:
from pydrive.drive import GoogleDrive
from pydrive.auth import RefreshError
from apiclient import errors
gdrive_support = True
except ImportError:
pass
gdrive_support = False
import os
from ub import config
@ -259,6 +260,13 @@ def copyDriveFileRemote(drive, origin_file_id, copy_title):
print ('An error occurred: %s' % error)
return None
# Download metadata.db from gdrive
def downloadFile(path, filename, output):
f = getFileFromEbooksFolder(path, filename)
f.GetContentFile(output)
def moveGdriveFolderRemote(origin_file, target_folder):
drive = getDrive(Gdrive.Instance().drive)
previous_parents = ",".join([parent["id"] for parent in origin_file.get('parents')])
@ -339,7 +347,7 @@ def uploadFileToEbooksFolder(destFile, f):
def watchChange(drive, channel_id, channel_type, channel_address,
channel_token=None, expiration=None):
drive = getDrive(drive)
# drive = getDrive(drive)
# Watch for all changes to a user's Drive.
# Args:
# service: Drive API service instance.
@ -382,7 +390,7 @@ def watchFile(drive, file_id, channel_id, channel_type, channel_address,
Raises:
apiclient.errors.HttpError: if http request to create channel fails.
"""
drive = getDrive(drive)
# drive = getDrive(drive)
body = {
'id': channel_id,
@ -405,7 +413,7 @@ def stopChannel(drive, channel_id, resource_id):
Raises:
apiclient.errors.HttpError: if http request to create channel fails.
"""
drive = getDrive(drive)
# drive = getDrive(drive)
# service=drive.auth.service
body = {
'id': channel_id,
@ -415,7 +423,7 @@ def stopChannel(drive, channel_id, resource_id):
def getChangeById (drive, change_id):
drive = getDrive(drive)
# drive = getDrive(drive)
# Print a single Change resource information.
#
# Args:

@ -53,6 +53,9 @@
</table>
<div class="btn btn-default" id="admin_edit_email"><a href="{{url_for('edit_mailsettings')}}">{{_('Change SMTP settings')}}</a></div>
{% if g.allow_registration %}
<div class="btn btn-default" id="admin_register_domain"><a href="{{url_for('edit_register_domains')}}">{{_('Edit allowed domains')}}</a></div>
{% endif %}
<h2>{{_('Configuration')}}</h2>
<table class="table table-striped" id="table_configuration">
@ -79,16 +82,14 @@
<div class="btn btn-default"><a href="{{url_for('configuration')}}">{{_('Basic Configuration')}}</a></div>
<div class="btn btn-default"><a href="{{url_for('view_configuration')}}">{{_('UI Configuration')}}</a></div>
<h2>{{_('Administration')}}</h2>
{% if not development %}
<div>{{_('Current commit timestamp')}}: <span>{{commit}} </span></div>
<div class="hidden" id="update_info">{{_('Newest commit timestamp')}}: <span></span></div>
<p></p>
<div class="btn btn-default" id="restart_database">{{_('Reconnect to Calibre DB')}}</div>
<div class="btn btn-default" data-toggle="modal" data-target="#RestartDialog">{{_('Restart Calibre-web')}}</div>
<div class="btn btn-default" data-toggle="modal" data-target="#ShutdownDialog">{{_('Stop Calibre-web')}}</div>
<div class="btn btn-default" id="check_for_update">{{_('Check for update')}}</div>
<div class="btn btn-default hidden" id="perform_update" data-toggle="modal" data-target="#UpdateprogressDialog">{{_('Perform Update')}}</div>
{% endif %}
<div>{{_('Current commit timestamp')}}: <span>{{commit}} </span></div>
<div class="hidden" id="update_info">{{_('Newest commit timestamp')}}: <span></span></div>
<p></p>
<div class="btn btn-default" id="restart_database">{{_('Reconnect to Calibre DB')}}</div>
<div class="btn btn-default" data-toggle="modal" data-target="#RestartDialog">{{_('Restart Calibre-web')}}</div>
<div class="btn btn-default" data-toggle="modal" data-target="#ShutdownDialog">{{_('Stop Calibre-web')}}</div>
<div class="btn btn-default" id="check_for_update">{{_('Check for update')}}</div>
<div class="btn btn-default hidden" id="perform_update" data-toggle="modal" data-target="#UpdateprogressDialog">{{_('Perform Update')}}</div>
</div>
<!-- Modal -->
<div id="RestartDialog" class="modal fade" role="dialog">

@ -50,7 +50,7 @@
</div>
<div class="form-group">
<label for="series_index">{{_('Series id')}}</label>
<input type="number" step="1" min="0" class="form-control" name="series_index" id="series_index" value="{{book.series_index}}">
<input type="number" step="0.1" min="0" class="form-control" name="series_index" id="series_index" value="{{book.series_index}}">
</div>
<div class="form-group">
<label for="rating">{{_('Rating')}}</label>

@ -7,10 +7,10 @@
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapseOne">
<a class="accordion-toggle" data-toggle="collapse" href="#collapseOne" style="text-decoration:none;">
<span class="glyphicon glyphicon-minus"></span>
{{_('Library Configuration')}}
</div>
</a>
</h4>
</div>
<div id="collapseOne" class="panel-collapse collapse in">
@ -31,11 +31,15 @@
</label>
</div>
{% else %}
{% if show_authenticate_google_drive %}
{% if show_authenticate_google_drive and g.user.is_authenticated %}
<div class="form-group required">
<a href="{{ url_for('authenticate_google_drive') }}" class="btn btn-primary">{{_('Authenticate Google Drive')}}</a>
</div>
{% else %}
{% if show_authenticate_google_drive and not g.user.is_authenticated %}
<div >{{_('Please finish Google Drive setup after login')}}</div>
{% endif %}
{% if not show_authenticate_google_drive %}
<div class="form-group required">
<label for="config_google_drive_folder">{{_('Google Drive Calibre folder')}}</label>
<select name="config_google_drive_folder" id="config_google_drive_folder" class="form-control">
@ -53,19 +57,21 @@
{% else %}
<a href="{{ url_for('watch_gdrive') }}" class="btn btn-primary">Enable watch of metadata.db</a>
{% endif %}
{% endif %}
{% endif %}
{% endif %}
</div>
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapsetwo">
<a class="accordion-toggle" data-toggle="collapse" href="#collapsetwo" style="text-decoration:none;">
<span class="glyphicon glyphicon-plus"></span>
{{_('Server Configuration')}}
</div>
</a>
</h4>
</div>
<div id="collapsetwo" class="panel-collapse collapse">
@ -88,10 +94,10 @@
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapsethree">
<a class="accordion-toggle" data-toggle="collapse" href="#collapsethree" style="text-decoration:none;">
<span class="glyphicon glyphicon-plus"></span>
{{_('Logfile Configuration')}}
</div>
</a>
</h4>
</div>
<div id="collapsethree" class="panel-collapse collapse">
@ -115,10 +121,10 @@
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapsefive">
<a class="accordion-toggle" data-toggle="collapse" href="#collapsefive">
<span class="glyphicon glyphicon-plus"></span>
{{_('Feature Configuration')}}
</div>
</a>
</h4>
</div>
<div id="collapsefive" class="panel-collapse collapse">
@ -162,10 +168,10 @@
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapseeight">
<a class="accordion-toggle" data-toggle="collapse" href="#collapseeight" style="text-decoration:none;">
<span class="glyphicon glyphicon-plus"></span>
{{_('E-Book converter')}}
</div>
</a>
</h4>
</div>
<div id="collapseeight" class="panel-collapse collapse">

@ -7,10 +7,10 @@
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapsefour">
<a class="accordion-toggle" data-toggle="collapse" href="#collapsefour" style="text-decoration:none;">
<span class="glyphicon glyphicon-plus"></span>
{{_('View Configuration')}}
</div>
</a>
</h4>
</div>
<div id="collapsefour" class="panel-collapse collapse">
@ -57,10 +57,10 @@
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapsesix">
<a class="accordion-toggle" data-toggle="collapse" href="#collapsesix" style="text-decoration:none;">
<span class="glyphicon glyphicon-plus"></span>
{{_('Default settings for new users')}}
</div>
</a>
</h4>
</div>
<div id="collapsesix" class="panel-collapse collapse">
@ -99,10 +99,10 @@
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapseseven">
<a class="accordion-toggle" data-toggle="collapse" href="#collapseseven" style="text-decoration:none;">
<span class="glyphicon glyphicon-plus"></span>
{{_('Default visibilities for new users')}}
</div>
</a>
</h4>
</div>
<div id="collapseseven" class="panel-collapse collapse">

@ -186,13 +186,13 @@
{% if pagination and (pagination.has_next or pagination.has_prev) %}
<div class="pagination">
{% if pagination.has_prev %}
<a class="previous" href="{{ url_for_other_page(pagination.page - 1)
<a class="previous" href="{{ (pagination.page - 1)|url_for_other_page
}}">&laquo; {{_('Previous')}}</a>
{% endif %}
{% for page in pagination.iter_pages() %}
{% if page %}
{% if page != pagination.page %}
<a href="{{ url_for_other_page(page) }}">{{ page }}</a>
<a href="{{ (page)|url_for_other_page }}">{{ page }}</a>
{% else %}
<strong>{{ page }}</strong>
{% endif %}
@ -201,7 +201,7 @@
{% endif %}
{% endfor %}
{% if pagination.has_next %}
<a class="next" href="{{ url_for_other_page(pagination.page + 1)
<a class="next" href="{{ (pagination.page + 1)|url_for_other_page
}}">{{_('Next')}} &raquo;</a>
{% endif %}
</div>

@ -1287,7 +1287,7 @@ msgstr "Über"
#: cps/templates/layout.html:187
msgid "Previous"
msgstr "Voerheriger"
msgstr "Vorheriger"
#: cps/templates/layout.html:214
msgid "Book Details"

@ -46,7 +46,6 @@ DEFAULT_PASS = "admin123"
DEFAULT_PORT = int(os.environ.get("CALIBRE_PORT", 8083))
DEVELOPMENT = False
class UserBase:
@ -291,10 +290,7 @@ class Settings(Base):
config_default_show = Column(SmallInteger, default=2047)
config_columns_to_ignore = Column(String)
config_use_google_drive = Column(Boolean)
# config_google_drive_client_id = Column(String)
# config_google_drive_client_secret = Column(String)
config_google_drive_folder = Column(String)
# config_google_drive_calibre_url_base = Column(String)
config_google_drive_watch_changes_response = Column(String)
config_remote_login = Column(Boolean)
config_use_goodreads = Column(Boolean)
@ -556,9 +552,6 @@ def migrate_Database():
except exc.OperationalError:
conn = engine.connect()
conn.execute("ALTER TABLE Settings ADD column `config_use_google_drive` INTEGER DEFAULT 0")
# conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_id` String DEFAULT ''")
# conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_secret` String DEFAULT ''")
# conn.execute("ALTER TABLE Settings ADD column `config_google_drive_calibre_url_base` INTEGER DEFAULT 0")
conn.execute("ALTER TABLE Settings ADD column `config_google_drive_folder` String DEFAULT ''")
conn.execute("ALTER TABLE Settings ADD column `config_google_drive_watch_changes_response` String DEFAULT ''")
try:

@ -1,11 +1,11 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
try:
from pydrive.auth import GoogleAuth
from googleapiclient.errors import HttpError
gdrive_support = True
# gdrive_support = True
except ImportError:
gdrive_support = False
# gdrive_support = False
pass
try:
from goodreads.client import GoodreadsClient
@ -47,6 +47,8 @@ from flask_principal import Principal
from flask_principal import __version__ as flask_principalVersion
from flask_babel import Babel
from flask_babel import gettext as _
import pytz
# from tzlocal import get_localzone
import requests
from werkzeug.security import generate_password_hash, check_password_hash
from werkzeug.datastructures import Headers
@ -64,11 +66,9 @@ from iso639 import __version__ as iso639Version
from uuid import uuid4
import os.path
import sys
import re
import db
from shutil import move, copyfile
# import shutil
import gdriveutils
import converter
import tempfile
@ -185,13 +185,13 @@ lm.anonymous_user = ub.Anonymous
app.secret_key = os.getenv('SECRET_KEY', 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT')
db.setup_db()
if config.config_log_level == logging.DEBUG:
'''if config.config_log_level == logging.DEBUG:
logging.getLogger("sqlalchemy.engine").addHandler(file_handler)
logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
logging.getLogger("sqlalchemy.pool").addHandler(file_handler)
logging.getLogger("sqlalchemy.pool").setLevel(config.config_log_level)
logging.getLogger("sqlalchemy.orm").addHandler(file_handler)
logging.getLogger("sqlalchemy.orm").setLevel(config.config_log_level)
logging.getLogger("sqlalchemy.orm").setLevel(config.config_log_level)'''
def is_gdrive_ready():
@ -309,16 +309,6 @@ class Pagination(object):
last = num
# pagination links in jinja
def url_for_other_page(page):
args = request.view_args.copy()
args['page'] = page
return url_for(request.endpoint, **args)
app.jinja_env.globals['url_for_other_page'] = url_for_other_page
def login_required_if_no_ano(func):
@wraps(func)
def decorated_view(*args, **kwargs):
@ -344,6 +334,15 @@ def remote_login_required(f):
# custom jinja filters
# pagination links in jinja
@app.template_filter('url_for_other_page')
def url_for_other_page(page):
args = request.view_args.copy()
args['page'] = page
return url_for(request.endpoint, **args)
# shortentitles to at longest nchar, shorten longer words if necessary
@app.template_filter('shortentitle')
def shortentitle_filter(s,nchar=20):
text = s.split()
@ -354,7 +353,7 @@ def shortentitle_filter(s,nchar=20):
res += '...'
break
# if word longer than 20 chars truncate line and append '...', otherwise add whole word to result
# string, and summarize total length to stop at 60 chars
# string, and summarize total length to stop at chars given by nchar
if len(line) > nchar:
res += line[:(nchar-3)] + '[..] '
suml += nchar+3
@ -583,6 +582,7 @@ def feed_search(term):
entriescount = len(entries) if len(entries) > 0 else 1
pagination = Pagination(1, entriescount, entriescount)
xml = render_title_template('feed.xml', searchterm=term, entries=entries, pagination=pagination)
else:
xml = render_title_template('feed.xml', searchterm="")
response = make_response(xml)
@ -596,8 +596,6 @@ def render_title_template(*args, **kwargs):
@app.before_request
def before_request():
if ub.DEVELOPMENT:
reload(ub)
g.user = current_user
g.allow_registration = config.config_public_reg
g.allow_upload = config.config_uploading
@ -620,7 +618,7 @@ def feed_index():
@app.route("/opds/osd")
@requires_basic_auth_if_no_ano
def feed_osd():
xml = render_title_template('osd.xml', lang='de-DE')
xml = render_title_template('osd.xml', lang='en-EN')
response = make_response(xml)
response.headers["Content-Type"] = "application/xml; charset=utf-8"
return response
@ -998,6 +996,7 @@ def get_matching_tags():
@login_required_if_no_ano
def get_update_status():
status = {}
tz = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone
if request.method == "GET":
# should be automatically replaced by git with current commit hash
commit_id = '$Format:%H$'
@ -1007,7 +1006,7 @@ def get_update_status():
status['status'] = True
commitdate = requests.get('https://api.github.com/repos/janeczku/calibre-web/git/commits/'+commit['object']['sha']).json()
if "committer" in commitdate:
form_date=datetime.datetime.strptime(commitdate['committer']['date'],"%Y-%m-%dT%H:%M:%SZ")
form_date=datetime.datetime.strptime(commitdate['committer']['date'],"%Y-%m-%dT%H:%M:%SZ") - datetime.timedelta(seconds=tz)
status['commit'] = format_datetime(form_date, format='short', locale=get_locale())
else:
status['commit'] = u'Unknown'
@ -1552,10 +1551,14 @@ def authenticate_google_drive():
@app.route("/gdrive/callback")
def google_drive_callback():
auth_code = request.args.get('code')
credentials = gdriveutils.Gauth.Instance().auth.flow.step2_exchange(auth_code)
with open(os.path.join(config.get_main_dir,'gdrive_credentials'), 'w') as f:
f.write(credentials.to_json())
return redirect(url_for('configuration'))
try:
credentials = gdriveutils.Gauth.Instance().auth.flow.step2_exchange(auth_code)
with open(os.path.join(config.get_main_dir,'gdrive_credentials'), 'w') as f:
f.write(credentials.to_json())
except ValueError as error:
app.logger.error(error)
finally:
return redirect(url_for('configuration'))
@app.route("/gdrive/watch/subscribe")
@ -1829,6 +1832,7 @@ def advanced_search():
q = q.all()
return render_title_template('search.html', searchterm=searchterm,
entries=q, title=_(u"search"), page="search")
# prepare data for search-form
tags = db.session.query(db.Tags).order_by(db.Tags.name).all()
series = db.session.query(db.Series).order_by(db.Series.name).all()
if current_user.filter_language() == u"all":
@ -2063,7 +2067,7 @@ def register():
flash(_(u"This username or email address is already in use."), category="error")
return render_title_template('register.html', title=_(u"register"), page="register")
return render_title_template('register.html', title=_(u"register"), page="regsiter")
return render_title_template('register.html', title=_(u"register"), page="register")
@app.route('/login', methods=['GET', 'POST'])
@ -2520,7 +2524,7 @@ def admin():
content = ub.session.query(ub.User).all()
settings = ub.session.query(ub.Settings).first()
return render_title_template("admin.html", content=content, email=settings, config=config, commit=commit,
development=ub.DEVELOPMENT, title=_(u"Admin page"), page="admin")
title=_(u"Admin page"), page="admin")
@app.route("/admin/config", methods=["GET", "POST"])
@ -2626,7 +2630,7 @@ def configuration_helper(origin):
gdriveError=None
db_change = False
success = False
if gdrive_support == False:
if gdriveutils.gdrive_support == False:
gdriveError = _('Import of optional Google Drive requirements missing')
else:
if not os.path.isfile(os.path.join(config.get_main_dir,'client_secrets.json')):
@ -2665,7 +2669,7 @@ def configuration_helper(origin):
else:
flash(_(u'client_secrets.json is not configured for web application'), category="error")
return render_title_template("config_edit.html", content=config, origin=origin,
gdrive=gdrive_support, gdriveError=gdriveError,
gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
goodreads=goodreads_support, title=_(u"Basic Configuration"),
page="config")
# always show google drive settings, but in case of error deny support
@ -2691,7 +2695,7 @@ def configuration_helper(origin):
ub.session.commit()
flash(_(u'Keyfile location is not valid, please enter correct path'), category="error")
return render_title_template("config_edit.html", content=config, origin=origin,
gdrive=gdrive_support, gdriveError=gdriveError,
gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
goodreads=goodreads_support, title=_(u"Basic Configuration"),
page="config")
if "config_certfile" in to_save:
@ -2703,7 +2707,7 @@ def configuration_helper(origin):
ub.session.commit()
flash(_(u'Certfile location is not valid, please enter correct path'), category="error")
return render_title_template("config_edit.html", content=config, origin=origin,
gdrive=gdrive_support, gdriveError=gdriveError,
gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
goodreads=goodreads_support, title=_(u"Basic Configuration"),
page="config")
content.config_uploading = 0
@ -2746,7 +2750,7 @@ def configuration_helper(origin):
ub.session.commit()
flash(_(u'Logfile location is not valid, please enter correct path'), category="error")
return render_title_template("config_edit.html", content=config, origin=origin,
gdrive=gdrive_support, gdriveError=gdriveError,
gdrive=gdriveutils.gdrive_support, gdriveError=gdriveError,
goodreads=goodreads_support, title=_(u"Basic Configuration"),
page="config")
else:
@ -2766,14 +2770,14 @@ def configuration_helper(origin):
logging.getLogger("book_formats").setLevel(config.config_log_level)
except Exception as e:
flash(e, category="error")
return render_title_template("config_edit.html", content=config, origin=origin, gdrive=gdrive_support,
return render_title_template("config_edit.html", content=config, origin=origin, gdrive=gdriveutils.gdrive_support,
gdriveError=gdriveError, goodreads=goodreads_support,
title=_(u"Basic Configuration"), page="config")
if db_change:
reload(db)
if not db.setup_db():
flash(_(u'DB location is not valid, please enter correct path'), category="error")
return render_title_template("config_edit.html", content=config, origin=origin, gdrive=gdrive_support,
return render_title_template("config_edit.html", content=config, origin=origin, gdrive=gdriveutils.gdrive_support,
gdriveError=gdriveError, goodreads=goodreads_support,
title=_(u"Basic Configuration"), page="config")
if reboot_required:
@ -2785,12 +2789,12 @@ def configuration_helper(origin):
app.logger.info('Reboot required, restarting')
if origin:
success = True
if is_gdrive_ready() and gdrive_support == True:
if is_gdrive_ready() and gdriveutils.gdrive_support == True:
gdrivefolders=gdriveutils.listRootFolders()
else:
gdrivefolders=None
return render_title_template("config_edit.html", origin=origin, success=success, content=config,
show_authenticate_google_drive=not is_gdrive_ready(), gdrive=gdrive_support,
show_authenticate_google_drive=not is_gdrive_ready(), gdrive=gdriveutils.gdrive_support,
gdriveError=gdriveError, gdrivefolders=gdrivefolders,
goodreads=goodreads_support, title=_(u"Basic Configuration"), page="config")
@ -3106,10 +3110,14 @@ def edit_book(book_id):
is_format = db.session.query(db.Data).filter(db.Data.book == book_id).filter(db.Data.format == file_ext.upper()).first()
if is_format:
# Format entry already exists, no need to update the database
app.logger.info('Bokk format already existing')
app.logger.info('Book format already existing')
else:
db_format = db.Data(book_id, file_ext.upper(), file_size, file_name)
db.session.add(db_format)
uploadText=_(u"File format %s added to %s" % (file_ext.upper(),book.title))
helper.global_WorkerThread.add_upload(current_user.nickname,
"<a href=\""+ url_for('show_book', book_id=book.id) +"\">"+ uploadText + "</a>")
to_save = request.form.to_dict()
@ -3307,9 +3315,9 @@ def edit_book(book_id):
def upload():
if not config.config_uploading:
abort(404)
# create the function for sorting...
if request.method == 'POST' and 'btn-upload' in request.files:
for requested_file in request.files.getlist("btn-upload"):
# create the function for sorting...
db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
db.session.connection().connection.connection.create_function('uuid4', 0, lambda: str(uuid4()))
if '.' in requested_file.filename:
@ -3407,7 +3415,7 @@ def upload():
db_book.data.append(db_data)
db.session.add(db_book)
db.session.flush() # flush content get db_book.id avalible
db.session.flush() # flush content get db_book.id available
# add comment
book_id = db_book.id
@ -3425,10 +3433,12 @@ def upload():
gdriveutils.updateGdriveCalibreFromLocal()
error = helper.update_dir_stucture(book.id, config.config_calibre_dir)
# ToDo: Handle error
if error:
pass
flash(error, category="error")
uploadText=_(u"File %s uploaded" % book.title)
helper.global_WorkerThread.add_upload(current_user.nickname,
"<a href=\"" + url_for('show_book', book_id=book.id) + "\">" + uploadText + "</a>")
if db_language is not None: # display Full name instead of iso639.part3
book.languages[0].language_name = _(meta.languages)
@ -3436,15 +3446,12 @@ def upload():
for author in db_book.authors:
author_names.append(author.name)
if len(request.files.getlist("btn-upload")) < 2:
# db.session.connection().connection.connection.create_function("title_sort", 1, db.title_sort)
cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
if current_user.role_edit() or current_user.role_admin():
return render_title_template('book_edit.html', book=book, authors=author_names,
cc=cc,title=_(u"edit metadata"), page="upload")
cc=cc, title=_(u"edit metadata"), page="upload")
book_in_shelfs = []
return render_title_template('detail.html', entry=book, cc=cc,
title=book.title, books_shelfs=book_in_shelfs, page="upload")
return redirect(url_for("index"))
else:
return redirect(url_for("index"))
return redirect(url_for("index"))

@ -13,7 +13,6 @@ import os
from email.generator import Generator
import web
from flask_babel import gettext as _
# from babel.dates import format_datetime
import re
import gdriveutils as gd
import subprocess
@ -42,6 +41,7 @@ STAT_FINISH_SUCCESS = 3
TASK_EMAIL = 1
TASK_CONVERT = 2
TASK_UPLOAD = 3
RET_FAIL = 0
RET_SUCCESS = 1
@ -55,7 +55,6 @@ def get_attachment(bookpath, filename):
if web.ub.config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(bookpath, filename)
if df:
datafile = os.path.join(calibrepath, bookpath, filename)
if not os.path.exists(os.path.join(calibrepath, bookpath)):
os.makedirs(os.path.join(calibrepath, bookpath))
@ -127,7 +126,7 @@ class emailbase():
if self.transferSize:
lock2 = threading.Lock()
lock2.acquire()
value = round(float(self.progress) / float(self.transferSize),2)*100
value = int((float(self.progress) / float(self.transferSize))*100)
lock2.release()
return str(value) + ' %'
else:
@ -173,6 +172,7 @@ class WorkerThread(threading.Thread):
self.send_raw_email()
if self.queue[self.current]['typ'] == TASK_CONVERT:
self.convert_mobi()
# TASK_UPLOAD is handled implicitly
self.current += 1
else:
doLock.release()
@ -205,17 +205,14 @@ class WorkerThread(threading.Thread):
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return self.UIqueue
def convert_mobi(self):
# convert book, and upload in case of google drive
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
if web.ub.config.config_ebookconverter == 2:
filename = self.convert_calibre()
else:
filename = self.convert_kindlegen()
filename=self.convert()
if web.ub.config.config_use_google_drive:
gd.updateGdriveCalibreFromLocal()
if(filename):
@ -223,19 +220,95 @@ class WorkerThread(threading.Thread):
self.queue[self.current]['settings'], self.queue[self.current]['kindle'],
self.UIqueue[self.current]['user'], _(u"E-Mail: %s" % self.queue[self.current]['title']))
def convert_kindlegen(self):
def convert(self):
error_message = None
file_path = self.queue[self.current]['file_path']
bookid = self.queue[self.current]['bookid']
# check if converter-excecutable is existing
if not os.path.exists(web.ub.config.config_converterpath):
error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=web.ub.config.config_converterpath)
web.app.logger.error("convert_kindlegen: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self._handleError(_(u"Convertertool %(converter)s not found", converter=web.ub.config.config_converterpath))
return
try:
# check which converter to use kindlegen is "1"
if web.ub.config.config_ebookconverter == 1:
command = (web.ub.config.config_converterpath + u" \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding())
else:
command = (u"\"" + web.ub.config.config_converterpath + u"\" \"" + file_path + u".epub\" \""
+ file_path + u".mobi\" " + web.ub.config.config_calibre).encode(sys.getfilesystemencoding())
if sys.version_info > (3, 0):
command = command.decode('Utf-8')
p = subprocess.Popen(command, stdout=subprocess.PIPE)
except Exception as e:
self._handleError(_(u"Ebook-converter failed, no execution permissions"))
return
if web.ub.config.config_ebookconverter == 1:
nextline = p.communicate()[0]
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE)
# If error occoures, log in every case
if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).strip())
web.app.logger.info("convert_kindlegen: " + error_message)
else:
web.app.logger.debug("convert_kindlegen: " + nextline)
else:
while p.poll() is None:
nextline = p.stdout.readline()
if sys.version_info > (3, 0):
nextline = nextline.decode('Utf-8', 'backslashreplace')
# if nextline == '' and p.poll() is not None:
# break
# while p.poll() is None:
web.app.logger.debug(nextline.strip('\r\n'))
# parse calibre-converter
progress = re.search("(\d+)%\s.*", nextline)
if progress:
self.UIqueue[self.current]['progress'] = progress.group(1) + ' %'
# if nextline != "\r\n" and web.ub.config.config_ebookconverter == 1:
#process returncode
check = p.returncode
# kindlegen returncodes
# 0 = Info(prcgen):I1036: Mobi file built successfully
# 1 = Info(prcgen):I1037: Mobi file built with WARNINGS!
# 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors!
if ( check < 2 and web.ub.config.config_ebookconverter == 1) or \
(check == 0 and web.ub.config.config_ebookconverter == 2):
cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
new_format = web.db.Data(name=cur_book.data[0].name,book_format="MOBI",
book=bookid,uncompressed_size=os.path.getsize(file_path + ".mobi"))
cur_book.data.append(new_format)
web.db.session.commit()
self.queue[self.current]['path'] = cur_book.path
self.queue[self.current]['title'] = cur_book.title
if web.ub.config.config_use_google_drive:
os.remove(file_path + u".epub")
self.queue[self.current]['status'] = STAT_FINISH_SUCCESS
self.UIqueue[self.current]['status'] = _('Finished')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return file_path + ".mobi"
else:
web.app.logger.info("ebook converter failed with error while converting book")
if not error_message: # ToDo Check
error_message = 'Ebook converter failed with unknown error'
self._handleError(error_message)
return
'''def convert_kindlegen(self):
error_message = None
file_path = self.queue[self.current]['file_path']
bookid = self.queue[self.current]['bookid']
if not os.path.exists(web.ub.config.config_converterpath):
self._handleError(_(u"kindlegen binary %(kindlepath)s not found", kindlepath=web.ub.config.config_converterpath))
return
try:
command = (web.ub.config.config_converterpath + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding())
@ -245,14 +318,7 @@ class WorkerThread(threading.Thread):
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception:
error_message = _(u"kindlegen failed, no execution permissions")
web.app.logger.error("convert_kindlegen: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
self._handleError(_(u"kindlegen failed, no execution permissions"))
return
# Poll process for new output until finished
while True:
@ -295,33 +361,21 @@ class WorkerThread(threading.Thread):
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
return file_path + ".mobi" #, RET_SUCCESS
return file_path + ".mobi"
else:
web.app.logger.info("convert_kindlegen: kindlegen failed with error while converting book")
if not error_message:
if not error_message:
error_message = 'kindlegen failed, no excecution permissions'
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return # error_message, RET_FAIL
self._handleError(error_message)
return
def convert_calibre(self):
error_message = None
file_path = self.queue[self.current]['file_path']
bookid = self.queue[self.current]['bookid']
if not os.path.exists(web.ub.config.config_converterpath):
error_message = _(u"Ebook-convert binary %(converterpath)s not found",
converterpath=web.ub.config.config_converterpath)
web.app.logger.error("convert_calibre: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
self._handleError(_(u"Ebook-convert binary %(converterpath)s not found",
converterpath=web.ub.config.config_converterpath))
return
try:
command = (u"\"" + web.ub.config.config_converterpath + u"\" \"" + file_path + u".epub\" \""
@ -330,16 +384,9 @@ class WorkerThread(threading.Thread):
p = subprocess.Popen(command.decode('Utf-8'), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
else:
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception as e:
error_message = _(u"Ebook-convert failed, no execution permissions")
web.app.logger.error("convert_calibre: " + error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return # error_message, RET_FAIL
except Exception:
self._handleError(_(u"Ebook-convert failed, no execution permissions"))
return
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
@ -354,8 +401,6 @@ class WorkerThread(threading.Thread):
web.app.logger.debug(nextline.strip('\r\n'))
else:
web.app.logger.debug(nextline.strip('\r\n').decode(sys.getfilesystemencoding()))
check = p.returncode
if check == 0:
cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first()
@ -377,13 +422,8 @@ class WorkerThread(threading.Thread):
web.app.logger.info("convert_calibre: Ebook-convert failed with error while converting book")
if not error_message:
error_message = 'Ebook-convert failed, no excecution permissions'
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
return # error_message, RET_FAIL
self._handleError(error_message)
return'''
def add_convert(self, file_path, bookid, user_name, typ, settings, kindle_mail):
addLock = threading.Lock()
@ -418,7 +458,27 @@ class WorkerThread(threading.Thread):
self.last=len(self.queue)
addLock.release()
def add_upload(self, user_name, typ):
# if more than 20 entries in the list, clean the list
addLock = threading.Lock()
addLock.acquire()
if self.last >= 20:
self.delete_completed_tasks()
# progress=100%, runtime=0, and status finished
self.queue.append({'starttime': datetime.now(), 'status': STAT_FINISH_SUCCESS, 'typ': TASK_UPLOAD})
self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': "100 %", 'type': typ,
'runtime': '0 s', 'status': _('Finished'),'id': self.id })
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
self.id += 1
self.last=len(self.queue)
addLock.release()
def send_raw_email(self):
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
obj=self.queue[self.current]
# create MIME message
msg = MIMEMultipart()
@ -432,13 +492,7 @@ class WorkerThread(threading.Thread):
if result:
msg.attach(result)
else:
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self._handleError(u"Attachment not found")
return
msg['From'] = obj['settings']["mail_from"]
@ -459,12 +513,6 @@ class WorkerThread(threading.Thread):
org_stderr = sys.stderr
sys.stderr = StderrLogger()
self.queue[self.current]['status'] = STAT_STARTED
self.UIqueue[self.current]['status'] = _('Started')
self.queue[self.current]['starttime'] = datetime.now()
self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime']
if use_ssl == 2:
self.asyncSMTP = email_SSL(obj['settings']["mail_server"], obj['settings']["mail_port"], timeout)
else:
@ -490,13 +538,8 @@ class WorkerThread(threading.Thread):
sys.stderr = org_stderr
except (socket.error, smtplib.SMTPRecipientsRefused, smtplib.SMTPException) as e:
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
web.app.logger.error(e)
# return None
self._handleError(error_message)
return None
def _formatRuntime(self, runtime):
self.UIqueue[self.current]['rt'] = runtime.total_seconds()
@ -509,6 +552,17 @@ class WorkerThread(threading.Thread):
if retVal == ' s':
retVal = '0 s'
return retVal
def _handleError(self, error_message):
web.app.logger.error(error_message)
self.queue[self.current]['status'] = STAT_FAIL
self.UIqueue[self.current]['status'] = _('Failed')
self.UIqueue[self.current]['progress'] = "100 %"
self.UIqueue[self.current]['runtime'] = self._formatRuntime(
datetime.now() - self.queue[self.current]['starttime'])
self.UIqueue[self.current]['message'] = error_message
class StderrLogger(object):

Loading…
Cancel
Save