Code cleaning Stats page

Enable calibre's ebook-convert as converter for mobi files (#411, #533)
pull/552/head
OzzieIsaacs 6 years ago
parent ae0c5d7ec2
commit 2449b4049b

@ -127,11 +127,11 @@ def get_versions():
else:
IVersion = _(u'not installed')
if use_pdf_meta:
PVersion=PyPdfVersion
PVersion='v'+PyPdfVersion
else:
PVersion=_(u'not installed')
if lxmlversion:
XVersion = '.'.join(map(str, lxmlversion))
XVersion = 'v'+'.'.join(map(str, lxmlversion))
else:
XVersion = _(u'not installed')
return {'ImageVersion': IVersion, 'PyPdfVersion': PVersion, 'LxmlVersion':XVersion}
return {'Image Magick': IVersion, 'PyPdf': PVersion, 'lxml':XVersion}

@ -0,0 +1,154 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import subprocess
import ub
import db
import re
import web
from flask_babel import gettext as _
RET_FAIL = 0
RET_SUCCESS = 1
def versionKindle():
versions = _(u'not installed')
if os.path.exists(ub.config.config_converterpath):
try:
p = subprocess.Popen(ub.config.config_converterpath, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
for lines in p.stdout.readlines():
if isinstance(lines, bytes):
lines = lines.decode('utf-8')
if re.search('Amazon kindlegen\(', lines):
versions = lines
except Exception:
versions = _(u'Excecution permissions missing')
return {'kindlegen' : versions}
def versionCalibre():
versions = _(u'not installed')
if os.path.exists(ub.config.config_converterpath):
try:
p = subprocess.Popen(ub.config.config_converterpath + ' --version', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
for lines in p.stdout.readlines():
if isinstance(lines, bytes):
lines = lines.decode('utf-8')
if re.search('.*\(calibre', lines):
versions = lines
except Exception:
versions = _(u'Excecution permissions missing')
return {'Calibre converter' : versions}
def convert_kindlegen(file_path, book):
error_message = None
# vendorpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) +
# os.sep + "../vendor" + os.sep))
#if sys.platform == "win32":
# kindlegen = (os.path.join(vendorpath, u"kindlegen.exe")).encode(sys.getfilesystemencoding())
#else:
# kindlegen = (os.path.join(vendorpath, u"kindlegen")).encode(sys.getfilesystemencoding())
if not os.path.exists(ub.config.config_converterpath):
error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=ub.config.config_converterpath)
web.app.logger.error("convert_kindlegen: " + error_message)
return error_message, RET_FAIL
try:
p = subprocess.Popen((ub.config.config_converterpath + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception as e:
error_message = _(u"kindlegen failed, no execution permissions")
web.app.logger.error("convert_kindlegen: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
if nextline != "\r\n":
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
# If error occoures, log in every case
if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
web.app.logger.info("convert_kindlegen: " + error_message)
web.app.logger.info(nextline.strip('\r\n'))
else:
web.app.logger.debug(nextline.strip('\r\n'))
check = p.returncode
if not check or check < 2:
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
return file_path + ".mobi", RET_SUCCESS
else:
web.app.logger.info("convert_kindlegen: kindlegen failed with error while converting book")
if not error_message:
error_message = 'kindlegen failed, no excecution permissions'
return error_message, RET_FAIL
def convert_calibre(file_path, book):
error_message = None
if not os.path.exists(ub.config.config_converterpath):
error_message = _(u"Ebook-convert binary %(converterpath)s not found", converterpath=ub.config.config_converterpath)
web.app.logger.error("convert_calibre: " + error_message)
return error_message, RET_FAIL
try:
command = ("\""+ub.config.config_converterpath + "\" " + ub.config.config_calibre +
" \"" + file_path + u".epub\" \"" + file_path + u".mobi\"").encode(sys.getfilesystemencoding())
p = subprocess.Popen(command,stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception as e:
error_message = _(u"Ebook-convert failed, no execution permissions")
web.app.logger.error("convert_calibre: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
web.app.logger.debug(nextline.strip('\r\n').decode(sys.getfilesystemencoding()))
check = p.returncode
if check == 0 :
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
return file_path + ".mobi", RET_SUCCESS
else:
web.app.logger.info("convert_calibre: Ebook-convert failed with error while converting book")
if not error_message:
error_message = 'Ebook-convert failed, no excecution permissions'
return error_message, RET_FAIL
def versioncheck():
if ub.config.config_ebookconverter == 1:
return versionKindle()
elif ub.config.config_ebookconverter == 2:
return versionCalibre()
else:
return {'ebook_converter':''}
def convert_mobi(file_path, book):
if ub.config.config_ebookconverter == 2:
return convert_calibre(file_path, book)
else:
return convert_kindlegen(file_path, book)

@ -14,6 +14,7 @@ import traceback
import re
import unicodedata
from io import BytesIO
import converter
try:
from StringIO import StringIO
@ -57,17 +58,6 @@ RET_FAIL = 0
def make_mobi(book_id, calibrepath):
error_message = None
vendorpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) +
os.sep + "../vendor" + os.sep))
if sys.platform == "win32":
kindlegen = (os.path.join(vendorpath, u"kindlegen.exe")).encode(sys.getfilesystemencoding())
else:
kindlegen = (os.path.join(vendorpath, u"kindlegen")).encode(sys.getfilesystemencoding())
if not os.path.exists(kindlegen):
error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=kindlegen)
app.logger.error("make_mobi: " + error_message)
return error_message, RET_FAIL
book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first()
if not data:
@ -77,45 +67,7 @@ def make_mobi(book_id, calibrepath):
file_path = os.path.join(calibrepath, book.path, data.name)
if os.path.exists(file_path + u".epub"):
try:
p = subprocess.Popen((kindlegen + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception:
error_message = _(u"kindlegen failed, no execution permissions")
app.logger.error("make_mobi: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
if nextline != "\r\n":
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
# If error occoures, log in every case
if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
app.logger.info("make_mobi: " + error_message)
app.logger.info(nextline.strip('\r\n'))
app.logger.debug(nextline.strip('\r\n'))
check = p.returncode
if not check or check < 2:
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
return file_path + ".mobi", RET_SUCCESS
else:
app.logger.info("make_mobi: kindlegen failed with error while converting book")
if not error_message:
error_message = 'kindlegen failed, no excecution permissions'
return error_message, RET_FAIL
return converter.convert_mobi(file_path, book)
else:
error_message = "make_mobi: epub not found: %s.epub" % file_path
return error_message, RET_FAIL

@ -92,9 +92,9 @@ class server:
def getNameVersion(self):
if gevent_present:
return {'gevent':geventVersion}
return {'Gevent':'v'+geventVersion}
else:
return {'tornado':tornadoVersion}
return {'Tornado':'v'+tornadoVersion}
# Start Instance of Server

@ -159,6 +159,38 @@
</div>
</div>
</div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapseeight">
<span class="glyphicon glyphicon-plus"></span>
{{_('E-Book converter')}}
</div>
</h4>
</div>
<div id="collapseeight" class="panel-collapse collapse">
<div class="panel-body">
<div class="form-group">
<div><input type="radio" name="config_ebookconverter" id="converter0" value="0" {% if content.config_ebookconverter == 0 %}checked{% endif %}>
<label for="converter0">{{_('No converter')}}</label></div>
<div><label><input type="radio" name="config_ebookconverter" id="converter1" value="1" {% if content.config_ebookconverter == 1 %}checked{% endif %}>
<label for="converter1">{{_('Use Kindlegen')}}</label></div>
<div><label><input type="radio" name="config_ebookconverter" id="converter2" value="2" {% if content.config_ebookconverter == 2 %}checked{% endif %}>
<label for="converter2">{{_('Use calibre\'s ebook converter')}}</label></div>
</div>
<div data-related="calibre">
<div class="form-group">
<label for="config_calibre">{{_('E-Book converter settings')}}</label>
<input type="text" class="form-control" id="config_calibre" name="config_calibre" value="{% if content.config_calibre != None %}{{ content.config_calibre }}{% endif %}" autocomplete="off">
</div>
<div class="form-group">
<label for="config_calibre">{{_('Path to convertertool')}}</label>
<input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if content.config_converterpath != None %}{{ content.config_converterpath }}{% endif %}" autocomplete="off">
</div>
</div>
</div>
</div>
</div>
</div>

@ -34,75 +34,12 @@
</tr>
</thead>
<tbody>
{% for library,version in versions.iteritems() %}
<tr>
<th>Python</th>
<td>{{versions['PythonVersion']}}</td>
<th>{{library}}</th>
<td>{{version}}</td>
</tr>
{% if 'tornado' in versions %}
<tr>
<th>Tornado web server</th>
<td>v{{versions['tornado']}}</td>
</tr>
{% endif %}
{% if 'gevent' in versions %}
<tr>
<th>Gevent web server</th>
<td>v{{versions['gevent']}}</td>
</tr>
{% endif %}
<tr>
<th>Kindlegen</th>
<td>{{versions['KindlegenVersion']}}</td>
</tr>
<tr>
<th>ImageMagick</th>
<td>{{versions['ImageVersion']}}</td>
</tr>
<tr>
<th>PyPDF2</th>
<td>v{{versions['PyPdfVersion']}}</td>
</tr>
<tr>
<th>Babel</th>
<td>v{{versions['babel']}}</td>
</tr>
<tr>
<th>SqlAlchemy</th>
<td>v{{versions['sqlalchemy']}}</td>
</tr>
<tr>
<th>Flask</th>
<td>v{{versions['flask']}}</td>
</tr>
<tr>
<th>Flask Login</th>
<td>v{{versions['flasklogin']}}</td>
</tr>
<tr>
<th>Flask Principal</th>
<td>v{{versions['flask_principal']}}</td>
</tr>
<tr>
<th>ISO639 Languages</th>
<td>v{{versions['iso639']}}</td>
</tr>
<tr>
<th>Requests</th>
<td>v{{versions['requests']}}</td>
</tr>
<tr>
<th>SQlite</th>
<td>v{{versions['sqlite']}}</td>
</tr>
<tr>
<th>Pysqlite</th>
<td>v{{versions['pysqlite']}}</td>
</tr>
<tr>
<th>lxml</th>
<td>v{{versions['LxmlVersion']}}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}

@ -301,8 +301,11 @@ class Settings(Base):
config_use_goodreads = Column(Boolean)
config_goodreads_api_key = Column(String)
config_goodreads_api_secret = Column(String)
config_mature_content_tags = Column(String) # type: str
config_mature_content_tags = Column(String)
config_logfile = Column(String)
config_ebookconverter = Column(Integer, default=0)
config_converterpath = Column(String)
config_calibre = Column(String)
def __repr__(self):
pass
@ -355,6 +358,9 @@ class Config:
self.config_columns_to_ignore = data.config_columns_to_ignore
self.config_use_google_drive = data.config_use_google_drive
self.config_google_drive_folder = data.config_google_drive_folder
self.config_ebookconverter = data.config_ebookconverter
self.config_converterpath = data.config_converterpath
self.config_calibre = data.config_calibre
if data.config_google_drive_watch_changes_response:
self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response)
else:
@ -657,6 +663,16 @@ def migrate_Database():
conn = engine.connect()
conn.execute("ALTER TABLE Settings ADD column `config_read_column` INTEGER DEFAULT 0")
session.commit()
try:
session.query(exists().where(Settings.config_ebookconverter)).scalar()
session.commit()
except exc.OperationalError: # Database is not compatible, some rows are missing
conn = engine.connect()
conn.execute("ALTER TABLE Settings ADD column `config_ebookconverter` INTEGER DEFAULT 0")
conn.execute("ALTER TABLE Settings ADD column `config_converterpath` String DEFAULT ''")
conn.execute("ALTER TABLE Settings ADD column `config_calibre` String DEFAULT ''")
session.commit()
# Remove login capability of user Guest
conn = engine.connect()
conn.execute("UPDATE user SET password='' where nickname = 'Guest' and password !=''")

@ -66,15 +66,16 @@ from iso639 import __version__ as iso639Version
from uuid import uuid4
import os.path
import sys
import subprocess
import re
import db
from shutil import move, copyfile
import shutil
import gdriveutils
import converter
import tempfile
import hashlib
from redirect import redirect_back, is_safe_url
from redirect import redirect_back
try:
from urllib.parse import quote
@ -1430,40 +1431,23 @@ def admin_forbidden():
@app.route("/stats")
@login_required
def stats():
counter = len(db.session.query(db.Books).all())
authors = len(db.session.query(db.Authors).all())
categorys = len(db.session.query(db.Tags).all())
series = len(db.session.query(db.Series).all())
counter = db.session.query(db.Books).count()
authors = db.session.query(db.Authors).count()
categorys = db.session.query(db.Tags).count()
series = db.session.query(db.Series).count()
versions = uploader.book_formats.get_versions()
vendorpath = os.path.join(config.get_main_dir, "vendor")
if sys.platform == "win32":
kindlegen = os.path.join(vendorpath, u"kindlegen.exe")
else:
kindlegen = os.path.join(vendorpath, u"kindlegen")
versions['KindlegenVersion'] = _('not installed')
if os.path.exists(kindlegen):
try:
p = subprocess.Popen(kindlegen, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
for lines in p.stdout.readlines():
if isinstance(lines, bytes):
lines = lines.decode('utf-8')
if re.search('Amazon kindlegen\(', lines):
versions['KindlegenVersion'] = lines
except Exception:
versions['KindlegenVersion'] = _(u'Excecution permissions missing')
versions['PythonVersion'] = sys.version
versions['babel'] = babelVersion
versions['sqlalchemy'] = sqlalchemyVersion
versions['flask'] = flaskVersion
versions['flasklogin'] = flask_loginVersion
versions['flask_principal'] = flask_principalVersion
versions['Babel'] = 'v'+babelVersion
versions['Sqlalchemy'] = 'v'+sqlalchemyVersion
versions['Flask'] = 'v'+flaskVersion
versions['Flask Login'] = 'v'+flask_loginVersion
versions['Flask Principal'] = 'v'+flask_principalVersion
versions['Iso 639'] = 'v'+iso639Version
versions['Requests'] = 'v'+requests.__version__
versions['pySqlite'] = 'v'+db.engine.dialect.dbapi.version
versions['Sqlite'] = 'v'+db.engine.dialect.dbapi.sqlite_version
versions.update(converter.versioncheck())
versions.update(server.Server.getNameVersion())
# versions['tornado'] = tornadoVersion
versions['iso639'] = iso639Version
versions['requests'] = requests.__version__
versions['pysqlite'] = db.engine.dialect.dbapi.version
versions['sqlite'] = db.engine.dialect.dbapi.sqlite_version
versions['Python'] = sys.version
return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=versions,
categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat")
@ -2572,6 +2556,14 @@ def view_configuration():
ub.session.commit()
flash(_(u"Calibre-web configuration updated"), category="success")
config.loadSettings()
if reboot_required:
# db.engine.dispose() # ToDo verify correct
ub.session.close()
ub.engine.dispose()
# stop Server
server.Server.setRestartTyp(True)
server.Server.stopServer()
app.logger.info('Reboot required, restarting')
readColumn = db.session.query(db.Custom_Columns)\
.filter(db.and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all()
return render_title_template("config_view_edit.html", content=config, readColumns=readColumn,
@ -2680,6 +2672,13 @@ def configuration_helper(origin):
if "config_public_reg" in to_save and to_save["config_public_reg"] == "on":
content.config_public_reg = 1
if "config_converterpath" in to_save:
content.config_converterpath = to_save["config_converterpath"].strip()
if "config_calibre" in to_save:
content.config_calibre = to_save["config_calibre"].strip()
if "config_ebookconverter" in to_save:
content.config_ebookconverter = int(to_save["config_ebookconverter"])
# Remote login configuration
content.config_remote_login = ("config_remote_login" in to_save and to_save["config_remote_login"] == "on")
if not content.config_remote_login:

Loading…
Cancel
Save