Code cleaning Stats page

Enable calibre's ebook-convert as converter for mobi files (#411, #533)
pull/552/head
OzzieIsaacs 6 years ago
parent ae0c5d7ec2
commit 2449b4049b

@ -127,11 +127,11 @@ def get_versions():
else: else:
IVersion = _(u'not installed') IVersion = _(u'not installed')
if use_pdf_meta: if use_pdf_meta:
PVersion=PyPdfVersion PVersion='v'+PyPdfVersion
else: else:
PVersion=_(u'not installed') PVersion=_(u'not installed')
if lxmlversion: if lxmlversion:
XVersion = '.'.join(map(str, lxmlversion)) XVersion = 'v'+'.'.join(map(str, lxmlversion))
else: else:
XVersion = _(u'not installed') XVersion = _(u'not installed')
return {'ImageVersion': IVersion, 'PyPdfVersion': PVersion, 'LxmlVersion':XVersion} return {'Image Magick': IVersion, 'PyPdf': PVersion, 'lxml':XVersion}

@ -0,0 +1,154 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import subprocess
import ub
import db
import re
import web
from flask_babel import gettext as _
RET_FAIL = 0
RET_SUCCESS = 1
def versionKindle():
versions = _(u'not installed')
if os.path.exists(ub.config.config_converterpath):
try:
p = subprocess.Popen(ub.config.config_converterpath, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
for lines in p.stdout.readlines():
if isinstance(lines, bytes):
lines = lines.decode('utf-8')
if re.search('Amazon kindlegen\(', lines):
versions = lines
except Exception:
versions = _(u'Excecution permissions missing')
return {'kindlegen' : versions}
def versionCalibre():
versions = _(u'not installed')
if os.path.exists(ub.config.config_converterpath):
try:
p = subprocess.Popen(ub.config.config_converterpath + ' --version', stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
for lines in p.stdout.readlines():
if isinstance(lines, bytes):
lines = lines.decode('utf-8')
if re.search('.*\(calibre', lines):
versions = lines
except Exception:
versions = _(u'Excecution permissions missing')
return {'Calibre converter' : versions}
def convert_kindlegen(file_path, book):
error_message = None
# vendorpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) +
# os.sep + "../vendor" + os.sep))
#if sys.platform == "win32":
# kindlegen = (os.path.join(vendorpath, u"kindlegen.exe")).encode(sys.getfilesystemencoding())
#else:
# kindlegen = (os.path.join(vendorpath, u"kindlegen")).encode(sys.getfilesystemencoding())
if not os.path.exists(ub.config.config_converterpath):
error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=ub.config.config_converterpath)
web.app.logger.error("convert_kindlegen: " + error_message)
return error_message, RET_FAIL
try:
p = subprocess.Popen((ub.config.config_converterpath + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception as e:
error_message = _(u"kindlegen failed, no execution permissions")
web.app.logger.error("convert_kindlegen: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
if nextline != "\r\n":
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
# If error occoures, log in every case
if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
web.app.logger.info("convert_kindlegen: " + error_message)
web.app.logger.info(nextline.strip('\r\n'))
else:
web.app.logger.debug(nextline.strip('\r\n'))
check = p.returncode
if not check or check < 2:
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
return file_path + ".mobi", RET_SUCCESS
else:
web.app.logger.info("convert_kindlegen: kindlegen failed with error while converting book")
if not error_message:
error_message = 'kindlegen failed, no excecution permissions'
return error_message, RET_FAIL
def convert_calibre(file_path, book):
error_message = None
if not os.path.exists(ub.config.config_converterpath):
error_message = _(u"Ebook-convert binary %(converterpath)s not found", converterpath=ub.config.config_converterpath)
web.app.logger.error("convert_calibre: " + error_message)
return error_message, RET_FAIL
try:
command = ("\""+ub.config.config_converterpath + "\" " + ub.config.config_calibre +
" \"" + file_path + u".epub\" \"" + file_path + u".mobi\"").encode(sys.getfilesystemencoding())
p = subprocess.Popen(command,stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception as e:
error_message = _(u"Ebook-convert failed, no execution permissions")
web.app.logger.error("convert_calibre: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
web.app.logger.debug(nextline.strip('\r\n').decode(sys.getfilesystemencoding()))
check = p.returncode
if check == 0 :
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
return file_path + ".mobi", RET_SUCCESS
else:
web.app.logger.info("convert_calibre: Ebook-convert failed with error while converting book")
if not error_message:
error_message = 'Ebook-convert failed, no excecution permissions'
return error_message, RET_FAIL
def versioncheck():
if ub.config.config_ebookconverter == 1:
return versionKindle()
elif ub.config.config_ebookconverter == 2:
return versionCalibre()
else:
return {'ebook_converter':''}
def convert_mobi(file_path, book):
if ub.config.config_ebookconverter == 2:
return convert_calibre(file_path, book)
else:
return convert_kindlegen(file_path, book)

@ -14,6 +14,7 @@ import traceback
import re import re
import unicodedata import unicodedata
from io import BytesIO from io import BytesIO
import converter
try: try:
from StringIO import StringIO from StringIO import StringIO
@ -57,17 +58,6 @@ RET_FAIL = 0
def make_mobi(book_id, calibrepath): def make_mobi(book_id, calibrepath):
error_message = None
vendorpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) +
os.sep + "../vendor" + os.sep))
if sys.platform == "win32":
kindlegen = (os.path.join(vendorpath, u"kindlegen.exe")).encode(sys.getfilesystemencoding())
else:
kindlegen = (os.path.join(vendorpath, u"kindlegen")).encode(sys.getfilesystemencoding())
if not os.path.exists(kindlegen):
error_message = _(u"kindlegen binary %(kindlepath)s not found", kindlepath=kindlegen)
app.logger.error("make_mobi: " + error_message)
return error_message, RET_FAIL
book = db.session.query(db.Books).filter(db.Books.id == book_id).first() book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first() data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == 'EPUB').first()
if not data: if not data:
@ -77,45 +67,7 @@ def make_mobi(book_id, calibrepath):
file_path = os.path.join(calibrepath, book.path, data.name) file_path = os.path.join(calibrepath, book.path, data.name)
if os.path.exists(file_path + u".epub"): if os.path.exists(file_path + u".epub"):
try: return converter.convert_mobi(file_path, book)
p = subprocess.Popen((kindlegen + " \"" + file_path + u".epub\"").encode(sys.getfilesystemencoding()),
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
except Exception:
error_message = _(u"kindlegen failed, no execution permissions")
app.logger.error("make_mobi: " + error_message)
return error_message, RET_FAIL
# Poll process for new output until finished
while True:
nextline = p.stdout.readline()
if nextline == '' and p.poll() is not None:
break
if nextline != "\r\n":
# Format of error message (kindlegen translates its output texts):
# Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting.
conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline)
# If error occoures, log in every case
if conv_error:
error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s",
error=conv_error.group(1), message=conv_error.group(2).decode('utf-8'))
app.logger.info("make_mobi: " + error_message)
app.logger.info(nextline.strip('\r\n'))
app.logger.debug(nextline.strip('\r\n'))
check = p.returncode
if not check or check < 2:
book.data.append(db.Data(
name=book.data[0].name,
book_format="MOBI",
book=book.id,
uncompressed_size=os.path.getsize(file_path + ".mobi")
))
db.session.commit()
return file_path + ".mobi", RET_SUCCESS
else:
app.logger.info("make_mobi: kindlegen failed with error while converting book")
if not error_message:
error_message = 'kindlegen failed, no excecution permissions'
return error_message, RET_FAIL
else: else:
error_message = "make_mobi: epub not found: %s.epub" % file_path error_message = "make_mobi: epub not found: %s.epub" % file_path
return error_message, RET_FAIL return error_message, RET_FAIL

@ -92,9 +92,9 @@ class server:
def getNameVersion(self): def getNameVersion(self):
if gevent_present: if gevent_present:
return {'gevent':geventVersion} return {'Gevent':'v'+geventVersion}
else: else:
return {'tornado':tornadoVersion} return {'Tornado':'v'+tornadoVersion}
# Start Instance of Server # Start Instance of Server

@ -159,6 +159,38 @@
</div> </div>
</div> </div>
</div> </div>
<div class="panel panel-default">
<div class="panel-heading">
<h4 class="panel-title">
<div class="accordion-toggle" data-toggle="collapse" href="#collapseeight">
<span class="glyphicon glyphicon-plus"></span>
{{_('E-Book converter')}}
</div>
</h4>
</div>
<div id="collapseeight" class="panel-collapse collapse">
<div class="panel-body">
<div class="form-group">
<div><input type="radio" name="config_ebookconverter" id="converter0" value="0" {% if content.config_ebookconverter == 0 %}checked{% endif %}>
<label for="converter0">{{_('No converter')}}</label></div>
<div><label><input type="radio" name="config_ebookconverter" id="converter1" value="1" {% if content.config_ebookconverter == 1 %}checked{% endif %}>
<label for="converter1">{{_('Use Kindlegen')}}</label></div>
<div><label><input type="radio" name="config_ebookconverter" id="converter2" value="2" {% if content.config_ebookconverter == 2 %}checked{% endif %}>
<label for="converter2">{{_('Use calibre\'s ebook converter')}}</label></div>
</div>
<div data-related="calibre">
<div class="form-group">
<label for="config_calibre">{{_('E-Book converter settings')}}</label>
<input type="text" class="form-control" id="config_calibre" name="config_calibre" value="{% if content.config_calibre != None %}{{ content.config_calibre }}{% endif %}" autocomplete="off">
</div>
<div class="form-group">
<label for="config_calibre">{{_('Path to convertertool')}}</label>
<input type="text" class="form-control" id="config_converterpath" name="config_converterpath" value="{% if content.config_converterpath != None %}{{ content.config_converterpath }}{% endif %}" autocomplete="off">
</div>
</div>
</div>
</div>
</div>
</div> </div>

@ -34,75 +34,12 @@
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{% for library,version in versions.iteritems() %}
<tr> <tr>
<th>Python</th> <th>{{library}}</th>
<td>{{versions['PythonVersion']}}</td> <td>{{version}}</td>
</tr> </tr>
{% if 'tornado' in versions %} {% endfor %}
<tr>
<th>Tornado web server</th>
<td>v{{versions['tornado']}}</td>
</tr>
{% endif %}
{% if 'gevent' in versions %}
<tr>
<th>Gevent web server</th>
<td>v{{versions['gevent']}}</td>
</tr>
{% endif %}
<tr>
<th>Kindlegen</th>
<td>{{versions['KindlegenVersion']}}</td>
</tr>
<tr>
<th>ImageMagick</th>
<td>{{versions['ImageVersion']}}</td>
</tr>
<tr>
<th>PyPDF2</th>
<td>v{{versions['PyPdfVersion']}}</td>
</tr>
<tr>
<th>Babel</th>
<td>v{{versions['babel']}}</td>
</tr>
<tr>
<th>SqlAlchemy</th>
<td>v{{versions['sqlalchemy']}}</td>
</tr>
<tr>
<th>Flask</th>
<td>v{{versions['flask']}}</td>
</tr>
<tr>
<th>Flask Login</th>
<td>v{{versions['flasklogin']}}</td>
</tr>
<tr>
<th>Flask Principal</th>
<td>v{{versions['flask_principal']}}</td>
</tr>
<tr>
<th>ISO639 Languages</th>
<td>v{{versions['iso639']}}</td>
</tr>
<tr>
<th>Requests</th>
<td>v{{versions['requests']}}</td>
</tr>
<tr>
<th>SQlite</th>
<td>v{{versions['sqlite']}}</td>
</tr>
<tr>
<th>Pysqlite</th>
<td>v{{versions['pysqlite']}}</td>
</tr>
<tr>
<th>lxml</th>
<td>v{{versions['LxmlVersion']}}</td>
</tr>
</tbody> </tbody>
</table> </table>
{% endblock %} {% endblock %}

@ -301,8 +301,11 @@ class Settings(Base):
config_use_goodreads = Column(Boolean) config_use_goodreads = Column(Boolean)
config_goodreads_api_key = Column(String) config_goodreads_api_key = Column(String)
config_goodreads_api_secret = Column(String) config_goodreads_api_secret = Column(String)
config_mature_content_tags = Column(String) # type: str config_mature_content_tags = Column(String)
config_logfile = Column(String) config_logfile = Column(String)
config_ebookconverter = Column(Integer, default=0)
config_converterpath = Column(String)
config_calibre = Column(String)
def __repr__(self): def __repr__(self):
pass pass
@ -355,6 +358,9 @@ class Config:
self.config_columns_to_ignore = data.config_columns_to_ignore self.config_columns_to_ignore = data.config_columns_to_ignore
self.config_use_google_drive = data.config_use_google_drive self.config_use_google_drive = data.config_use_google_drive
self.config_google_drive_folder = data.config_google_drive_folder self.config_google_drive_folder = data.config_google_drive_folder
self.config_ebookconverter = data.config_ebookconverter
self.config_converterpath = data.config_converterpath
self.config_calibre = data.config_calibre
if data.config_google_drive_watch_changes_response: if data.config_google_drive_watch_changes_response:
self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response) self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response)
else: else:
@ -657,6 +663,16 @@ def migrate_Database():
conn = engine.connect() conn = engine.connect()
conn.execute("ALTER TABLE Settings ADD column `config_read_column` INTEGER DEFAULT 0") conn.execute("ALTER TABLE Settings ADD column `config_read_column` INTEGER DEFAULT 0")
session.commit() session.commit()
try:
session.query(exists().where(Settings.config_ebookconverter)).scalar()
session.commit()
except exc.OperationalError: # Database is not compatible, some rows are missing
conn = engine.connect()
conn.execute("ALTER TABLE Settings ADD column `config_ebookconverter` INTEGER DEFAULT 0")
conn.execute("ALTER TABLE Settings ADD column `config_converterpath` String DEFAULT ''")
conn.execute("ALTER TABLE Settings ADD column `config_calibre` String DEFAULT ''")
session.commit()
# Remove login capability of user Guest # Remove login capability of user Guest
conn = engine.connect() conn = engine.connect()
conn.execute("UPDATE user SET password='' where nickname = 'Guest' and password !=''") conn.execute("UPDATE user SET password='' where nickname = 'Guest' and password !=''")

@ -66,15 +66,16 @@ from iso639 import __version__ as iso639Version
from uuid import uuid4 from uuid import uuid4
import os.path import os.path
import sys import sys
import subprocess
import re import re
import db import db
from shutil import move, copyfile from shutil import move, copyfile
import shutil import shutil
import gdriveutils import gdriveutils
import converter
import tempfile import tempfile
import hashlib import hashlib
from redirect import redirect_back, is_safe_url from redirect import redirect_back
try: try:
from urllib.parse import quote from urllib.parse import quote
@ -1430,40 +1431,23 @@ def admin_forbidden():
@app.route("/stats") @app.route("/stats")
@login_required @login_required
def stats(): def stats():
counter = len(db.session.query(db.Books).all()) counter = db.session.query(db.Books).count()
authors = len(db.session.query(db.Authors).all()) authors = db.session.query(db.Authors).count()
categorys = len(db.session.query(db.Tags).all()) categorys = db.session.query(db.Tags).count()
series = len(db.session.query(db.Series).all()) series = db.session.query(db.Series).count()
versions = uploader.book_formats.get_versions() versions = uploader.book_formats.get_versions()
vendorpath = os.path.join(config.get_main_dir, "vendor") versions['Babel'] = 'v'+babelVersion
if sys.platform == "win32": versions['Sqlalchemy'] = 'v'+sqlalchemyVersion
kindlegen = os.path.join(vendorpath, u"kindlegen.exe") versions['Flask'] = 'v'+flaskVersion
else: versions['Flask Login'] = 'v'+flask_loginVersion
kindlegen = os.path.join(vendorpath, u"kindlegen") versions['Flask Principal'] = 'v'+flask_principalVersion
versions['KindlegenVersion'] = _('not installed') versions['Iso 639'] = 'v'+iso639Version
if os.path.exists(kindlegen): versions['Requests'] = 'v'+requests.__version__
try: versions['pySqlite'] = 'v'+db.engine.dialect.dbapi.version
p = subprocess.Popen(kindlegen, stdout=subprocess.PIPE, stderr=subprocess.PIPE) versions['Sqlite'] = 'v'+db.engine.dialect.dbapi.sqlite_version
p.wait() versions.update(converter.versioncheck())
for lines in p.stdout.readlines():
if isinstance(lines, bytes):
lines = lines.decode('utf-8')
if re.search('Amazon kindlegen\(', lines):
versions['KindlegenVersion'] = lines
except Exception:
versions['KindlegenVersion'] = _(u'Excecution permissions missing')
versions['PythonVersion'] = sys.version
versions['babel'] = babelVersion
versions['sqlalchemy'] = sqlalchemyVersion
versions['flask'] = flaskVersion
versions['flasklogin'] = flask_loginVersion
versions['flask_principal'] = flask_principalVersion
versions.update(server.Server.getNameVersion()) versions.update(server.Server.getNameVersion())
# versions['tornado'] = tornadoVersion versions['Python'] = sys.version
versions['iso639'] = iso639Version
versions['requests'] = requests.__version__
versions['pysqlite'] = db.engine.dialect.dbapi.version
versions['sqlite'] = db.engine.dialect.dbapi.sqlite_version
return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=versions, return render_title_template('stats.html', bookcounter=counter, authorcounter=authors, versions=versions,
categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat") categorycounter=categorys, seriecounter=series, title=_(u"Statistics"), page="stat")
@ -2572,6 +2556,14 @@ def view_configuration():
ub.session.commit() ub.session.commit()
flash(_(u"Calibre-web configuration updated"), category="success") flash(_(u"Calibre-web configuration updated"), category="success")
config.loadSettings() config.loadSettings()
if reboot_required:
# db.engine.dispose() # ToDo verify correct
ub.session.close()
ub.engine.dispose()
# stop Server
server.Server.setRestartTyp(True)
server.Server.stopServer()
app.logger.info('Reboot required, restarting')
readColumn = db.session.query(db.Custom_Columns)\ readColumn = db.session.query(db.Custom_Columns)\
.filter(db.and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all() .filter(db.and_(db.Custom_Columns.datatype == 'bool',db.Custom_Columns.mark_for_delete == 0)).all()
return render_title_template("config_view_edit.html", content=config, readColumns=readColumn, return render_title_template("config_view_edit.html", content=config, readColumns=readColumn,
@ -2680,6 +2672,13 @@ def configuration_helper(origin):
if "config_public_reg" in to_save and to_save["config_public_reg"] == "on": if "config_public_reg" in to_save and to_save["config_public_reg"] == "on":
content.config_public_reg = 1 content.config_public_reg = 1
if "config_converterpath" in to_save:
content.config_converterpath = to_save["config_converterpath"].strip()
if "config_calibre" in to_save:
content.config_calibre = to_save["config_calibre"].strip()
if "config_ebookconverter" in to_save:
content.config_ebookconverter = int(to_save["config_ebookconverter"])
# Remote login configuration # Remote login configuration
content.config_remote_login = ("config_remote_login" in to_save and to_save["config_remote_login"] == "on") content.config_remote_login = ("config_remote_login" in to_save and to_save["config_remote_login"] == "on")
if not content.config_remote_login: if not content.config_remote_login:

Loading…
Cancel
Save