simoon
/
parallel-library
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'worker_task' into Develop

Browse Source
pull/1539/merge
Ozzieisaacs 4 years ago
parent 0d64692c84 6e6f144b7a
commit 28116c49dc
11 changed files with 118 additions and 74 deletions
Show Stats Download Patch File Download Diff File

7
cps/db.py
Unescape Escape View File

@ -648,9 +648,10 @@ class CalibreDB():
# read search results from calibre-database and return it (function is used for feed and simple search
def get_search_results(self, term, offset=None, order=None, limit=None):
order = order or [Books.sort]
pagination = None
if offset != None and limit != None:
offset = int(offset)
limit = offset + int(limit)
limit_all = offset + int(limit)
term.strip().lower()
self.session.connection().connection.connection.create_function("lower", 1, lcase)
q = list()
@ -665,7 +666,9 @@ class CalibreDB():
func.lower(Books.title).ilike("%" + term + "%")
)).order_by(*order).all()
result_count = len(result)
return result[offset:limit], result_count
if offset != None and limit != None:
pagination = Pagination((offset / (int(limit)) + 1), limit, result_count)
return result[offset:limit_all], result_count, pagination
# Creates for all stored languages a translated speaking name in the array for the UI
def speaking_language(self, languages=None):

43
cps/helper.py
Unescape Escape View File

@ -74,40 +74,33 @@ log = logger.create()
def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None):
book = calibre_db.get_book(book_id)
data = calibre_db.get_book_format(book.id, old_book_format)
file_path = os.path.join(calibrepath, book.path, data.name)
if not data:
error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
log.error("convert_book_format: %s", error_message)
return error_message
if config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower())
if df:
datafile = os.path.join(calibrepath, book.path, data.name + u"." + old_book_format.lower())
if not os.path.exists(os.path.join(calibrepath, book.path)):
os.makedirs(os.path.join(calibrepath, book.path))
df.GetContentFile(datafile)
else:
if not gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()):
error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
format=old_book_format, fn=data.name + "." + old_book_format.lower())
return error_message
file_path = os.path.join(calibrepath, book.path, data.name)
if os.path.exists(file_path + "." + old_book_format.lower()):
# read settings and append converter task to queue
if kindle_mail:
settings = config.get_mail_settings()
settings['subject'] = _('Send to Kindle') # pretranslate Subject for e-mail
settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
# text = _(u"%(format)s: %(book)s", format=new_book_format, book=book.title)
else:
settings = dict()
txt = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title))
settings['old_book_format'] = old_book_format
settings['new_book_format'] = new_book_format
WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, kindle_mail))
return None
else:
error_message = _(u"%(format)s not found: %(fn)s",
format=old_book_format, fn=data.name + "." + old_book_format.lower())
return error_message
if not os.path.exists(file_path + "." + old_book_format.lower()):
error_message = _(u"%(format)s not found: %(fn)s",
format=old_book_format, fn=data.name + "." + old_book_format.lower())
return error_message
# read settings and append converter task to queue
if kindle_mail:
settings = config.get_mail_settings()
settings['subject'] = _('Send to Kindle') # pretranslate Subject for e-mail
settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
else:
settings = dict()
txt = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title))
settings['old_book_format'] = old_book_format
settings['new_book_format'] = new_book_format
WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, kindle_mail, user_id))
return None
def send_test_mail(kindle_mail, user_name):

2
cps/jinjia.py
Unescape Escape View File

@ -44,6 +44,8 @@ log = logger.create()
def url_for_other_page(page):
args = request.view_args.copy()
args['page'] = page
for get, val in request.args.items():
args[get] = val
return url_for(request.endpoint, **args)

2
cps/services/worker.py
Unescape Escape View File

@ -210,7 +210,7 @@ class CalibreTask:
self._progress = x
def _handleError(self, error_message):
log.error(error_message)
log.exception(error_message)
self.stat = STAT_FAIL
self.progress = 1
self.error = error_message

1
cps/static/js/table.js
Unescape Escape View File

@ -109,6 +109,7 @@ $(function() {
$("#books-table").bootstrapTable({
sidePagination: "server",
pagination: true,
paginationLoop: false,
paginationDetailHAlign: " hidden",
paginationHAlign: "left",
idField: "id",

41
cps/tasks/convert.py
Unescape Escape View File

@ -20,34 +20,52 @@ log = logger.create()
class TaskConvert(CalibreTask):
def __init__(self, file_path, bookid, taskMessage, settings, kindle_mail):
def __init__(self, file_path, bookid, taskMessage, settings, kindle_mail, user=None):
super(TaskConvert, self).__init__(taskMessage)
self.file_path = file_path
self.bookid = bookid
self.settings = settings
self.kindle_mail = kindle_mail
self.user = user
self.results = dict()
def run(self, worker_thread):
self.worker_thread = worker_thread
if config.config_use_google_drive:
cur_book = calibre_db.get_book(self.bookid)
data = calibre_db.get_book_format(self.bookid, self.settings['old_book_format'])
df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
data.name + "." + self.settings['old_book_format'].lower())
if df:
datafile = os.path.join(config.config_calibre_dir,
cur_book.path,
data.name + u"." + self.settings['old_book_format'].lower())
if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)):
os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path))
df.GetContentFile(datafile)
else:
error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
format=self.settings['old_book_format'],
fn=data.name + "." + self.settings['old_book_format'].lower())
return error_message
filename = self._convert_ebook_format()
if config.config_use_google_drive:
os.remove(self.file_path + u'.' + self.settings['old_book_format'].lower())
if filename:
if config.config_use_google_drive:
# Upload files to gdrive
gdriveutils.updateGdriveCalibreFromLocal()
self._handleSuccess()
if self.kindle_mail:
# if we're sending to kindle after converting, create a one-off task and run it immediately
# todo: figure out how to incorporate this into the progress
try:
task = TaskEmail(self.settings['subject'], self.results["path"],
worker_thread.add(self.user, TaskEmail(self.settings['subject'], self.results["path"],
filename, self.settings, self.kindle_mail,
self.settings['subject'], self.settings['body'], internal=True)
task.start(worker_thread)
# even though the convert task might be finished, if this task fails, fail the whole thing
if task.stat != STAT_FINISH_SUCCESS:
raise Exception(task.error)
self.settings['subject'], self.settings['body'], internal=True))
except Exception as e:
return self._handleError(str(e))
@ -101,9 +119,8 @@ class TaskConvert(CalibreTask):
return
self.results['path'] = cur_book.path
self.results['title'] = cur_book.title
if config.config_use_google_drive:
os.remove(file_path + format_old_ext)
self._handleSuccess()
if not config.config_use_google_drive:
self._handleSuccess()
return os.path.basename(file_path + format_new_ext)
else:
error_message = _('%(format)s format not found on disk', format=format_new_ext.upper())
@ -179,6 +196,8 @@ class TaskConvert(CalibreTask):
progress = re.search(r"(\d+)%\s.*", nextline)
if progress:
self.progress = int(progress.group(1)) / 100
if config.config_use_google_drive:
self.progress *= 0.9
# process returncode
check = p.returncode

7
cps/tasks/mail.py
Unescape Escape View File

@ -169,7 +169,7 @@ class TaskEmail(CalibreTask):
except (MemoryError) as e:
log.exception(e)
self._handleError(u'MemoryError sending email: ' + str(e))
return None
# return None
except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
if hasattr(e, "smtp_error"):
text = e.smtp_error.decode('utf-8').replace("\n", '. ')
@ -181,10 +181,11 @@ class TaskEmail(CalibreTask):
log.exception(e)
text = ''
self._handleError(u'Smtplib Error sending email: ' + text)
return None
# return None
except (socket.error) as e:
self._handleError(u'Socket Error sending email: ' + e.strerror)
return None
# return None
@property
def progress(self):

4
cps/templates/book_edit.html
Unescape Escape View File

@ -12,7 +12,9 @@
{% if book.data|length > 1 %}
<div class="text-center more-stuff"><h4>{{_('Delete formats:')}}</h4>
{% for file in book.data %}
<button type="button" class="btn btn-danger" id="delete_format" data-toggle="modal" data-delete-id="{{ book.id }}" data-delete-format="{{ file.format }}" data-target="#deleteModal">{{_('Delete')}} - {{file.format}}</button>
<div class="form-group">
<button type="button" class="btn btn-danger" id="delete_format" data-toggle="modal" data-delete-id="{{ book.id }}" data-delete-format="{{ file.format }}" data-target="#deleteModal">{{_('Delete')}} - {{file.format}}</button>
</div>
{% endfor %}
</div>
{% endif %}

16
cps/templates/book_table.html
Unescape Escape View File

@ -2,13 +2,13 @@
{% macro text_table_row(parameter, edit_text, show_text, validate) -%}
<th data-field="{{ parameter }}" id="{{ parameter }}" data-sortable="true"
data-visible = "{{visiblility.get(parameter)}}"
{% if g.user.role_edit() %}
{% if g.user.role_edit() %}
data-editable-type="text"
data-editable-url="{{ url_for('editbook.edit_list_book', param=parameter)}}"
data-editable-title="{{ edit_text }}"
data-edit="true"
{% if validate %}data-edit-validate="{{ _('This Field is Required') }}" {% endif %}
{% endif %}
{% if validate %}data-edit-validate="{{ _('This Field is Required') }}" {% endif %}
{% endif %}
>{{ show_text }}</th>
{%- endmacro %}
@ -39,9 +39,9 @@
data-url="{{url_for('web.list_books')}}">
<thead>
<tr>
{% if g.user.role_edit() %}
{% if g.user.role_edit() %}
<th data-field="state" data-checkbox="true" data-sortable="true"></th>
{% endif %}
{% endif %}
<th data-field="id" id="id" data-visible="false" data-switchable="false"></th>
{{ text_table_row('title', _('Enter Title'),_('Title'), true) }}
{{ text_table_row('sort', _('Enter Title Sort'),_('Title Sort'), false) }}
@ -49,13 +49,13 @@
{{ text_table_row('authors', _('Enter Authors'),_('Authors'), true) }}
{{ text_table_row('tags', _('Enter Categories'),_('Categories'), false) }}
{{ text_table_row('series', _('Enter Series'),_('Series'), false) }}
<th data-field="series_index" id="series_index" data-visible="{{visiblility.get('series_index')}}" data-edit-validate="{{ _('This Field is Required') }}" data-sortable="true" {% if g.user.role_edit() %} data-editable-type="number" data-editable-placeholder="1" data-editable-step="0.01" data-editable-min="0" data-editable-url="{{ url_for('editbook.edit_list_book', param='series_index')}}" data-edit="true" data-editable-title="{{_('Enter title')}}"{% endif %}>{{_('Series Index')}}</th>
<th data-field="series_index" id="series_index" data-visible="{{visiblility.get('series_index')}}" data-edit-validate="{{ _('This Field is Required') }}" data-sortable="true" {% if g.user.role_edit() %} data-editable-type="number" data-editable-placeholder="1" data-editable-step="0.01" data-editable-min="0" data-editable-url="{{ url_for('editbook.edit_list_book', param='series_index')}}" data-edit="true" data-editable-title="{{_('Enter title')}}"{% endif %}>{{_('Series Index')}}</th>
{{ text_table_row('languages', _('Enter Languages'),_('Languages'), false) }}
<!--th data-field="pubdate" data-type="date" data-visible="{{visiblility.get('pubdate')}}" data-viewformat="dd.mm.yyyy" id="pubdate" data-sortable="true">{{_('Publishing Date')}}</th-->
{{ text_table_row('publishers', _('Enter Publishers'),_('Publishers'), false) }}
{% if g.user.role_edit() %}
{% if g.user.role_edit() %}
<th data-align="right" data-formatter="EbookActions" data-switchable="false">{{_('Delete')}}</th>
{% endif %}
{% endif %}
</tr>
</thead>
</table>

1
cps/updater.py
Unescape Escape View File

@ -227,6 +227,7 @@ class Updater(threading.Thread):
os.sep + 'vendor', os.sep + 'calibre-web.log', os.sep + '.git', os.sep + 'client_secrets.json',
os.sep + 'gdrive_credentials', os.sep + 'settings.yaml', os.sep + 'venv', os.sep + 'virtualenv',
os.sep + 'access.log', os.sep + 'access.log1', os.sep + 'access.log2',
os.sep + '.calibre-web.log.swp'
)
additional_path = self.is_venv()
if additional_path:

68
cps/web.py
Unescape Escape View File

@ -35,6 +35,7 @@ from babel import Locale as LC
from babel.core import UnknownLocaleError
from flask import Blueprint, jsonify
from flask import render_template, request, redirect, send_from_directory, make_response, g, flash, abort, url_for
from flask import session as flask_session
from flask_babel import gettext as _
from flask_login import login_user, logout_user, login_required, current_user, confirm_login
from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError
@ -669,10 +670,11 @@ def render_books_list(data, sort, book_id, page):
elif data == "search":
term = (request.args.get('query') or '')
offset = int(int(config.config_books_per_page) * (page - 1))
if '&' not in term:
return render_search_results(term, offset, order, config.config_books_per_page)
else:
return render_adv_search_results(term, offset, order, config.config_books_per_page)
return render_search_results(term, offset, order, config.config_books_per_page)
elif data == "advsearch":
term = json.loads(flask_session['query'])
offset = int(int(config.config_books_per_page) * (page - 1))
return render_adv_search_results(term, offset, order, config.config_books_per_page)
else:
website = data or "newest"
entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order)
@ -953,13 +955,14 @@ def render_prepare_search_form(cc):
def render_search_results(term, offset=None, order=None, limit=None):
entries, result_count = calibre_db.get_search_results(term, offset, order, limit)
entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit)
ids = list()
for element in entries:
ids.append(element.id)
searched_ids[current_user.id] = ids
return render_title_template('search.html',
searchterm=term,
pagination=pagination,
query=term,
adv_searchterm=term,
entries=entries,
@ -967,6 +970,7 @@ def render_search_results(term, offset=None, order=None, limit=None):
title=_(u"Search"),
page="search")
# ################################### View Books list ##################################################################
@ -1007,7 +1011,7 @@ def list_books():
search = request.args.get("search")
total_count = calibre_db.session.query(db.Books).count()
if search:
entries, filtered_count = calibre_db.get_search_results(search, off, order, limit)
entries, filtered_count, pagination = calibre_db.get_search_results(search, off, order, limit)
else:
entries, __, __ = calibre_db.fill_indexpage((int(off) / (int(limit)) + 1), limit, db.Books, True, order)
filtered_count = total_count
@ -1227,7 +1231,7 @@ def reconnect():
def search():
term = request.args.get("query")
if term:
return render_search_results(term)
return render_search_results(term, 0, None, config.config_books_per_page)
else:
return render_title_template('search.html',
searchterm="",
@ -1238,9 +1242,19 @@ def search():
@web.route("/advanced_search", methods=['POST'])
@login_required_if_no_ano
def advanced_search():
term = request.form
return render_adv_search_results(term, 0, None, config.config_books_per_page)
def render_adv_search_results(term, offset=None, order=None, limit=None):
order = order or [db.Books.sort]
pagination = None
if offset != None and limit != None:
offset = int(offset)
limit_all = offset + int(limit)
cc = get_cc_columns(filter_config_custom_read=True)
calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True)).order_by(db.Books.sort)
q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True))
include_tag_inputs = request.form.getlist('include_tag')
exclude_tag_inputs = request.form.getlist('exclude_tag')
@ -1251,14 +1265,14 @@ def advanced_search():
include_extension_inputs = request.form.getlist('include_extension')
exclude_extension_inputs = request.form.getlist('exclude_extension')
author_name = request.form.get("author_name")
book_title = request.form.get("book_title")
publisher = request.form.get("publisher")
pub_start = request.form.get("Publishstart")
pub_end = request.form.get("Publishend")
rating_low = request.form.get("ratinghigh")
rating_high = request.form.get("ratinglow")
description = request.form.get("comment")
author_name = term.get("author_name")
book_title = term.get("book_title")
publisher = term.get("publisher")
pub_start = term.get("Publishstart")
pub_end = term.get("Publishend")
rating_low = term.get("ratinghigh")
rating_high = term.get("ratinglow")
description = term.get("comment")
if author_name:
author_name = author_name.strip().lower().replace(',', '|')
if book_title:
@ -1367,17 +1381,25 @@ def advanced_search():
else:
q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
func.lower(db.cc_classes[c.id].value).ilike("%" + custom_query + "%")))
q = q.all()
q = q.order_by(*order).all()
flask_session['query'] = json.dumps(term)
# ToDo: Check saved ids mechanism ?
ids = list()
for element in q:
ids.append(element.id)
searched_ids[current_user.id] = ids
return render_title_template('search.html',
adv_searchterm=searchterm,
query=request.form,
entries=q,
result_count=len(q),
title=_(u"search"), page="search")
# entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit)
result_count = len(q)
if offset != None and limit != None:
pagination = Pagination((offset / (int(limit)) + 1), limit, result_count)
return render_title_template('search.html',
adv_searchterm=searchterm,
pagination=pagination,
# query=request.form,
entries=q[offset:limit_all],
result_count=result_count,
title=_(u"search"), page="advsearch")
@web.route("/advanced_search", methods=['GET'])

Write Preview
Loading…
Cancel
Save