Merge branch 'worker_task' into Develop

pull/1539/merge
Ozzieisaacs 4 years ago
commit 28116c49dc

@ -648,9 +648,10 @@ class CalibreDB():
# read search results from calibre-database and return it (function is used for feed and simple search # read search results from calibre-database and return it (function is used for feed and simple search
def get_search_results(self, term, offset=None, order=None, limit=None): def get_search_results(self, term, offset=None, order=None, limit=None):
order = order or [Books.sort] order = order or [Books.sort]
pagination = None
if offset != None and limit != None: if offset != None and limit != None:
offset = int(offset) offset = int(offset)
limit = offset + int(limit) limit_all = offset + int(limit)
term.strip().lower() term.strip().lower()
self.session.connection().connection.connection.create_function("lower", 1, lcase) self.session.connection().connection.connection.create_function("lower", 1, lcase)
q = list() q = list()
@ -665,7 +666,9 @@ class CalibreDB():
func.lower(Books.title).ilike("%" + term + "%") func.lower(Books.title).ilike("%" + term + "%")
)).order_by(*order).all() )).order_by(*order).all()
result_count = len(result) result_count = len(result)
return result[offset:limit], result_count if offset != None and limit != None:
pagination = Pagination((offset / (int(limit)) + 1), limit, result_count)
return result[offset:limit_all], result_count, pagination
# Creates for all stored languages a translated speaking name in the array for the UI # Creates for all stored languages a translated speaking name in the array for the UI
def speaking_language(self, languages=None): def speaking_language(self, languages=None):

@ -74,40 +74,33 @@ log = logger.create()
def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None): def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id, kindle_mail=None):
book = calibre_db.get_book(book_id) book = calibre_db.get_book(book_id)
data = calibre_db.get_book_format(book.id, old_book_format) data = calibre_db.get_book_format(book.id, old_book_format)
file_path = os.path.join(calibrepath, book.path, data.name)
if not data: if not data:
error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id) error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id)
log.error("convert_book_format: %s", error_message) log.error("convert_book_format: %s", error_message)
return error_message return error_message
if config.config_use_google_drive: if config.config_use_google_drive:
df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()) if not gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()):
if df:
datafile = os.path.join(calibrepath, book.path, data.name + u"." + old_book_format.lower())
if not os.path.exists(os.path.join(calibrepath, book.path)):
os.makedirs(os.path.join(calibrepath, book.path))
df.GetContentFile(datafile)
else:
error_message = _(u"%(format)s not found on Google Drive: %(fn)s", error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
format=old_book_format, fn=data.name + "." + old_book_format.lower()) format=old_book_format, fn=data.name + "." + old_book_format.lower())
return error_message return error_message
file_path = os.path.join(calibrepath, book.path, data.name) else:
if os.path.exists(file_path + "." + old_book_format.lower()): if not os.path.exists(file_path + "." + old_book_format.lower()):
error_message = _(u"%(format)s not found: %(fn)s",
format=old_book_format, fn=data.name + "." + old_book_format.lower())
return error_message
# read settings and append converter task to queue # read settings and append converter task to queue
if kindle_mail: if kindle_mail:
settings = config.get_mail_settings() settings = config.get_mail_settings()
settings['subject'] = _('Send to Kindle') # pretranslate Subject for e-mail settings['subject'] = _('Send to Kindle') # pretranslate Subject for e-mail
settings['body'] = _(u'This e-mail has been sent via Calibre-Web.') settings['body'] = _(u'This e-mail has been sent via Calibre-Web.')
# text = _(u"%(format)s: %(book)s", format=new_book_format, book=book.title)
else: else:
settings = dict() settings = dict()
txt = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title)) txt = (u"%s -> %s: %s" % (old_book_format, new_book_format, book.title))
settings['old_book_format'] = old_book_format settings['old_book_format'] = old_book_format
settings['new_book_format'] = new_book_format settings['new_book_format'] = new_book_format
WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, kindle_mail)) WorkerThread.add(user_id, TaskConvert(file_path, book.id, txt, settings, kindle_mail, user_id))
return None return None
else:
error_message = _(u"%(format)s not found: %(fn)s",
format=old_book_format, fn=data.name + "." + old_book_format.lower())
return error_message
def send_test_mail(kindle_mail, user_name): def send_test_mail(kindle_mail, user_name):

@ -44,6 +44,8 @@ log = logger.create()
def url_for_other_page(page): def url_for_other_page(page):
args = request.view_args.copy() args = request.view_args.copy()
args['page'] = page args['page'] = page
for get, val in request.args.items():
args[get] = val
return url_for(request.endpoint, **args) return url_for(request.endpoint, **args)

@ -210,7 +210,7 @@ class CalibreTask:
self._progress = x self._progress = x
def _handleError(self, error_message): def _handleError(self, error_message):
log.error(error_message) log.exception(error_message)
self.stat = STAT_FAIL self.stat = STAT_FAIL
self.progress = 1 self.progress = 1
self.error = error_message self.error = error_message

@ -109,6 +109,7 @@ $(function() {
$("#books-table").bootstrapTable({ $("#books-table").bootstrapTable({
sidePagination: "server", sidePagination: "server",
pagination: true, pagination: true,
paginationLoop: false,
paginationDetailHAlign: " hidden", paginationDetailHAlign: " hidden",
paginationHAlign: "left", paginationHAlign: "left",
idField: "id", idField: "id",

@ -20,34 +20,52 @@ log = logger.create()
class TaskConvert(CalibreTask): class TaskConvert(CalibreTask):
def __init__(self, file_path, bookid, taskMessage, settings, kindle_mail): def __init__(self, file_path, bookid, taskMessage, settings, kindle_mail, user=None):
super(TaskConvert, self).__init__(taskMessage) super(TaskConvert, self).__init__(taskMessage)
self.file_path = file_path self.file_path = file_path
self.bookid = bookid self.bookid = bookid
self.settings = settings self.settings = settings
self.kindle_mail = kindle_mail self.kindle_mail = kindle_mail
self.user = user
self.results = dict() self.results = dict()
def run(self, worker_thread): def run(self, worker_thread):
self.worker_thread = worker_thread self.worker_thread = worker_thread
if config.config_use_google_drive:
cur_book = calibre_db.get_book(self.bookid)
data = calibre_db.get_book_format(self.bookid, self.settings['old_book_format'])
df = gdriveutils.getFileFromEbooksFolder(cur_book.path,
data.name + "." + self.settings['old_book_format'].lower())
if df:
datafile = os.path.join(config.config_calibre_dir,
cur_book.path,
data.name + u"." + self.settings['old_book_format'].lower())
if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)):
os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path))
df.GetContentFile(datafile)
else:
error_message = _(u"%(format)s not found on Google Drive: %(fn)s",
format=self.settings['old_book_format'],
fn=data.name + "." + self.settings['old_book_format'].lower())
return error_message
filename = self._convert_ebook_format() filename = self._convert_ebook_format()
if config.config_use_google_drive:
os.remove(self.file_path + u'.' + self.settings['old_book_format'].lower())
if filename: if filename:
if config.config_use_google_drive: if config.config_use_google_drive:
# Upload files to gdrive
gdriveutils.updateGdriveCalibreFromLocal() gdriveutils.updateGdriveCalibreFromLocal()
self._handleSuccess()
if self.kindle_mail: if self.kindle_mail:
# if we're sending to kindle after converting, create a one-off task and run it immediately # if we're sending to kindle after converting, create a one-off task and run it immediately
# todo: figure out how to incorporate this into the progress # todo: figure out how to incorporate this into the progress
try: try:
task = TaskEmail(self.settings['subject'], self.results["path"], worker_thread.add(self.user, TaskEmail(self.settings['subject'], self.results["path"],
filename, self.settings, self.kindle_mail, filename, self.settings, self.kindle_mail,
self.settings['subject'], self.settings['body'], internal=True) self.settings['subject'], self.settings['body'], internal=True))
task.start(worker_thread)
# even though the convert task might be finished, if this task fails, fail the whole thing
if task.stat != STAT_FINISH_SUCCESS:
raise Exception(task.error)
except Exception as e: except Exception as e:
return self._handleError(str(e)) return self._handleError(str(e))
@ -101,8 +119,7 @@ class TaskConvert(CalibreTask):
return return
self.results['path'] = cur_book.path self.results['path'] = cur_book.path
self.results['title'] = cur_book.title self.results['title'] = cur_book.title
if config.config_use_google_drive: if not config.config_use_google_drive:
os.remove(file_path + format_old_ext)
self._handleSuccess() self._handleSuccess()
return os.path.basename(file_path + format_new_ext) return os.path.basename(file_path + format_new_ext)
else: else:
@ -179,6 +196,8 @@ class TaskConvert(CalibreTask):
progress = re.search(r"(\d+)%\s.*", nextline) progress = re.search(r"(\d+)%\s.*", nextline)
if progress: if progress:
self.progress = int(progress.group(1)) / 100 self.progress = int(progress.group(1)) / 100
if config.config_use_google_drive:
self.progress *= 0.9
# process returncode # process returncode
check = p.returncode check = p.returncode

@ -169,7 +169,7 @@ class TaskEmail(CalibreTask):
except (MemoryError) as e: except (MemoryError) as e:
log.exception(e) log.exception(e)
self._handleError(u'MemoryError sending email: ' + str(e)) self._handleError(u'MemoryError sending email: ' + str(e))
return None # return None
except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e: except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e:
if hasattr(e, "smtp_error"): if hasattr(e, "smtp_error"):
text = e.smtp_error.decode('utf-8').replace("\n", '. ') text = e.smtp_error.decode('utf-8').replace("\n", '. ')
@ -181,10 +181,11 @@ class TaskEmail(CalibreTask):
log.exception(e) log.exception(e)
text = '' text = ''
self._handleError(u'Smtplib Error sending email: ' + text) self._handleError(u'Smtplib Error sending email: ' + text)
return None # return None
except (socket.error) as e: except (socket.error) as e:
self._handleError(u'Socket Error sending email: ' + e.strerror) self._handleError(u'Socket Error sending email: ' + e.strerror)
return None # return None
@property @property
def progress(self): def progress(self):

@ -12,7 +12,9 @@
{% if book.data|length > 1 %} {% if book.data|length > 1 %}
<div class="text-center more-stuff"><h4>{{_('Delete formats:')}}</h4> <div class="text-center more-stuff"><h4>{{_('Delete formats:')}}</h4>
{% for file in book.data %} {% for file in book.data %}
<div class="form-group">
<button type="button" class="btn btn-danger" id="delete_format" data-toggle="modal" data-delete-id="{{ book.id }}" data-delete-format="{{ file.format }}" data-target="#deleteModal">{{_('Delete')}} - {{file.format}}</button> <button type="button" class="btn btn-danger" id="delete_format" data-toggle="modal" data-delete-id="{{ book.id }}" data-delete-format="{{ file.format }}" data-target="#deleteModal">{{_('Delete')}} - {{file.format}}</button>
</div>
{% endfor %} {% endfor %}
</div> </div>
{% endif %} {% endif %}

@ -2,13 +2,13 @@
{% macro text_table_row(parameter, edit_text, show_text, validate) -%} {% macro text_table_row(parameter, edit_text, show_text, validate) -%}
<th data-field="{{ parameter }}" id="{{ parameter }}" data-sortable="true" <th data-field="{{ parameter }}" id="{{ parameter }}" data-sortable="true"
data-visible = "{{visiblility.get(parameter)}}" data-visible = "{{visiblility.get(parameter)}}"
{% if g.user.role_edit() %} {% if g.user.role_edit() %}
data-editable-type="text" data-editable-type="text"
data-editable-url="{{ url_for('editbook.edit_list_book', param=parameter)}}" data-editable-url="{{ url_for('editbook.edit_list_book', param=parameter)}}"
data-editable-title="{{ edit_text }}" data-editable-title="{{ edit_text }}"
data-edit="true" data-edit="true"
{% if validate %}data-edit-validate="{{ _('This Field is Required') }}" {% endif %} {% if validate %}data-edit-validate="{{ _('This Field is Required') }}" {% endif %}
{% endif %} {% endif %}
>{{ show_text }}</th> >{{ show_text }}</th>
{%- endmacro %} {%- endmacro %}
@ -39,9 +39,9 @@
data-url="{{url_for('web.list_books')}}"> data-url="{{url_for('web.list_books')}}">
<thead> <thead>
<tr> <tr>
{% if g.user.role_edit() %} {% if g.user.role_edit() %}
<th data-field="state" data-checkbox="true" data-sortable="true"></th> <th data-field="state" data-checkbox="true" data-sortable="true"></th>
{% endif %} {% endif %}
<th data-field="id" id="id" data-visible="false" data-switchable="false"></th> <th data-field="id" id="id" data-visible="false" data-switchable="false"></th>
{{ text_table_row('title', _('Enter Title'),_('Title'), true) }} {{ text_table_row('title', _('Enter Title'),_('Title'), true) }}
{{ text_table_row('sort', _('Enter Title Sort'),_('Title Sort'), false) }} {{ text_table_row('sort', _('Enter Title Sort'),_('Title Sort'), false) }}
@ -49,13 +49,13 @@
{{ text_table_row('authors', _('Enter Authors'),_('Authors'), true) }} {{ text_table_row('authors', _('Enter Authors'),_('Authors'), true) }}
{{ text_table_row('tags', _('Enter Categories'),_('Categories'), false) }} {{ text_table_row('tags', _('Enter Categories'),_('Categories'), false) }}
{{ text_table_row('series', _('Enter Series'),_('Series'), false) }} {{ text_table_row('series', _('Enter Series'),_('Series'), false) }}
<th data-field="series_index" id="series_index" data-visible="{{visiblility.get('series_index')}}" data-edit-validate="{{ _('This Field is Required') }}" data-sortable="true" {% if g.user.role_edit() %} data-editable-type="number" data-editable-placeholder="1" data-editable-step="0.01" data-editable-min="0" data-editable-url="{{ url_for('editbook.edit_list_book', param='series_index')}}" data-edit="true" data-editable-title="{{_('Enter title')}}"{% endif %}>{{_('Series Index')}}</th> <th data-field="series_index" id="series_index" data-visible="{{visiblility.get('series_index')}}" data-edit-validate="{{ _('This Field is Required') }}" data-sortable="true" {% if g.user.role_edit() %} data-editable-type="number" data-editable-placeholder="1" data-editable-step="0.01" data-editable-min="0" data-editable-url="{{ url_for('editbook.edit_list_book', param='series_index')}}" data-edit="true" data-editable-title="{{_('Enter title')}}"{% endif %}>{{_('Series Index')}}</th>
{{ text_table_row('languages', _('Enter Languages'),_('Languages'), false) }} {{ text_table_row('languages', _('Enter Languages'),_('Languages'), false) }}
<!--th data-field="pubdate" data-type="date" data-visible="{{visiblility.get('pubdate')}}" data-viewformat="dd.mm.yyyy" id="pubdate" data-sortable="true">{{_('Publishing Date')}}</th--> <!--th data-field="pubdate" data-type="date" data-visible="{{visiblility.get('pubdate')}}" data-viewformat="dd.mm.yyyy" id="pubdate" data-sortable="true">{{_('Publishing Date')}}</th-->
{{ text_table_row('publishers', _('Enter Publishers'),_('Publishers'), false) }} {{ text_table_row('publishers', _('Enter Publishers'),_('Publishers'), false) }}
{% if g.user.role_edit() %} {% if g.user.role_edit() %}
<th data-align="right" data-formatter="EbookActions" data-switchable="false">{{_('Delete')}}</th> <th data-align="right" data-formatter="EbookActions" data-switchable="false">{{_('Delete')}}</th>
{% endif %} {% endif %}
</tr> </tr>
</thead> </thead>
</table> </table>

@ -227,6 +227,7 @@ class Updater(threading.Thread):
os.sep + 'vendor', os.sep + 'calibre-web.log', os.sep + '.git', os.sep + 'client_secrets.json', os.sep + 'vendor', os.sep + 'calibre-web.log', os.sep + '.git', os.sep + 'client_secrets.json',
os.sep + 'gdrive_credentials', os.sep + 'settings.yaml', os.sep + 'venv', os.sep + 'virtualenv', os.sep + 'gdrive_credentials', os.sep + 'settings.yaml', os.sep + 'venv', os.sep + 'virtualenv',
os.sep + 'access.log', os.sep + 'access.log1', os.sep + 'access.log2', os.sep + 'access.log', os.sep + 'access.log1', os.sep + 'access.log2',
os.sep + '.calibre-web.log.swp'
) )
additional_path = self.is_venv() additional_path = self.is_venv()
if additional_path: if additional_path:

@ -35,6 +35,7 @@ from babel import Locale as LC
from babel.core import UnknownLocaleError from babel.core import UnknownLocaleError
from flask import Blueprint, jsonify from flask import Blueprint, jsonify
from flask import render_template, request, redirect, send_from_directory, make_response, g, flash, abort, url_for from flask import render_template, request, redirect, send_from_directory, make_response, g, flash, abort, url_for
from flask import session as flask_session
from flask_babel import gettext as _ from flask_babel import gettext as _
from flask_login import login_user, logout_user, login_required, current_user, confirm_login from flask_login import login_user, logout_user, login_required, current_user, confirm_login
from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError from sqlalchemy.exc import IntegrityError, InvalidRequestError, OperationalError
@ -669,9 +670,10 @@ def render_books_list(data, sort, book_id, page):
elif data == "search": elif data == "search":
term = (request.args.get('query') or '') term = (request.args.get('query') or '')
offset = int(int(config.config_books_per_page) * (page - 1)) offset = int(int(config.config_books_per_page) * (page - 1))
if '&' not in term:
return render_search_results(term, offset, order, config.config_books_per_page) return render_search_results(term, offset, order, config.config_books_per_page)
else: elif data == "advsearch":
term = json.loads(flask_session['query'])
offset = int(int(config.config_books_per_page) * (page - 1))
return render_adv_search_results(term, offset, order, config.config_books_per_page) return render_adv_search_results(term, offset, order, config.config_books_per_page)
else: else:
website = data or "newest" website = data or "newest"
@ -953,13 +955,14 @@ def render_prepare_search_form(cc):
def render_search_results(term, offset=None, order=None, limit=None): def render_search_results(term, offset=None, order=None, limit=None):
entries, result_count = calibre_db.get_search_results(term, offset, order, limit) entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit)
ids = list() ids = list()
for element in entries: for element in entries:
ids.append(element.id) ids.append(element.id)
searched_ids[current_user.id] = ids searched_ids[current_user.id] = ids
return render_title_template('search.html', return render_title_template('search.html',
searchterm=term, searchterm=term,
pagination=pagination,
query=term, query=term,
adv_searchterm=term, adv_searchterm=term,
entries=entries, entries=entries,
@ -967,6 +970,7 @@ def render_search_results(term, offset=None, order=None, limit=None):
title=_(u"Search"), title=_(u"Search"),
page="search") page="search")
# ################################### View Books list ################################################################## # ################################### View Books list ##################################################################
@ -1007,7 +1011,7 @@ def list_books():
search = request.args.get("search") search = request.args.get("search")
total_count = calibre_db.session.query(db.Books).count() total_count = calibre_db.session.query(db.Books).count()
if search: if search:
entries, filtered_count = calibre_db.get_search_results(search, off, order, limit) entries, filtered_count, pagination = calibre_db.get_search_results(search, off, order, limit)
else: else:
entries, __, __ = calibre_db.fill_indexpage((int(off) / (int(limit)) + 1), limit, db.Books, True, order) entries, __, __ = calibre_db.fill_indexpage((int(off) / (int(limit)) + 1), limit, db.Books, True, order)
filtered_count = total_count filtered_count = total_count
@ -1227,7 +1231,7 @@ def reconnect():
def search(): def search():
term = request.args.get("query") term = request.args.get("query")
if term: if term:
return render_search_results(term) return render_search_results(term, 0, None, config.config_books_per_page)
else: else:
return render_title_template('search.html', return render_title_template('search.html',
searchterm="", searchterm="",
@ -1238,9 +1242,19 @@ def search():
@web.route("/advanced_search", methods=['POST']) @web.route("/advanced_search", methods=['POST'])
@login_required_if_no_ano @login_required_if_no_ano
def advanced_search(): def advanced_search():
term = request.form
return render_adv_search_results(term, 0, None, config.config_books_per_page)
def render_adv_search_results(term, offset=None, order=None, limit=None):
order = order or [db.Books.sort]
pagination = None
if offset != None and limit != None:
offset = int(offset)
limit_all = offset + int(limit)
cc = get_cc_columns(filter_config_custom_read=True) cc = get_cc_columns(filter_config_custom_read=True)
calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase) calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase)
q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True)).order_by(db.Books.sort) q = calibre_db.session.query(db.Books).filter(calibre_db.common_filters(True))
include_tag_inputs = request.form.getlist('include_tag') include_tag_inputs = request.form.getlist('include_tag')
exclude_tag_inputs = request.form.getlist('exclude_tag') exclude_tag_inputs = request.form.getlist('exclude_tag')
@ -1251,14 +1265,14 @@ def advanced_search():
include_extension_inputs = request.form.getlist('include_extension') include_extension_inputs = request.form.getlist('include_extension')
exclude_extension_inputs = request.form.getlist('exclude_extension') exclude_extension_inputs = request.form.getlist('exclude_extension')
author_name = request.form.get("author_name") author_name = term.get("author_name")
book_title = request.form.get("book_title") book_title = term.get("book_title")
publisher = request.form.get("publisher") publisher = term.get("publisher")
pub_start = request.form.get("Publishstart") pub_start = term.get("Publishstart")
pub_end = request.form.get("Publishend") pub_end = term.get("Publishend")
rating_low = request.form.get("ratinghigh") rating_low = term.get("ratinghigh")
rating_high = request.form.get("ratinglow") rating_high = term.get("ratinglow")
description = request.form.get("comment") description = term.get("comment")
if author_name: if author_name:
author_name = author_name.strip().lower().replace(',', '|') author_name = author_name.strip().lower().replace(',', '|')
if book_title: if book_title:
@ -1367,17 +1381,25 @@ def advanced_search():
else: else:
q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any( q = q.filter(getattr(db.Books, 'custom_column_' + str(c.id)).any(
func.lower(db.cc_classes[c.id].value).ilike("%" + custom_query + "%"))) func.lower(db.cc_classes[c.id].value).ilike("%" + custom_query + "%")))
q = q.all() q = q.order_by(*order).all()
flask_session['query'] = json.dumps(term)
# ToDo: Check saved ids mechanism ?
ids = list() ids = list()
for element in q: for element in q:
ids.append(element.id) ids.append(element.id)
searched_ids[current_user.id] = ids searched_ids[current_user.id] = ids
# entries, result_count, pagination = calibre_db.get_search_results(term, offset, order, limit)
result_count = len(q)
if offset != None and limit != None:
pagination = Pagination((offset / (int(limit)) + 1), limit, result_count)
return render_title_template('search.html', return render_title_template('search.html',
adv_searchterm=searchterm, adv_searchterm=searchterm,
query=request.form, pagination=pagination,
entries=q, # query=request.form,
result_count=len(q), entries=q[offset:limit_all],
title=_(u"search"), page="search") result_count=result_count,
title=_(u"search"), page="advsearch")
@web.route("/advanced_search", methods=['GET']) @web.route("/advanced_search", methods=['GET'])

Loading…
Cancel
Save