From 4b006fba99ad2c5fb5c3ff578aa72d2980d2ec01 Mon Sep 17 00:00:00 2001 From: bodybybuddha Date: Tue, 28 Aug 2018 23:32:30 -0400 Subject: [PATCH] Re-commit: #584 & #587 Convert any book format to another format --- cps/helper.py | 45 +++++++++++++++- cps/web.py | 28 ++++++---- cps/worker.py | 138 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 198 insertions(+), 13 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 152bcd25..e95c35c7 100755 --- a/cps/helper.py +++ b/cps/helper.py @@ -136,9 +136,50 @@ def send_mail(book_id, kindle_mail, calibrepath, user_id): # Convert existing book entry to new format -#def convert_book_format(book_id, calibrepath, new_book_format, user_id): +def convert_book_format(book_id, calibrepath, old_book_format, new_book_format, user_id): + book = db.session.query(db.Books).filter(db.Books.id == book_id).first() + data = db.session.query(db.Data).filter(db.Data.book == book.id).filter(db.Data.format == old_book_format).first() + if not data: + error_message = _(u"%(format)s format not found for book id: %(book)d", format=old_book_format, book=book_id) + app.logger.error("convert_book_format: " + error_message) + return error_message + if ub.config.config_use_google_drive: + df = gd.getFileFromEbooksFolder(book.path, data.name + "." + old_book_format.lower()) + if df: + datafile = os.path.join(calibrepath, book.path, data.name + "." + old_book_format.lower()) + if not os.path.exists(os.path.join(calibrepath, book.path)): + os.makedirs(os.path.join(calibrepath, book.path)) + df.GetContentFile(datafile) + else: + error_message = _(u"convert_book_format: %(format)s not found on gdrive: %(fn)s", + format=old_book_format, fn=data.name + "." + old_book_format.lower()) + return error_message + file_path = os.path.join(calibrepath, book.path, data.name) + if os.path.exists(file_path + "." + old_book_format.lower()): + # append converter to queue + settings = {'old_book_format': old_book_format, + 'new_book_format': new_book_format} + + app.logger.debug("Creating worker thread:") + app.logger.debug("filepath: " + file_path + " " + + "bookid: " + str(book.id) + " " + + "userid: " + str(user_id) + " " + + "taskmsg: " + _(u"Convert to %(format)s: %(book)s", + format=new_book_format, book=book.title) + " " + + "settings:old_book_format: " + settings['old_book_format'] + " " + + "settings:new_book_format: " + settings['new_book_format'] + ) + + global_WorkerThread.add_convert_any(file_path, book.id, + user_id, _(u"Convert to %(format)s: %(book)s", + format=new_book_format, book=book.title), + settings) + return None + else: + error_message = _(u"convert_book_format: %(format)s not found: %(fn)s", + format=old_book_format, fn=data.name + "." + old_book_format.lower()) + return error_message -# return def get_valid_filename(value, replace_whitespace=True): """ diff --git a/cps/web.py b/cps/web.py index 843805d4..9957b82a 100644 --- a/cps/web.py +++ b/cps/web.py @@ -3177,19 +3177,19 @@ def edit_book(book_id): for authr in book.authors: author_names.append(authr.name.replace('|', ',')) - #Option for showing convertbook button + # Option for showing convertbook button if config.config_ebookconverter == 2: display_convertbtn = True else: display_convertbtn = False - #Determine what formats don't already exist - allowed_conversion_formats = ALLOWED_EXTENSIONS + # Determine what formats don't already exist + allowed_conversion_formats = ALLOWED_EXTENSIONS.copy() for file in book.data: try: allowed_conversion_formats.remove(file.format.lower()) except Exception: - app.logger.warning(file.formate.lower() + ' already removed from list.') + app.logger.warning(file.format.lower() + ' already removed from list.') app.logger.debug(allowed_conversion_formats) @@ -3629,11 +3629,19 @@ def upload(): @login_required_if_no_ano @edit_required def convert_bookformat(book_id): - # rtn = convert_book_format(book_id, calibrepath, new_book_format, user_id) - app.logger.debug('got here') - app.logger.debug('book id:' + str(book_id)) - app.logger.debug('from format:'+request.form['book_format_from']) - app.logger.debug('to format:' + request.form['book_format_to']) + app.logger.debug('converting: book id: ' + str(book_id) + + ' from: ' + request.form['book_format_from'] + + ' to: ' + request.form['book_format_to']) + rtn = helper.convert_book_format(book_id, config.config_calibre_dir, request.form['book_format_from'].upper(), + request.form['book_format_to'].upper(), current_user.nickname) + + if rtn is None: + flash(_(u"Book successfully queued for converting to %(book_format)s", + book_format=request.form['book_format_from']), + category="success") + else: + flash(_(u"There was an error sending this book: %(res)s", res=rtn), category="error") + + return redirect(url_for("get_tasks_status")) - return redirect(url_for("index")) diff --git a/cps/worker.py b/cps/worker.py index e2958232..1d0eb6cf 100644 --- a/cps/worker.py +++ b/cps/worker.py @@ -42,6 +42,7 @@ STAT_FINISH_SUCCESS = 3 TASK_EMAIL = 1 TASK_CONVERT = 2 TASK_UPLOAD = 3 +TASK_CONVERT_ANY = 4 RET_FAIL = 0 RET_SUCCESS = 1 @@ -172,6 +173,8 @@ class WorkerThread(threading.Thread): self.send_raw_email() if self.queue[self.current]['typ'] == TASK_CONVERT: self.convert_mobi() + if self.queue[self.current]['typ'] == TASK_CONVERT_ANY: + self.convert_any_format() # TASK_UPLOAD is handled implicitly self.current += 1 else: @@ -357,7 +360,140 @@ class WorkerThread(threading.Thread): self.id += 1 self.last=len(self.queue) addLock.release() - + + def convert_any_format(self): + # convert book, and upload in case of google drive + self.queue[self.current]['status'] = STAT_STARTED + self.UIqueue[self.current]['status'] = _('Started') + self.queue[self.current]['starttime'] = datetime.now() + self.UIqueue[self.current]['formStarttime'] = self.queue[self.current]['starttime'] + filename=self.convert_ebook_format() + if web.ub.config.config_use_google_drive: + gd.updateGdriveCalibreFromLocal() + + def add_convert_any(self, file_path, bookid, user_name, typ, settings): + addLock = threading.Lock() + addLock.acquire() + if self.last >= 20: + self.delete_completed_tasks() + # progress, runtime, and status = 0 + self.id += 1 + self.queue.append({'file_path':file_path, 'bookid':bookid, 'starttime': 0, + 'status': STAT_WAITING, 'typ': TASK_CONVERT_ANY, 'settings':settings}) + self.UIqueue.append({'user': user_name, 'formStarttime': '', 'progress': " 0 %", 'type': typ, + 'runtime': '0 s', 'status': _('Waiting'),'id': self.id } ) + self.id += 1 + + self.last=len(self.queue) + addLock.release() + + def convert_ebook_format(self): + error_message = None + file_path = self.queue[self.current]['file_path'] + bookid = self.queue[self.current]['bookid'] + format_old_ext = u'.' + self.queue[self.current]['settings']['old_book_format'].lower() + format_new_ext = u'.' + self.queue[self.current]['settings']['new_book_format'].lower() + # check if converter-executable is existing + if not os.path.exists(web.ub.config.config_converterpath): + self._handleError(_(u"Convertertool %(converter)s not found", converter=web.ub.config.config_converterpath)) + return + try: + # check which converter to use kindlegen is "1" //backward compatibility assumption epub original format + if web.ub.config.config_ebookconverter == 1: + command = [web.ub.config.config_converterpath, u'"' + file_path + u'.epub"'] + else: + # Linux py2.7 encode as list without quotes no empty element for parameters + # linux py3.x no encode and as list without quotes no empty element for parameters + # windows py2.7 encode as string with quotes empty element for parameters is okay + # windows py 3.x no encode and as string with quotes empty element for parameters is okay + # separate handling for windows and linux + if os.name == 'nt': + command = web.ub.config.config_converterpath + u' "' + file_path + format_old_ext + u'" "' + \ + file_path + format_new_ext + u'" ' + web.ub.config.config_calibre + if sys.version_info < (3, 0): + command = command.encode(sys.getfilesystemencoding()) + else: + command = [web.ub.config.config_converterpath, (file_path + format_old_ext), + (file_path + format_new_ext)] + if web.ub.config.config_calibre: + command.append(web.ub.config.config_calibre) + if sys.version_info < (3, 0): + command = [ x.encode(sys.getfilesystemencoding()) for x in command ] + + p = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True) + except OSError as e: + self._handleError(_(u"Ebook-converter failed: %s" % e)) + return + if web.ub.config.config_ebookconverter == 1: + nextline = p.communicate()[0] + # Format of error message (kindlegen translates its output texts): + # Error(prcgen):E23006: Language not recognized in metadata.The dc:Language field is mandatory.Aborting. + conv_error = re.search(".*\(.*\):(E\d+):\s(.*)", nextline, re.MULTILINE) + # If error occoures, store error message for logfile + if conv_error: + error_message = _(u"Kindlegen failed with Error %(error)s. Message: %(message)s", + error=conv_error.group(1), message=conv_error.group(2).strip()) + web.app.logger.debug("convert_kindlegen: " + nextline) + + else: + while p.poll() is None: + nextline = p.stdout.readline() + if os.name == 'nt' and sys.version_info < (3, 0): + nextline = nextline.decode('windows-1252') + web.app.logger.debug(nextline.strip('\r\n')) + # parse progress string from calibre-converter + progress = re.search("(\d+)%\s.*", nextline) + if progress: + self.UIqueue[self.current]['progress'] = progress.group(1) + ' %' + + #process returncode + check = p.returncode + + # kindlegen returncodes + # 0 = Info(prcgen):I1036: Mobi file built successfully + # 1 = Info(prcgen):I1037: Mobi file built with WARNINGS! + # 2 = Info(prcgen):I1038: MOBI file could not be generated because of errors! + if ( check < 2 and web.ub.config.config_ebookconverter == 1) or \ + (check == 0 and web.ub.config.config_ebookconverter == 2): + cur_book = web.db.session.query(web.db.Books).filter(web.db.Books.id == bookid).first() + + if web.ub.config.config_ebookconverter == 1: + new_format = web.db.Data(name=cur_book.data[0].name, + book_format=self.queue[self.current]['settings']['new_book_format'], + book=bookid,uncompressed_size=os.path.getsize(file_path + '.mobi')) + else: + new_format = web.db.Data(name=cur_book.data[0].name, + book_format=self.queue[self.current]['settings']['new_book_format'], + book=bookid, uncompressed_size=os.path.getsize(file_path + format_new_ext)) + + cur_book.data.append(new_format) + web.db.session.commit() + self.queue[self.current]['path'] = cur_book.path + self.queue[self.current]['title'] = cur_book.title + self.queue[self.current]['status'] = STAT_FINISH_SUCCESS + self.UIqueue[self.current]['status'] = _('Finished') + self.UIqueue[self.current]['progress'] = "100 %" + self.UIqueue[self.current]['runtime'] = self._formatRuntime( + datetime.now() - self.queue[self.current]['starttime']) + + if web.ub.config.config_ebookconverter == 1: + if web.ub.config.config_use_google_drive: + os.remove(file_path + u".epub") + + return file_path + '.mobi' + else: + if web.ub.config.config_use_google_drive: + os.remove(file_path + format_old_ext) + + return file_path + format_new_ext + + else: + web.app.logger.info("ebook converter failed with error while converting book") + if not error_message: + error_message = 'Ebook converter failed with unknown error' + self._handleError(error_message) + return + def send_raw_email(self): self.queue[self.current]['starttime'] = datetime.now()