From 4afb4869861da1d43f143debefe9d00e5b0b51d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E6=AA=8E?= Date: Thu, 2 Mar 2017 19:59:35 +0800 Subject: [PATCH 1/5] add languages field for BookMeta --- cps/book_formats.py | 6 ++++-- cps/epub.py | 3 ++- cps/fb2.py | 3 ++- cps/uploader.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cps/book_formats.py b/cps/book_formats.py index 64ec86e8..e2911561 100644 --- a/cps/book_formats.py +++ b/cps/book_formats.py @@ -63,7 +63,8 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension): description="", tags="", series="", - series_id="") + series_id="", + languages=None) def pdf_meta(tmp_file_path, original_file_name, original_file_extension): @@ -91,7 +92,8 @@ def pdf_meta(tmp_file_path, original_file_name, original_file_extension): description=subject, tags="", series="", - series_id="") + series_id="", + languages=None) def pdf_preview(tmp_file_path, tmp_dir): diff --git a/cps/epub.py b/cps/epub.py index 62520aa3..e3e03cca 100644 --- a/cps/epub.py +++ b/cps/epub.py @@ -88,4 +88,5 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): description=epub_metadata['description'], tags="", series="", - series_id="") + series_id="", + languages=None) diff --git a/cps/fb2.py b/cps/fb2.py index 205f69ce..98aefdeb 100644 --- a/cps/fb2.py +++ b/cps/fb2.py @@ -59,4 +59,5 @@ def get_fb2_info(tmp_file_path, original_file_extension): description=description, tags="", series="", - series_id="") + series_id="", + languages=None) diff --git a/cps/uploader.py b/cps/uploader.py index 41283361..2772e033 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -7,7 +7,7 @@ import hashlib from collections import namedtuple import book_formats -BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, series_id') +BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, series_id, languages') """ :rtype: BookMeta From 879510da82d0b4f2db4f809d6c7eca4d86ab06e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E6=AA=8E?= Date: Thu, 2 Mar 2017 22:57:02 +0800 Subject: [PATCH 2/5] Upload support detection of language --- cps/book_formats.py | 4 ++-- cps/db.py | 2 +- cps/epub.py | 21 +++++++++++++++------ cps/fb2.py | 2 +- cps/web.py | 18 +++++++++++++++++- 5 files changed, 36 insertions(+), 11 deletions(-) diff --git a/cps/book_formats.py b/cps/book_formats.py index e2911561..a88e5e0e 100644 --- a/cps/book_formats.py +++ b/cps/book_formats.py @@ -64,7 +64,7 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension): tags="", series="", series_id="", - languages=None) + languages="") def pdf_meta(tmp_file_path, original_file_name, original_file_extension): @@ -93,7 +93,7 @@ def pdf_meta(tmp_file_path, original_file_name, original_file_extension): tags="", series="", series_id="", - languages=None) + languages="") def pdf_preview(tmp_file_path, tmp_dir): diff --git a/cps/db.py b/cps/db.py index f25848c7..9c48cfbb 100755 --- a/cps/db.py +++ b/cps/db.py @@ -247,7 +247,7 @@ class Books(Base): identifiers = relationship('Identifiers', backref='books') def __init__(self, title, sort, author_sort, timestamp, pubdate, series_index, last_modified, path, has_cover, - authors, tags): + authors, tags, languages = None): self.title = title self.sort = sort self.author_sort = author_sort diff --git a/cps/epub.py b/cps/epub.py index e3e03cca..853abcaa 100644 --- a/cps/epub.py +++ b/cps/epub.py @@ -5,7 +5,7 @@ import zipfile from lxml import etree import os import uploader - +from iso639 import languages as isoLanguages def extractCover(zip, coverFile, coverpath, tmp_file_name): if coverFile is None: @@ -47,6 +47,15 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): epub_metadata[s] = p.xpath('dc:%s/text()' % s, namespaces=ns)[0] else: epub_metadata[s] = "Unknown" + #detect lang need futher modification in web.py /upload + lang = p.xpath('dc:language/text()', namespaces=ns)[0] + lang = lang.split('-', 1)[0] + if len(lang) == 2: + epub_metadata['languages'] = isoLanguages.get(part1=lang).name + elif len(lang) == 3: + epub_metadata['languages'] = isoLanguages.get(part3=lang).name + else: + epub_metadata['languages'] = "" coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns) if len(coversection) > 0: @@ -59,16 +68,16 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): markup = zip.read(os.path.join(coverpath,coversection[0])) markupTree = etree.fromstring(markup) #no matter xhtml or html with no namespace - imgsrc = markupTree.xpath( "//*[local-name() = 'img']/@src") + imgsrc = markupTree.xpath("//*[local-name() = 'img']/@src") #imgsrc maybe startwith "../"" so fullpath join then relpath to cwd - filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(coverpath,coversection[0])),imgsrc[0])) - coverfile = extractCover(zip, filename, "", tmp_file_path) + filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(coverpath, coversection[0])), imgsrc[0])) + coverfile = extractCover(zip, filename, "", tmp_file_path) else: coverfile = extractCover(zip, coversection[0], coverpath, tmp_file_path) else: meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns) if len(meta_cover) > 0: - meta_cover_content = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href",namespaces=ns) + meta_cover_content = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns) if len(meta_cover_content) > 0: coverfile = extractCover(zip, meta_cover_content[0], coverpath, tmp_file_path) else: @@ -89,4 +98,4 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): tags="", series="", series_id="", - languages=None) + languages=epub_metadata['languages']) diff --git a/cps/fb2.py b/cps/fb2.py index 98aefdeb..7f0bf2af 100644 --- a/cps/fb2.py +++ b/cps/fb2.py @@ -60,4 +60,4 @@ def get_fb2_info(tmp_file_path, original_file_extension): tags="", series="", series_id="", - languages=None) + languages="") diff --git a/cps/web.py b/cps/web.py index 680b534e..566e6fce 100755 --- a/cps/web.py +++ b/cps/web.py @@ -2209,16 +2209,32 @@ def upload(): else: db_author = db.Authors(author, helper.get_sorted_author(author), "") db.session.add(db_author) + + #add language actually one value in list + input_language = meta.languages + db_language = None + if input_language != "": + input_language = isoLanguages.get(name=input_language).part3 + hasLanguage = db.session.query(db.Languages).filter(db.Languages.lang_code == input_language).first() + if hasLanguage: + db_language = hasLanguage + else: + db_language = db.Languages(input_language) + db.session.add(db_language) # combine path and normalize path from windows systems path = os.path.join(author_dir, title_dir).replace('\\','/') db_book = db.Books(title, "", db_author.sort, datetime.datetime.now(), datetime.datetime(101, 01, 01), 1, - datetime.datetime.now(), path, has_cover, db_author, []) + datetime.datetime.now(), path, has_cover, db_author, [], db_language) db_book.authors.append(db_author) + if db_language is not None: + db_book.languages.append(db_language) db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, data_name) db_book.data.append(db_data) db.session.add(db_book) db.session.commit() + if db_language is not None: #display Full name instead of iso639.part3 + db_book.languages[0].language_name = meta.languages author_names = [] for author in db_book.authors: author_names.append(author.name) From 13b4ae4c245025835fde6fced49792806761766a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E6=AA=8E?= Date: Thu, 2 Mar 2017 23:15:15 +0800 Subject: [PATCH 3/5] more error adaptive --- cps/epub.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/cps/epub.py b/cps/epub.py index 853abcaa..2393c256 100644 --- a/cps/epub.py +++ b/cps/epub.py @@ -48,13 +48,16 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): else: epub_metadata[s] = "Unknown" #detect lang need futher modification in web.py /upload - lang = p.xpath('dc:language/text()', namespaces=ns)[0] - lang = lang.split('-', 1)[0] - if len(lang) == 2: - epub_metadata['languages'] = isoLanguages.get(part1=lang).name - elif len(lang) == 3: - epub_metadata['languages'] = isoLanguages.get(part3=lang).name - else: + try:#maybe dc:language isn't present, less possible but possible + lang = p.xpath('dc:language/text()', namespaces=ns)[0] + lang = lang.split('-', 1)[0] + if len(lang) == 2: + epub_metadata['languages'] = isoLanguages.get(part1=lang).name + elif len(lang) == 3: + epub_metadata['languages'] = isoLanguages.get(part3=lang).name + else: + epub_metadata['languages'] = "" + except: epub_metadata['languages'] = "" coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns) From d5e525305df9cd021abbe46eae9ed94878244edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E6=AA=8E?= Date: Thu, 2 Mar 2017 23:20:37 +0800 Subject: [PATCH 4/5] Lower case text, some epub have uppercased which is beyond iso639 standard. --- cps/epub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cps/epub.py b/cps/epub.py index 2393c256..6e42add9 100644 --- a/cps/epub.py +++ b/cps/epub.py @@ -51,6 +51,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): try:#maybe dc:language isn't present, less possible but possible lang = p.xpath('dc:language/text()', namespaces=ns)[0] lang = lang.split('-', 1)[0] + lang.lower() if len(lang) == 2: epub_metadata['languages'] = isoLanguages.get(part1=lang).name elif len(lang) == 3: From 1a10859a57741c6214cd0207d75163aeb8b5fe76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9E=97=E6=AA=8E?= Date: Fri, 3 Mar 2017 12:46:19 +0800 Subject: [PATCH 5/5] When upload pub, add description to comment accord opf file --- cps/book_formats.py | 2 ++ cps/epub.py | 12 +++++++++--- cps/uploader.py | 2 +- cps/web.py | 9 ++++++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/cps/book_formats.py b/cps/book_formats.py index a88e5e0e..934638fc 100644 --- a/cps/book_formats.py +++ b/cps/book_formats.py @@ -64,6 +64,7 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension): tags="", series="", series_id="", + comments="", languages="") @@ -93,6 +94,7 @@ def pdf_meta(tmp_file_path, original_file_name, original_file_extension): tags="", series="", series_id="", + comments="", languages="") diff --git a/cps/epub.py b/cps/epub.py index 6e42add9..7e0f5665 100644 --- a/cps/epub.py +++ b/cps/epub.py @@ -41,6 +41,12 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0] epub_metadata = {} + try:#maybe description isn't present + comments = tree.xpath("//*[local-name() = 'description']/text()")[0] + epub_metadata['comments'] = comments + except: + epub_metadata['comments'] = "" + for s in ['title', 'description', 'creator']: tmp = p.xpath('dc:%s/text()' % s, namespaces=ns) if len(tmp) > 0: @@ -62,6 +68,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): epub_metadata['languages'] = "" coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns) + coverfile = None if len(coversection) > 0: coverfile = extractCover(zip, coversection[0], coverpath, tmp_file_path) else: @@ -84,9 +91,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): meta_cover_content = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns) if len(meta_cover_content) > 0: coverfile = extractCover(zip, meta_cover_content[0], coverpath, tmp_file_path) - else: - coverfile = None - + if epub_metadata['title'] is None: title = original_file_name else: @@ -102,4 +107,5 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): tags="", series="", series_id="", + comments=epub_metadata['comments'], languages=epub_metadata['languages']) diff --git a/cps/uploader.py b/cps/uploader.py index 2772e033..18e4bd43 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -7,7 +7,7 @@ import hashlib from collections import namedtuple import book_formats -BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, series_id, languages') +BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, series_id, comments, languages') """ :rtype: BookMeta diff --git a/cps/web.py b/cps/web.py index 566e6fce..080e770e 100755 --- a/cps/web.py +++ b/cps/web.py @@ -5,7 +5,7 @@ import logging from logging.handlers import RotatingFileHandler import textwrap from flask import Flask, render_template, request, Response, redirect, url_for, send_from_directory, \ - make_response, g, flash, abort + make_response, g, flash, abort, Markup from flask import __version__ as flaskVersion import ub from ub import config @@ -2232,6 +2232,13 @@ def upload(): db_book.data.append(db_data) db.session.add(db_book) + db.session.flush()# flush content get db_book.id avalible + #add comment + upload_comment = Markup(meta.comments).unescape() + db_comment = None + if upload_comment != "": + db_comment = db.Comments(upload_comment, db_book.id) + db.session.add(db_comment) db.session.commit() if db_language is not None: #display Full name instead of iso639.part3 db_book.languages[0].language_name = meta.languages