Merge branch 'develop' into personal

.gitignore

@@ -25,4 +25,7 @@ cps/static/[0-9]*
 tags
 
 settings.yaml
-gdrive_credentials
+gdrive_credentials
+
+#kindlegen
+vendor/kindlegen

cps/book_formats.py

@@ -63,7 +63,8 @@ def default_meta(tmp_file_path, original_file_name, original_file_extension):
         description="",
         tags="",
         series="",
-        series_id="")
+        series_id="",
+        languages="")
 
 
 def pdf_meta(tmp_file_path, original_file_name, original_file_extension):
@@ -91,7 +92,8 @@ def pdf_meta(tmp_file_path, original_file_name, original_file_extension):
         description=subject,
         tags="",
         series="",
-        series_id="")
+        series_id="",
+        languages="")
 
 
 def pdf_preview(tmp_file_path, tmp_dir):

cps/db.py

@@ -247,7 +247,7 @@ class Books(Base):
     identifiers = relationship('Identifiers', backref='books')
 
     def __init__(self, title, sort, author_sort, timestamp, pubdate, series_index, last_modified, path, has_cover,
-                 authors, tags):
+                 authors, tags, languages = None):
         self.title = title
         self.sort = sort
         self.author_sort = author_sort

cps/epub.py

@@ -5,7 +5,7 @@ import zipfile
 from lxml import etree
 import os
 import uploader
-
+from iso639 import languages as isoLanguages
 
 def extractCover(zip, coverFile, coverpath, tmp_file_name):
     if coverFile is None:
@@ -41,23 +41,53 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
     p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0]
 
     epub_metadata = {}
-    for s in ['title', 'description', 'creator']:
+
+    for s in ['title', 'description', 'creator', 'language']:
         tmp = p.xpath('dc:%s/text()' % s, namespaces=ns)
         if len(tmp) > 0:
             epub_metadata[s] = p.xpath('dc:%s/text()' % s, namespaces=ns)[0]
         else:
             epub_metadata[s] = "Unknown"
 
+    if epub_metadata['description'] == "Unknown":
+        description = tree.xpath("//*[local-name() = 'description']/text()")
+        if len(description) > 0:
+            epub_metadata['description'] = description
+        else:
+            epub_metadata['description'] = ""
+
+    if epub_metadata['language'] == "Unknown":
+        epub_metadata['language'] == ""
+    else:
+        lang = epub_metadata['language'].split('-', 1)[0].lower()
+        if len(lang) == 2:
+            epub_metadata['language'] = isoLanguages.get(part1=lang).name
+        elif len(lang) == 3:
+            epub_metadata['language'] = isoLanguages.get(part3=lang).name
+        else:
+            epub_metadata['language'] = ""
+
     coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns)
+    coverfile = None
     if len(coversection) > 0:
         coverfile = extractCover(zip, coversection[0], coverpath, tmp_file_path)
     else:
-        coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover']/@href", namespaces=ns)
+        meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns)
-        if len(coversection) > 0:
+        if len(meta_cover) > 0:
-            coverfile = extractCover(zip, coversection[0], coverpath, tmp_file_path)
+            coversection = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='"+meta_cover[0]+"']/@href", namespaces=ns)
-        else:
+            if len(coversection) > 0:
-            coverfile = None
+                filetype = coversection[0].rsplit('.',1)[-1]
-
+                if filetype == "xhtml" or filetype == "html": #if cover is (x)html format
+                    markup = zip.read(os.path.join(coverpath,coversection[0]))
+                    markupTree = etree.fromstring(markup)
+                    #no matter xhtml or html with no namespace
+                    imgsrc = markupTree.xpath("//*[local-name() = 'img']/@src")
+                    #imgsrc maybe startwith "../"" so fullpath join then relpath to cwd
+                    filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(coverpath, coversection[0])), imgsrc[0]))
+                    coverfile = extractCover(zip, filename, "", tmp_file_path)
+                else:
+                    coverfile = extractCover(zip, coversection[0], coverpath, tmp_file_path)
+            
     if epub_metadata['title'] is None:
         title = original_file_name
     else:
@@ -72,4 +102,5 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension):
         description=epub_metadata['description'],
         tags="",
         series="",
-        series_id="")
+        series_id="",
+        languages=epub_metadata['language'])

cps/fb2.py

@@ -59,4 +59,5 @@ def get_fb2_info(tmp_file_path, original_file_extension):
         description=description,
         tags="",
         series="",
-        series_id="")
+        series_id="",
+        languages="")

cps/helper.py

@@ -240,7 +240,7 @@ def get_valid_filename(value, replace_whitespace=True):
         value=value.replace(u'§',u'SS')
         value=value.replace(u'ß',u'ss')
         value = unicodedata.normalize('NFKD', value)
-        re_slugify = re.compile('[\W\s-]', re.UNICODE)
+        re_slugify = re.compile('[^\w\s-]', re.UNICODE)
         value = unicode(re_slugify.sub('', value).strip())
     if replace_whitespace:
         #*+:\"/<>? werden durch _ ersetzt

cps/uploader.py

@@ -7,7 +7,7 @@ import hashlib
 from collections import namedtuple
 import book_formats
 
-BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, series_id')
+BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, series_id, languages')
 
 """
  :rtype: BookMeta

cps/web.py

@@ -7,7 +7,7 @@ import logging
 from logging.handlers import RotatingFileHandler
 import textwrap
 from flask import Flask, render_template, session, request, Response, redirect, url_for, send_from_directory, \
-        make_response, g, flash, abort, send_file
+        make_response, g, flash, abort, send_file, Markup
 from flask import __version__ as flaskVersion
 import ub
 from ub import config
@@ -765,16 +765,16 @@ def get_opds_download_link(book_id, format):
     file_name = book.title
     if len(book.authors) > 0:
         file_name = book.authors[0].name + '-' + file_name
-
+    file_name = helper.get_valid_filename(file_name)
     if config.config_use_google_drive:
-        df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format))
+        df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, data.name + "." + format)
         download_url = df.metadata.get('downloadUrl')
         resp, content = df.auth.Get_Http_Object().request(download_url)
         response=send_file(io.BytesIO(content))
     else:
         # file_name = helper.get_valid_filename(file_name)
         response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
-    response.headers["Content-Disposition"] = "attachment; filename=\"%s.%s\"" % (data.name, format)
+    response.headers["Content-Disposition"] = "attachment; filename*=UTF-8''%s.%s" % (urllib.quote(file_name.encode('utf8')), format)
     return response
 
 
@@ -1169,7 +1169,7 @@ def stats():
     categorys = len(db.session.query(db.Tags).all())
     series = len(db.session.query(db.Series).all())
     versions = uploader.book_formats.get_versions()
-    vendorpath = os.path.join(config.get_main_dir + "vendor" + os.sep)
+    vendorpath = os.path.join(config.get_main_dir, "vendor")
     if sys.platform == "win32":
         kindlegen = os.path.join(vendorpath, u"kindlegen.exe")
     else:
@@ -1586,7 +1586,7 @@ def get_download_link(book_id, format):
             response.headers["Content-Type"] = mimetypes.types_map['.' + format]
         except:
             pass
-        response.headers["Content-Disposition"] = "attachment; filename=\"%s.%s\"" % (file_name.encode('utf-8'), format)
+        response.headers["Content-Disposition"] = "attachment; filename*=UTF-8''%s.%s" % (urllib.quote(file_name.encode('utf-8')), format)
         return response
     else:
         abort(404)
@@ -2560,16 +2560,39 @@ def upload():
         else:
             db_author = db.Authors(author, helper.get_sorted_author(author), "") 
             db.session.add(db_author)
+
+        #add language actually one value in list
+        input_language = meta.languages
+        db_language = None
+        if input_language != "":
+            input_language = isoLanguages.get(name=input_language).part3
+            hasLanguage = db.session.query(db.Languages).filter(db.Languages.lang_code == input_language).first()
+            if hasLanguage:
+                db_language = hasLanguage
+            else:
+                db_language = db.Languages(input_language)
+                db.session.add(db_language)
         # combine path and normalize path from windows systems
         path = os.path.join(author_dir, title_dir).replace('\\','/')
         db_book = db.Books(title, "", db_author.sort, datetime.datetime.now(), datetime.datetime(101, 01, 01), 1,
-                           datetime.datetime.now(), path, has_cover, db_author, [])
+                           datetime.datetime.now(), path, has_cover, db_author, [], db_language)
         db_book.authors.append(db_author)
+        if db_language is not None:
+            db_book.languages.append(db_language)
         db_data = db.Data(db_book, meta.extension.upper()[1:], file_size, data_name)
         db_book.data.append(db_data)
 
         db.session.add(db_book)
+        db.session.flush()# flush content get db_book.id avalible
+       #add comment
+        upload_comment = Markup(meta.description).unescape()
+        db_comment = None
+        if upload_comment != "":
+            db_comment = db.Comments(upload_comment, db_book.id)
+            db.session.add(db_comment)
         db.session.commit()
+        if db_language is not None: #display Full name instead of iso639.part3
+            db_book.languages[0].language_name = _(meta.languages)
         author_names = []
         for author in db_book.authors:
             author_names.append(author.name)