Merge remote-tracking branch 'kobo_book_delete' into Develop

# Conflicts:
#	cps/kobo.py
#	cps/services/SyncToken.py
#	cps/templates/book_edit.html
#	cps/ub.py

cps/config_sql.py

@@ -71,7 +71,7 @@ class _Settings(_Base):
     config_kobo_sync = Column(Boolean, default=False)
 
     config_default_role = Column(SmallInteger, default=0)
-    config_default_show = Column(SmallInteger, default=6143)
+    config_default_show = Column(SmallInteger, default=38911)
     config_columns_to_ignore = Column(String)
 
     config_denied_tags = Column(String, default="")

cps/constants.py

@@ -80,9 +80,10 @@ MATURE_CONTENT          = 1 << 11
 SIDEBAR_PUBLISHER       = 1 << 12
 SIDEBAR_RATING          = 1 << 13
 SIDEBAR_FORMAT          = 1 << 14
+SIDEBAR_ARCHIVED        = 1 << 15
 
 ADMIN_USER_ROLES        = sum(r for r in ALL_ROLES.values()) & ~ROLE_EDIT_SHELFS & ~ROLE_ANONYMOUS
-ADMIN_USER_SIDEBAR      = (SIDEBAR_FORMAT << 1) - 1
+ADMIN_USER_SIDEBAR      = (SIDEBAR_ARCHIVED << 1) - 1
 
 UPDATE_STABLE       = 0 << 0
 AUTO_UPDATE_STABLE  = 1 << 0

cps/editbooks.py

@@ -179,6 +179,7 @@ def delete_book(book_id, book_format):
                 # delete book from Shelfs, Downloads, Read list
                 ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete()
                 ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete()
+                ub.session.query(ub.ArchivedBook).filter(ub.ReadBook.book_id == book_id).delete()
                 ub.delete_download(book_id)
                 ub.session.commit()
 
@@ -261,6 +262,7 @@ def render_edit_book(book_id):
     return render_title_template('book_edit.html', book=book, authors=author_names, cc=cc,
                                  title=_(u"edit metadata"), page="editbook",
                                  conversion_formats=allowed_conversion_formats,
+                                 config=config,
                                  source_formats=valid_source_formats)
 
 

cps/helper.py

@@ -684,7 +684,19 @@ def render_task_status(tasklist):
 
 
 # Language and content filters for displaying in the UI
-def common_filters():
+def common_filters(allow_show_archived=False):
+    if not allow_show_archived:
+        archived_books = (
+            ub.session.query(ub.ArchivedBook)
+            .filter(ub.ArchivedBook.user_id == int(current_user.id))
+            .filter(ub.ArchivedBook.is_archived == True)
+            .all()
+        )
+        archived_book_ids = [archived_book.book_id for archived_book in archived_books]
+        archived_filter = db.Books.id.notin_(archived_book_ids)
+    else:
+        archived_filter = true()
+
     if current_user.filter_language() != "all":
         lang_filter = db.Books.languages.any(db.Languages.lang_code == current_user.filter_language())
     else:
@@ -706,7 +718,7 @@ def common_filters():
         pos_content_cc_filter = true()
         neg_content_cc_filter = false()
     return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter,
-                pos_content_cc_filter, ~neg_content_cc_filter)
+                pos_content_cc_filter, ~neg_content_cc_filter, archived_filter)
 
 
 def tags_filters():
@@ -764,15 +776,19 @@ def order_authors(entry):
 
 # Fill indexpage with all requested data from database
 def fill_indexpage(page, database, db_filter, order, *join):
+    return fill_indexpage_with_archived_books(page, database, db_filter, order, False, *join)
+
+
+def fill_indexpage_with_archived_books(page, database, db_filter, order, allow_show_archived, *join):
     if current_user.show_detail_random():
-        randm = db.session.query(db.Books).filter(common_filters())\
+        randm = db.session.query(db.Books).filter(common_filters(allow_show_archived))\
             .order_by(func.random()).limit(config.config_random_books)
     else:
         randm = false()
     off = int(int(config.config_books_per_page) * (page - 1))
     pagination = Pagination(page, config.config_books_per_page,
-                            len(db.session.query(database).filter(db_filter).filter(common_filters()).all()))
+                            len(db.session.query(database).filter(db_filter).filter(common_filters(allow_show_archived)).all()))
-    entries = db.session.query(database).join(*join, isouter=True).filter(db_filter).filter(common_filters()).\
+    entries = db.session.query(database).join(*join, isouter=True).filter(db_filter).filter(common_filters(allow_show_archived)).\
         order_by(*order).offset(off).limit(config.config_books_per_page).all()
     for book in entries:
         book = order_authors(book)

cps/kobo.py

@@ -23,6 +23,7 @@ import base64
 import os
 import uuid
 from time import gmtime, strftime
+
 try:
     from urllib import unquote
 except ImportError:
@@ -38,10 +39,10 @@ from flask import (
     redirect,
     abort
 )
-from flask_login import current_user, login_required
+from flask_login import login_required, current_user
 from werkzeug.datastructures import Headers
 from sqlalchemy import func
-from sqlalchemy.sql.expression import and_
+from sqlalchemy.sql.expression import or_
 import requests
 
 from . import config, logger, kobo_auth, db, helper, ub
@@ -58,6 +59,7 @@ kobo_auth.register_url_value_preprocessor(kobo)
 
 log = logger.create()
 
+
 def get_store_url_for_current_request():
     # Programmatically modify the current url to point to the official Kobo store
     base, sep, request_path_with_auth_token = request.full_path.rpartition("/kobo/")
@@ -99,9 +101,6 @@ def redirect_or_proxy_request():
     if config.config_kobo_proxy:
         if request.method == "GET":
             return redirect(get_store_url_for_current_request(), 307)
-        if request.method == "DELETE":
-            log.info('Delete Book')
-            return make_response(jsonify({}))
         else:
             # The Kobo device turns other request types into GET requests on redirects, so we instead proxy to the Kobo store ourselves.
             store_response = make_request_to_kobo_store()
@@ -141,13 +140,35 @@ def HandleSyncRequest():
     # in case of external changes (e.g: adding a book through Calibre).
     db.reconnect_db(config)
 
+    archived_books = (
+        ub.session.query(ub.ArchivedBook)
+        .filter(ub.ArchivedBook.user_id == int(current_user.id))
+        .all()
+    )
+
+    # We join-in books that have had their Archived bit recently modified in order to either:
+    #   * Restore them to the user's device.
+    #   * Delete them from the user's device.
+    # (Ideally we would use a join for this logic, however cross-database joins don't look trivial in SqlAlchemy.)
+    recently_restored_or_archived_books = []
+    archived_book_ids = {}
+    new_archived_last_modified = datetime.min
+    for archived_book in archived_books:
+        if archived_book.last_modified > sync_token.archive_last_modified:
+            recently_restored_or_archived_books.append(archived_book.book_id)
+        if archived_book.is_archived:
+            archived_book_ids[archived_book.book_id] = True
+        new_archived_last_modified = max(
+            new_archived_last_modified, archived_book.last_modified)
+
     # sqlite gives unexpected results when performing the last_modified comparison without the datetime cast.
     # It looks like it's treating the db.Books.last_modified field as a string and may fail
     # the comparison because of the +00:00 suffix.
     changed_entries = (
         db.session.query(db.Books)
         .join(db.Data)
-        .filter(func.datetime(db.Books.last_modified) > sync_token.books_last_modified)
+        .filter(or_(func.datetime(db.Books.last_modified) > sync_token.books_last_modified,
+                    db.Books.id.in_(recently_restored_or_archived_books)))
         .filter(db.Data.format.in_(KOBO_FORMATS))
         .all()
     )
@@ -156,7 +177,7 @@ def HandleSyncRequest():
     for book in changed_entries:
         kobo_reading_state = get_or_create_reading_state(book.id)
         entitlement = {
-            "BookEntitlement": create_book_entitlement(book),
+            "BookEntitlement": create_book_entitlement(book, archived=(book.id in archived_book_ids)),
             "BookMetadata": get_metadata(book),
         }
 
@@ -191,7 +212,7 @@ def HandleSyncRequest():
 
     sync_token.books_last_created = new_books_last_created
     sync_token.books_last_modified = new_books_last_modified
-    sync_token.reading_state_last_modified = new_reading_state_last_modified
+    sync_token.archive_last_modified = new_archived_last_modified
 
     if config.config_kobo_proxy:
         return generate_sync_response(request, sync_token, sync_results)
@@ -256,7 +277,7 @@ def get_download_url_for_book(book, book_format):
     )
 
 
-def create_book_entitlement(book):
+def create_book_entitlement(book, archived):
     book_uuid = book.uuid
     return {
         "Accessibility": "Full",
@@ -264,17 +285,20 @@ def create_book_entitlement(book):
         "Created": convert_to_kobo_timestamp_string(book.timestamp),
         "CrossRevisionId": book_uuid,
         "Id": book_uuid,
+        "IsRemoved": archived,
         "IsHiddenFromArchive": False,
         "IsLocked": False,
-        # Setting this to true removes from the device.
+        "LastModified": book.last_modified,
-        "IsRemoved": False,
-        "LastModified": convert_to_kobo_timestamp_string(book.last_modified),
         "OriginCategory": "Imported",
         "RevisionId": book_uuid,
         "Status": "Active",
     }
 
 
+def current_time():
+    return strftime("%Y-%m-%dT%H:%M:%SZ", gmtime())
+
+
 def get_description(book):
     if not book.comments:
         return None
@@ -526,13 +550,39 @@ def TopLevelEndpoint():
     return make_response(jsonify({}))
 
 
+@kobo.route("/v1/library/<book_uuid>", methods=["DELETE"])
+@login_required
+def HandleBookDeletionRequest(book_uuid):
+    log.info("Kobo book deletion request received for book %s" % book_uuid)
+    book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first()
+    if not book:
+        log.info(u"Book %s not found in database", book_uuid)
+        return redirect_or_proxy_request()
+
+    book_id = book.id
+    archived_book = (
+        ub.session.query(ub.ArchivedBook)
+        .filter(ub.ArchivedBook.book_id == book_id)
+        .first()
+    )
+    if not archived_book:
+        archived_book = ub.ArchivedBook(user_id=current_user.id, book_id=book_id)
+    archived_book.is_archived = True
+    archived_book.last_modified = datetime.utcnow()
+
+    ub.session.merge(archived_book)
+    ub.session.commit()
+
+    return ("", 204)
+
+
 # TODO: Implement the following routes
-@kobo.route("/v1/library/<dummy>", methods=["DELETE", "GET"])
+@kobo.route("/v1/library/<book_uuid>/state", methods=["PUT"])
 @kobo.route("/v1/library/tags", methods=["POST"])
 @kobo.route("/v1/library/tags/<shelf_name>", methods=["POST"])
 @kobo.route("/v1/library/tags/<tag_id>", methods=["DELETE"])
-def HandleUnimplementedRequest(dummy=None, book_uuid=None, shelf_name=None, tag_id=None):
+def HandleUnimplementedRequest(book_uuid=None, shelf_name=None, tag_id=None):
-    log.debug("Unimplemented Library Request received: %s", request.base_url)
+    log.debug("Alternative Request received:")
     return redirect_or_proxy_request()
 
 

cps/services/SyncToken.py

@@ -76,7 +76,7 @@ class SyncToken():
             "raw_kobo_store_token": {"type": "string"},
             "books_last_modified": {"type": "string"},
             "books_last_created": {"type": "string"},
-            "reading_state_last_modified": {"type": "string"},
+            "archive_last_modified": {"type": "string"},
         },
     }
 
@@ -85,13 +85,12 @@ class SyncToken():
         raw_kobo_store_token="",
         books_last_created=datetime.min,
         books_last_modified=datetime.min,
-        reading_state_last_modified=datetime.min,
+        archive_last_modified=datetime.min,
     ):
         self.raw_kobo_store_token = raw_kobo_store_token
         self.books_last_created = books_last_created
         self.books_last_modified = books_last_modified
-        self.reading_state_last_modified = reading_state_last_modified
+        self.archive_last_modified = archive_last_modified
-
 
     @staticmethod
     def from_headers(headers):
@@ -123,7 +122,7 @@ class SyncToken():
         try:
             books_last_modified = get_datetime_from_json(data_json, "books_last_modified")
             books_last_created = get_datetime_from_json(data_json, "books_last_created")
-            reading_state_last_modified = get_datetime_from_json(data_json, "reading_state_last_modified")
+            archive_last_modified = get_datetime_from_json(data_json, "archive_last_modified")
         except TypeError:
             log.error("SyncToken timestamps don't parse to a datetime.")
             return SyncToken(raw_kobo_store_token=raw_kobo_store_token)
@@ -132,7 +131,7 @@ class SyncToken():
             raw_kobo_store_token=raw_kobo_store_token,
             books_last_created=books_last_created,
             books_last_modified=books_last_modified,
-            reading_state_last_modified=reading_state_last_modified
+            archive_last_modified=archive_last_modified
         )
 
     def set_kobo_store_header(self, store_headers):
@@ -153,7 +152,7 @@ class SyncToken():
                 "raw_kobo_store_token": self.raw_kobo_store_token,
                 "books_last_modified": to_epoch_timestamp(self.books_last_modified),
                 "books_last_created": to_epoch_timestamp(self.books_last_created),
-                "reading_state_last_modified": to_epoch_timestamp(self.reading_state_last_modified)
+                "archive_last_modified": to_epoch_timestamp(self.archive_last_modified)
             },
         }
         return b64encode_json(token)

cps/static/js/caliBlur.js

@@ -216,6 +216,8 @@ if ( $( 'body.book' ).length > 0 ) {
     .prependTo( '[aria-label^="Download, send"]' );
   $( '#have_read_cb' )
     .after( '<label class="block-label readLbl" for="#have_read_cb"></label>' );
+  $( '#archived_cb' )
+    .after( '<label class="block-label readLbl" for="#archived_cb"></label>' );
   $( '#shelf-actions' ).prependTo( '[aria-label^="Download, send"]' );
 
 
@@ -586,6 +588,20 @@ $( '#have_read_cb:checked' ).attr({
   'data-viewport': '.btn-toolbar' })
   .addClass('readunread-btn-tooltip');
 
+  $( '#archived_cb' ).attr({
+    'data-toggle': 'tooltip',
+    'title': $( '#archived_cb').attr('data-unchecked'),
+    'data-placement': 'bottom',
+    'data-viewport': '.btn-toolbar' })
+    .addClass('readunread-btn-tooltip');
+
+  $( '#archived_cb:checked' ).attr({
+    'data-toggle': 'tooltip',
+    'title': $( '#archived_cb').attr('data-checked'),
+    'data-placement': 'bottom',
+    'data-viewport': '.btn-toolbar' })
+    .addClass('readunread-btn-tooltip');
+
   $( 'button#delete' ).attr({
     'data-toggle-two': 'tooltip',
     'title': $( 'button#delete' ).text(),           //'Delete'
@@ -601,6 +617,14 @@ $( '#have_read_cb' ).click(function() {
   }
 });
 
+$( '#archived_cb' ).click(function() {
+  if ( $( '#archived_cb:checked' ).length > 0 ) {
+      $( this ).attr('data-original-title', $('#archived_cb').attr('data-checked'));
+  } else {
+      $( this).attr('data-original-title', $('#archived_cb').attr('data-unchecked'));
+  }
+});
+
 $( '.btn-group[aria-label="Edit/Delete book"] a' ).attr({
    'data-toggle': 'tooltip',
    'title': $( '#edit_book' ).text(),               // 'Edit'

cps/static/js/details.js

@@ -25,6 +25,14 @@ $("#have_read_cb").on("change", function() {
     $(this).closest("form").submit();
 });
 
+$(function() {
+    $("#archived_form").ajaxForm();
+});
+
+$("#archived_cb").on("change", function() {
+    $(this).closest("form").submit();
+});
+
 (function() {
     var templates = {
         add: _.template(

cps/templates/book_edit.html

@@ -200,8 +200,16 @@
           <span>{{_('Are you really sure?')}}</span>
       </div>
         <div class="modal-body text-center">
+          <p>
           <span>{{_('This book will be permanently erased from database')}}</span>
           <span>{{_('and hard disk')}}</span>
+		  </p>
+          {% if config.config_kobo_sync %}
+          <p>
+            <span>{{_('Important Kobo Note: deleted books will remain on any paired Kobo device.')}}</span>
+            <span>{{_('Books must first be archived and the device synced before a book can safely be deleted.')}}</span>
+          </p>
+          {% endif %}
         </div>
 
       <div class="modal-footer">

cps/templates/detail.html

@@ -202,6 +202,14 @@
             </label>
           </form>
           </p>
+          <p>
+            <form id="archived_form" action="{{ url_for('web.toggle_archived', book_id=entry.id)}}" method="POST">
+              <label class="block-label">
+                <input id="archived_cb" data-checked="{{_('Restore from archive')}}" data-unchecked="{{_('Add to archive')}}" type="checkbox" {% if is_archived %}checked{% endif %} >
+                <span>{{_('Archived')}}</span>
+              </label>
+            </form>
+          </p>
         </div>
       {% endif %}
 

cps/ub.py

@@ -98,10 +98,13 @@ def get_sidebar_config(kwargs=None):
     sidebar.append({"glyph": "glyphicon-file", "text": _('File formats'), "link": 'web.formats_list', "id": "format",
                     "visibility": constants.SIDEBAR_FORMAT, 'public': True,
                     "page": "format", "show_text": _('Show file formats selection'), "config_show":True})
+    sidebar.append(
+        {"glyph": "glyphicon-trash", "text": _('Archived Books'), "link": 'web.books_list', "id": "archived",
+         "visibility": constants.SIDEBAR_ARCHIVED, 'public': (not g.user.is_anonymous), "page": "archived",
+         "show_text": _('Show archived books'), "config_show": True})
     return sidebar
 
 
-
 class UserBase:
 
     @property
@@ -310,6 +313,16 @@ class Bookmark(Base):
     format = Column(String(collation='NOCASE'))
     bookmark_key = Column(String)
 
+# Baseclass representing books that are archived on the user's Kobo device.
+class ArchivedBook(Base):
+    __tablename__ = 'archived_book'
+
+    id = Column(Integer, primary_key=True)
+    user_id = Column(Integer, ForeignKey('user.id'))
+    book_id = Column(Integer)
+    is_archived = Column(Boolean, unique=False)
+    last_modified = Column(DateTime, default=datetime.datetime.utcnow)
+
 
 # The Kobo ReadingState API keeps track of 4 timestamped entities:
 #   ReadingState, StatusInfo, Statistics, CurrentBookmark
@@ -417,6 +430,8 @@ def migrate_Database(session):
         KoboBookmark.__table__.create(bind=engine)
     if not engine.dialect.has_table(engine.connect(), "kobo_statistics"):
         KoboStatistics.__table__.create(bind=engine)
+    if not engine.dialect.has_table(engine.connect(), "archived_book"):
+        ArchivedBook.__table__.create(bind=engine)
     if not engine.dialect.has_table(engine.connect(), "registration"):
         ReadBook.__table__.create(bind=engine)
         conn = engine.connect()

cps/web.py

@@ -45,10 +45,10 @@ from werkzeug.security import generate_password_hash, check_password_hash
 from . import constants, logger, isoLanguages, services, worker
 from . import searched_ids, lm, babel, db, ub, config, get_locale, app
 from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download
-from .helper import common_filters, get_search_results, fill_indexpage, speaking_language, check_valid_domain, \
+from .helper import common_filters, get_search_results, fill_indexpage, fill_indexpage_with_archived_books, \
-        order_authors, get_typeahead, render_task_status, json_serial, get_cc_columns, \
+    speaking_language, check_valid_domain, order_authors, get_typeahead, render_task_status, json_serial, \
-        get_book_cover, get_download_link, send_mail, generate_random_password, send_registration_mail, \
+    get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \
-        check_send_to_kindle, check_read_formats, lcase, tags_filters, reset_password
+    send_registration_mail, check_send_to_kindle, check_read_formats, lcase, tags_filters, reset_password
 from .pagination import Pagination
 from .redirect import redirect_back
 
@@ -347,6 +347,22 @@ def toggle_read(book_id):
     return ""
 
 
+@web.route("/ajax/togglearchived/<int:book_id>", methods=['POST'])
+@login_required
+def toggle_archived(book_id):
+    archived_book = ub.session.query(ub.ArchivedBook).filter(and_(ub.ArchivedBook.user_id == int(current_user.id),
+                                                                  ub.ArchivedBook.book_id == book_id)).first()
+    if archived_book:
+        archived_book.is_archived = not archived_book.is_archived
+        archived_book.last_modified = datetime.datetime.utcnow()
+    else:
+        archived_book = ub.ArchivedBook(user_id=current_user.id, book_id=book_id)
+        archived_book.is_archived = True
+    ub.session.merge(archived_book)
+    ub.session.commit()
+    return ""
+
+
 '''
 @web.route("/ajax/getcomic/<int:book_id>/<book_format>/<int:page>")
 @login_required
@@ -542,6 +558,8 @@ def books_list(data, sort, book_id, page):
         return render_category_books(page, book_id, order)
     elif data == "language":
         return render_language_books(page, book_id, order)
+    elif data == "archived":
+        return render_archived_books(page, order)
     else:
         entries, random, pagination = fill_indexpage(page, db.Books, True, order)
         return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
@@ -1018,6 +1036,26 @@ def render_read_books(page, are_read, as_xml=False, order=None, *args, **kwargs)
                                      title=name, page=pagename)
 
 
+def render_archived_books(page, order):
+    order = order or []
+    archived_books = (
+        ub.session.query(ub.ArchivedBook)
+        .filter(ub.ArchivedBook.user_id == int(current_user.id))
+        .filter(ub.ArchivedBook.is_archived == True)
+        .all()
+    )
+    archived_book_ids = [archived_book.book_id for archived_book in archived_books]
+
+    archived_filter = db.Books.id.in_(archived_book_ids)
+
+    entries, random, pagination = fill_indexpage_with_archived_books(page, db.Books, archived_filter, order,
+                                                                     allow_show_archived=True)
+
+    name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')'
+    pagename = "archived"
+    return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
+                                 title=name, page=pagename)
+
 # ################################### Download/Send ##################################################################
 
 
@@ -1435,7 +1473,8 @@ def read_book(book_id, book_format):
 @web.route("/book/<int:book_id>")
 @login_required_if_no_ano
 def show_book(book_id):
-    entries = db.session.query(db.Books).filter(db.Books.id == book_id).filter(common_filters()).first()
+    entries = db.session.query(db.Books).filter(and_(db.Books.id == book_id,
+                                                     common_filters(allow_show_archived=True))).first()
     if entries:
         for index in range(0, len(entries.languages)):
             try:
@@ -1464,8 +1503,14 @@ def show_book(book_id):
                     log.error("Custom Column No.%d is not existing in calibre database", config.config_read_column)
                     have_read = None
 
+            archived_book = ub.session.query(ub.ArchivedBook).\
+                filter(and_(ub.ArchivedBook.user_id == int(current_user.id),
+                            ub.ArchivedBook.book_id == book_id)).first()
+            is_archived = archived_book and archived_book.is_archived
+
         else:
             have_read = None
+            is_archived = None
 
         entries.tags = sort(entries.tags, key=lambda tag: tag.name)
 
@@ -1481,7 +1526,7 @@ def show_book(book_id):
 
         return render_title_template('detail.html', entry=entries, audioentries=audioentries, cc=cc,
                                      is_xhr=request.headers.get('X-Requested-With')=='XMLHttpRequest', title=entries.title, books_shelfs=book_in_shelfs,
-                                     have_read=have_read, kindle_list=kindle_list, reader_list=reader_list, page="book")
+                                     have_read=have_read, is_archived=is_archived, kindle_list=kindle_list, reader_list=reader_list, page="book")
     else:
         log.debug(u"Error opening eBook. File does not exist or file is not accessible:")
         flash(_(u"Error opening eBook. File does not exist or file is not accessible:"), category="error")