Merge remote-tracking branch 'kobo_sync/kobo' into Develop

# Conflicts:
#	cps.py
#	cps/kobo.py
#	cps/kobo_auth.py
#	cps/ub.py

cps.py

@@ -42,6 +42,7 @@ from cps.admin import admi
 from cps.gdrive import gdrive
 from cps.editbooks import editbook
 from cps.kobo import kobo
+from cps.kobo_auth import kobo_auth
 
 try:
     from cps.oauth_bb import oauth
@@ -61,6 +62,7 @@ def main():
     app.register_blueprint(gdrive)
     app.register_blueprint(editbook)
     app.register_blueprint(kobo)
+    app.register_blueprint(kobo_auth)
     if oauth_available:
         app.register_blueprint(oauth)
     success = web_server.start()

cps/kobo.py

@@ -25,15 +25,20 @@ from datetime import datetime
 from time import gmtime, strftime
 
 from jsonschema import validate, exceptions
-from flask import Blueprint, request, make_response, jsonify, json
+from flask import Blueprint, request, make_response, jsonify, json, current_app, url_for
+
 from flask_login import login_required
-from sqlalchemy import func, or_
+from sqlalchemy import func
 
 from . import config, logger, kobo_auth, db, helper
 from .web import download_required
 
-kobo = Blueprint("kobo", __name__)
+#TODO: Test more formats :) .
+KOBO_SUPPORTED_FORMATS = {"KEPUB"}
+
+kobo = Blueprint("kobo", __name__, url_prefix="/kobo/<auth_token>")
 kobo_auth.disable_failed_auth_redirect_for_blueprint(kobo)
+kobo_auth.register_url_value_preprocessor(kobo)
 
 log = logger.create()
 
@@ -166,9 +171,10 @@ def HandleSyncRequest():
     # It looks like it's treating the db.Books.last_modified field as a string and may fail
     # the comparison because of the +00:00 suffix.
     changed_entries = (
-        db.session.query(db.Books).join(db.Data)
+        db.session.query(db.Books)
-        .filter(func.datetime(db.Books.last_modified) != sync_token.books_last_modified)
+        .join(db.Data)
-        .filter(or_(db.Data.format == 'KEPUB', db.Data.format == 'EPUB'))
+        .filter(func.datetime(db.Books.last_modified) > sync_token.books_last_modified)
+        .filter(db.Data.format.in_(KOBO_SUPPORTED_FORMATS))
         .all()
     )
     for book in changed_entries:
@@ -217,10 +223,11 @@ def HandleMetadataRequest(book_uuid):
 
 
 def get_download_url_for_book(book, book_format):
-    return "{url_base}/download/{book_id}/{book_format}".format(
+    return url_for(
-        url_base=get_base_url(), # request.environ['werkzeug.request'].base_url,
+        "web.download_link",
         book_id=book.id,
-        book_format="kepub",
+        book_format=book_format.lower(),
+        _external=True,
     )
 
 
@@ -273,14 +280,13 @@ def get_series(book):
 
 
 def get_metadata(book):
-    ALLOWED_FORMATS = {"KEPUB", "EPUB"}
     download_urls = []
 
     for book_data in book.data:
-        if book_data.format in ALLOWED_FORMATS:
+        if book_data.format in KOBO_SUPPORTED_FORMATS:
             download_urls.append(
                 {
-                    "Format": "KEPUB",
+                    "Format": book_data.format,
                     "Size": book_data.uncompressed_size,
                     "Url": get_download_url_for_book(book, book_data.format),
                     # "DrmType": "None", # Not required
@@ -354,9 +360,14 @@ def HandleCoverImageRequest(book_uuid, horizontal, vertical, jpeg_quality, monoc
     return book_cover
 
 
+@kobo.route("")
+def TopLevelEndpoint():
+    return make_response(jsonify({}))
+
+
 @kobo.route("/v1/user/profile")
 @kobo.route("/v1/user/loyalty/benefits")
-@kobo.route("/v1/analytics/gettests", methods=["GET", "POST"])
+@kobo.route("/v1/analytics/gettests/", methods=["GET", "POST"])
 @kobo.route("/v1/user/wishlist")
 @kobo.route("/v1/user/<dummy>")
 @kobo.route("/v1/user/recommendations")
@@ -386,12 +397,11 @@ def HandleAuthRequest():
     return response
 
 
-def get_base_url():
-    return "{root}:{port}".format(root=request.url_root[:-1], port=str(config.config_port))
-
 @kobo.route("/v1/initialization")
 def HandleInitRequest():
-    resources = NATIVE_KOBO_RESOURCES(calibre_web_url=get_base_url())
+    resources = NATIVE_KOBO_RESOURCES(
+        calibre_web_url=url_for("web.index", _external=True).strip("/")
+    )
     response = make_response(jsonify({"Resources": resources}))
     response.headers["x-kobo-apitoken"] = "e30="
     return response

cps/kobo_auth.py

@@ -29,7 +29,6 @@ which serves the following response:
     <script type='text/javascript'>location.href='kobo://UserAuthenticated?userId=<redacted>&userKey<redacted>&email=<redacted>&returnUrl=https%3a%2f%2fwww.kobo.com';</script>.
 And triggers the insertion of a userKey into the device's User table.
 
-IMPORTANT SECURITY CAUTION:
 Together, the device's DeviceId and UserKey act as an *irrevocable* authentication 
 token to most (if not all) Kobo APIs. In fact, in most cases only the UserKey is
 required to authorize the API call.
@@ -48,55 +47,80 @@ v1/auth/device endpoint with the secret UserKey and the device's DeviceId.
 * The book download endpoint passes an auth token as a URL param instead of a header.
 
 Our implementation:
-For now, we rely on the official Kobo store's UserKey for authentication.
+We pretty much ignore all of the above. To authenticate the user, we generate a random
-Once authenticated, we set the login cookie on the response that will be sent back for
+and unique token that they append to the CalibreWeb Url when setting up the api_store
-the duration of the session to authorize subsequent API calls.
+setting on the device.
-Ideally we'd only perform UserKey-based authentication for the v1/initialization or the
+Thus, every request from the device to the api_store will hit CalibreWeb with the
-v1/device/auth call, however sessions don't always start with those calls.
+auth_token in the url (e.g: https://mylibrary.com/<auth_token>/v1/library/sync).
-
+In addition, once authenticated we also set the login cookie on the response that will
-Because of the irrevocable power granted by the key, we only ever store and compare a
+be sent back for the duration of the session to authorize subsequent API calls (in
-hash of the key. To obtain their UserKey, a user can query the user table from the
+particular calls to non-Kobo specific endpoints such as the CalibreWeb book download).
-.kobo/KoboReader.sqlite database found on their device.
-This isn't exactly user friendly however.
-
-Some possible alternatives that require more research:
- * Instead of having users query the device database to find out their UserKey, we could
- provide a list of recent Kobo sync attempts in the calibre-web UI for users to
- authenticate sync attempts (e.g: 'this was me' button).
- * We may be able to craft a sign-in flow with a redirect back to the CalibreWeb
- server containing the KoboStore's UserKey (if the same as the devices?). 
- * Can we create our own UserKey instead of relying on the real store's userkey?
-  (Maybe using something like location.href=kobo://UserAuthenticated?userId=...?)
 """
 
-from functools import wraps
+from binascii import hexlify
-from flask import request, make_response
+from datetime import datetime
-from flask_login import login_user
+from os import urandom
-from werkzeug.security import check_password_hash
 
-from . import logger, ub, lm
+from flask import g, Blueprint, url_for
+from flask_login import login_user, current_user, login_required
+from flask_babel import gettext as _
 
-USER_KEY_HEADER = "x-kobo-userkey"
+from . import logger, ub, lm
-USER_KEY_URL_PARAM = "kobo_userkey"
+from .web import render_title_template
 
 log = logger.create()
 
 
+def register_url_value_preprocessor(kobo):
+    @kobo.url_value_preprocessor
+    def pop_auth_token(endpoint, values):
+        g.auth_token = values.pop("auth_token")
+
+
 def disable_failed_auth_redirect_for_blueprint(bp):
     lm.blueprint_login_views[bp.name] = None
 
 
 @lm.request_loader
 def load_user_from_kobo_request(request):
-    user_key = request.headers.get(USER_KEY_HEADER)
+    if "auth_token" in g:
-    if user_key:
+        auth_token = g.get("auth_token")
-        for user in (
+        user = (
-            ub.session.query(ub.User).filter(ub.User.kobo_user_key_hash != "").all()
+            ub.session.query(ub.User)
-        ):
+            .join(ub.RemoteAuthToken)
-            if check_password_hash(str(user.kobo_user_key_hash), user_key):
+            .filter(ub.RemoteAuthToken.auth_token == auth_token)
-                # The Kobo device won't preserve the cookie accross sessions, even if we
+            .first()
-                # were to set remember_me=true.
+        )
-                login_user(user)
+        if user is not None:
-                return user
+            login_user(user)
-    log.info("Received Kobo request without a recognizable UserKey.")
+            return user
+    log.info("Received Kobo request without a recognizable auth token.")
     return None
+
+
+kobo_auth = Blueprint("kobo_auth", __name__, url_prefix="/kobo_auth")
+
+
+@kobo_auth.route("/generate_auth_token")
+@login_required
+def generate_auth_token():
+    # Invalidate any prevously generated Kobo Auth token for this user.
+    ub.session.query(ub.RemoteAuthToken).filter(
+        ub.RemoteAuthToken.user_id == current_user.id
+    ).delete()
+
+    auth_token = ub.RemoteAuthToken()
+    auth_token.user_id = current_user.id
+    auth_token.expiration = datetime.max
+    auth_token.auth_token = (hexlify(urandom(16))).decode("utf-8")
+
+    ub.session.add(auth_token)
+    ub.session.commit()
+
+    return render_title_template(
+        "generate_kobo_auth_url.html",
+        title=_(u"Kobo Set-up"),
+        kobo_auth_url=url_for(
+            "kobo.TopLevelEndpoint", auth_token=auth_token.auth_token, _external=True
+        ),
+    )

cps/templates/generate_kobo_auth_url.html

@@ -0,0 +1,15 @@
+{% extends "layout.html" %}
+{% block body %}
+<div class="well">
+  <h2 style="margin-top: 0">{{_('Generate Kobo Auth URL')}}</h2>
+  <p>
+    {{_('Open the .kobo/Kobo eReader.conf file in a text editor and add (or edit):')}}</a>.
+  </p>
+  <p>
+    {{_('api_endpoint=')}}{{kobo_auth_url}}</a>
+  </p>
+  <p>
+    {{_('Please note that every visit to this current page invalidates any previously generated Authentication url for this user.')}}</a>.
+  </p>
+</div>
+{% endblock %}

cps/ub.py

@@ -173,7 +173,6 @@ class User(UserBase, Base):
     role = Column(SmallInteger, default=constants.ROLE_USER)
     password = Column(String)
     kindle_mail = Column(String(120), default="")
-    kobo_user_key_hash = Column(String, unique=True, default="")
     shelf = relationship('Shelf', backref='user', lazy='dynamic', order_by='Shelf.name')
     downloads = relationship('Downloads', backref='user', lazy='dynamic')
     locale = Column(String(2), default="en")
@@ -308,7 +307,7 @@ class RemoteAuthToken(Base):
     __tablename__ = 'remote_auth_token'
 
     id = Column(Integer, primary_key=True)
-    auth_token = Column(String(8), unique=True)
+    auth_token = Column(String, unique=True)
     user_id = Column(Integer, ForeignKey('user.id'))
     verified = Column(Boolean, default=False)
     expiration = Column(DateTime)
@@ -376,12 +375,6 @@ def migrate_Database(session):
     except exc.OperationalError:
         conn = engine.connect()
         conn.execute("ALTER TABLE user ADD column `mature_content` INTEGER DEFAULT 1")
-    try:
-        session.query(exists().where(User.kobo_user_key_hash)).scalar()
-    except exc.OperationalError:  # Database is not compatible, some rows are missing
-        conn = engine.connect()
-        conn.execute("ALTER TABLE user ADD column `kobo_user_key_hash` VARCHAR")
-        session.commit()
     if session.query(User).filter(User.role.op('&')(constants.ROLE_ANONYMOUS) == constants.ROLE_ANONYMOUS).first() is None:
         create_anonymous_user(session)
     try:
@@ -397,7 +390,6 @@ def migrate_Database(session):
                             "role SMALLINT,"
                             "password VARCHAR,"
                             "kindle_mail VARCHAR(120),"
-                            "kobo_user_key_hash VARCHAR,"
                             "locale VARCHAR(2),"
                             "sidebar_view INTEGER,"
                             "default_language VARCHAR(3),"
@@ -405,9 +397,9 @@ def migrate_Database(session):
                             "UNIQUE (nickname),"
                             "UNIQUE (email),"
                             "CHECK (mature_content IN (0, 1)))")
-        conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,kobo_user_key_hash, locale,"
+        conn.execute("INSERT INTO user_id(id, nickname, email, role, password, kindle_mail,locale,"
                         "sidebar_view, default_language, mature_content) "
-                     "SELECT id, nickname, email, role, password, kindle_mail, kobo_user_key_hash, locale,"
+                     "SELECT id, nickname, email, role, password, kindle_mail, locale,"
                         "sidebar_view, default_language, mature_content FROM user")
         # delete old user table and rename new user_id table to user:
         conn.execute("DROP TABLE user")