diff --git a/README.md b/README.md index d81e7ece..a3e05cd3 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Calibre-Web is a web app providing a clean interface for browsing, reading and d ## Quick start -1. Install dependencies by running `pip install --target vendor -r requirements.txt`. +1. Install dependencies by running `pip3 install --target vendor -r requirements.txt`. 2. Execute the command: `python cps.py` (or `nohup python cps.py` - recommended if you want to exit the terminal window) 3. Point your browser to `http://localhost:8083` or `http://localhost:8083/opds` for the OPDS catalog 4. Set `Location of Calibre database` to the path of the folder where your Calibre library (metadata.db) lives, push "submit" button\ @@ -46,7 +46,7 @@ Please note that running the above install command can fail on some versions of ## Requirements -Python 2.7+, python 3.x+ +python 3.x+, (Python 2.7+) Optionally, to enable on-the-fly conversion from one ebook format to another when using the send-to-kindle feature, or during editing of ebooks metadata: diff --git a/cps/admin.py b/cps/admin.py index ab3d0f22..7467ec45 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -532,6 +532,9 @@ def _configuration_update_helper(): _config_checkbox_int("config_uploading") _config_checkbox_int("config_anonbrowse") _config_checkbox_int("config_public_reg") + _config_checkbox_int("config_kobo_sync") + _config_checkbox_int("config_kobo_proxy") + _config_int("config_ebookconverter") _config_string("config_calibre") diff --git a/cps/config_sql.py b/cps/config_sql.py index 97c05067..bbdda872 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -68,6 +68,7 @@ class _Settings(_Base): config_anonbrowse = Column(SmallInteger, default=0) config_public_reg = Column(SmallInteger, default=0) config_remote_login = Column(Boolean, default=False) + config_kobo_sync = Column(Boolean, default=False) config_default_role = Column(SmallInteger, default=0) config_default_show = Column(SmallInteger, default=38911) @@ -89,7 +90,8 @@ class _Settings(_Base): config_login_type = Column(Integer, default=0) - # config_oauth_provider = Column(Integer) + config_kobo_proxy = Column(Boolean, default=False) + config_ldap_provider_url = Column(String, default='localhost') config_ldap_port = Column(SmallInteger, default=389) diff --git a/cps/constants.py b/cps/constants.py index e0d56922..3023c67b 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -126,7 +126,7 @@ def selected_roles(dictionary): BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, ' 'series_id, languages') -STABLE_VERSION = {'version': '0.6.5 Beta'} +STABLE_VERSION = {'version': '0.6.6 Beta'} NIGHTLY_VERSION = {} NIGHTLY_VERSION[0] = '$Format:%H$' diff --git a/cps/kobo.py b/cps/kobo.py index 3c19b66e..d06b924d 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -19,7 +19,6 @@ import sys import uuid -from datetime import datetime from time import gmtime, strftime try: @@ -32,9 +31,10 @@ from flask import ( request, make_response, jsonify, - json, + current_app, url_for, redirect, + abort ) from flask_login import login_required, current_user from werkzeug.datastructures import Headers @@ -46,7 +46,7 @@ from . import config, logger, kobo_auth, db, helper, ub from .services import SyncToken as SyncToken from .web import download_required -KOBO_FORMATS = {"KEPUB": ["KEPUB"], "EPUB": ["EPUB", "EPUB3"]} +KOBO_FORMATS = {"KEPUB": ["KEPUB"], "EPUB": ["EPUB3", "EPUB"]} KOBO_STOREAPI_URL = "https://storeapi.kobo.com" kobo = Blueprint("kobo", __name__, url_prefix="/kobo/") @@ -74,30 +74,33 @@ CONNECTION_SPECIFIC_HEADERS = [ def redirect_or_proxy_request(): - if request.method == "GET": - return redirect(get_store_url_for_current_request(), 307) - if request.method == "DELETE": - return make_response(jsonify({})) - else: - # The Kobo device turns other request types into GET requests on redirects, so we instead proxy to the Kobo store ourselves. - outgoing_headers = Headers(request.headers) - outgoing_headers.remove("Host") - store_response = requests.request( - method=request.method, - url=get_store_url_for_current_request(), - headers=outgoing_headers, - data=request.get_data(), - allow_redirects=False, - ) - - response_headers = store_response.headers - for header_key in CONNECTION_SPECIFIC_HEADERS: - response_headers.pop(header_key, default=None) + if config.config_kobo_proxy: + if request.method == "GET": + return redirect(get_store_url_for_current_request(), 307) + if request.method == "DELETE": + log.info('Delete Book') + return make_response(jsonify({})) + else: + # The Kobo device turns other request types into GET requests on redirects, so we instead proxy to the Kobo store ourselves. + outgoing_headers = Headers(request.headers) + outgoing_headers.remove("Host") + store_response = requests.request( + method=request.method, + url=get_store_url_for_current_request(), + headers=outgoing_headers, + data=request.get_data(), + allow_redirects=False, + ) - return make_response( - store_response.content, store_response.status_code, response_headers.items() - ) + response_headers = store_response.headers + for header_key in CONNECTION_SPECIFIC_HEADERS: + response_headers.pop(header_key, default=None) + return make_response( + store_response.content, store_response.status_code, response_headers.items() + ) + else: + return make_response(jsonify({})) @kobo.route("/v1/library/sync") @login_required @@ -105,6 +108,8 @@ def redirect_or_proxy_request(): def HandleSyncRequest(): sync_token = SyncToken.SyncToken.from_headers(request.headers) log.info("Kobo library sync request received.") + if not current_app.wsgi_app.is_proxied: + log.debug('Kobo: Received unproxied request, changed request port to server port') # TODO: Limit the number of books return per sync call, and rely on the sync-continuatation header # instead so that the device triggers another sync. @@ -149,6 +154,7 @@ def HandleSyncRequest(): .filter(db.Data.format.in_(KOBO_FORMATS)) .all() ) + for book in changed_entries: entitlement = { "BookEntitlement": create_book_entitlement(book, archived=(book.id in archived_book_ids)), @@ -169,7 +175,11 @@ def HandleSyncRequest(): sync_token.books_last_modified = new_books_last_modified sync_token.archive_last_modified = new_archived_last_modified - return generate_sync_response(request, sync_token, entitlements) + if config.config_kobo_proxy: + return generate_sync_response(request, sync_token, entitlements) + + return make_response(jsonify(entitlements)) + # Missing feature: Detect server-side book deletions. def generate_sync_response(request, sync_token, entitlements): @@ -210,6 +220,8 @@ def generate_sync_response(request, sync_token, entitlements): @login_required @download_required def HandleMetadataRequest(book_uuid): + if not current_app.wsgi_app.is_proxied: + log.debug('Kobo: Received unproxied request, changed request port to server port') log.info("Kobo library metadata request received for book %s" % book_uuid) book = db.session.query(db.Books).filter(db.Books.uuid == book_uuid).first() if not book or not book.data: @@ -221,12 +233,21 @@ def HandleMetadataRequest(book_uuid): def get_download_url_for_book(book, book_format): - return url_for( - "web.download_link", - book_id=book.id, - book_format=book_format.lower(), - _external=True, - ) + if not current_app.wsgi_app.is_proxied: + return "{url_scheme}://{url_base}:{url_port}/download/{book_id}/{book_format}".format( + url_scheme=request.environ['wsgi.url_scheme'], + url_base=request.environ['SERVER_NAME'], + url_port=config.config_port, + book_id=book.id, + book_format=book_format.lower() + ) + else: + return url_for( + "web.download_link", + book_id=book.id, + book_format=book_format.lower(), + _external=True, + ) def create_book_entitlement(book, archived): @@ -278,11 +299,11 @@ def get_series(book): def get_metadata(book): download_urls = [] - for book_data in book.data: if book_data.format not in KOBO_FORMATS: continue for kobo_format in KOBO_FORMATS[book_data.format]: + # log.debug('Id: %s, Format: %s' % (book.id, kobo_format)) download_urls.append( { "Format": kobo_format, @@ -356,7 +377,10 @@ def HandleCoverImageRequest(book_uuid): book_uuid, use_generic_cover_on_failure=False ) if not book_cover: - return redirect(get_store_url_for_current_request(), 307) + if config.config_kobo_proxy: + return redirect(get_store_url_for_current_request(), 307) + else: + abort(404) return book_cover @@ -397,39 +421,196 @@ def HandleBookDeletionRequest(book_uuid): @kobo.route("/v1/library/tags/", methods=["POST"]) @kobo.route("/v1/library/tags/", methods=["DELETE"]) def HandleUnimplementedRequest(book_uuid=None, shelf_name=None, tag_id=None): + log.debug("Alternative Request received:") return redirect_or_proxy_request() +# TODO: Implement the following routes +@kobo.route("/v1/user/loyalty/", methods=["GET", "POST"]) +@kobo.route("/v1/user/profile", methods=["GET", "POST"]) +@kobo.route("/v1/user/wishlist", methods=["GET", "POST"]) +@kobo.route("/v1/user/recommendations", methods=["GET", "POST"]) +@kobo.route("/v1/analytics/", methods=["GET", "POST"]) +def HandleUserRequest(dummy=None): + log.debug("Unimplemented Request received: %s", request.base_url) + return redirect_or_proxy_request() + @kobo.app_errorhandler(404) def handle_404(err): # This handler acts as a catch-all for endpoints that we don't have an interest in # implementing (e.g: v1/analytics/gettests, v1/user/recommendations, etc) + log.debug("Unknown Request received: %s", request.base_url) return redirect_or_proxy_request() @kobo.route("/v1/initialization") @login_required def HandleInitRequest(): - outgoing_headers = Headers(request.headers) - outgoing_headers.remove("Host") - store_response = requests.request( - method=request.method, - url=get_store_url_for_current_request(), - headers=outgoing_headers, - data=request.get_data(), - ) + if not current_app.wsgi_app.is_proxied: + log.debug('Kobo: Received unproxied request, changed request port to server port') + calibre_web_url = "{url_scheme}://{url_base}:{url_port}".format( + url_scheme=request.environ['wsgi.url_scheme'], + url_base=request.environ['SERVER_NAME'], + url_port=config.config_port + ) + else: + calibre_web_url = url_for("web.index", _external=True).strip("/") + if config.config_kobo_proxy: + outgoing_headers = Headers(request.headers) + outgoing_headers.remove("Host") + store_response = requests.request( + method=request.method, + url=get_store_url_for_current_request(), + headers=outgoing_headers, + data=request.get_data(), + ) - store_response_json = store_response.json() - if "Resources" in store_response_json: - kobo_resources = store_response_json["Resources"] + store_response_json = store_response.json() + if "Resources" in store_response_json: + kobo_resources = store_response_json["Resources"] + # calibre_web_url = url_for("web.index", _external=True).strip("/") + kobo_resources["image_host"] = calibre_web_url + kobo_resources["image_url_quality_template"] = unquote(calibre_web_url + url_for("kobo.HandleCoverImageRequest", + auth_token = kobo_auth.get_auth_token(), + book_uuid="{ImageId}")) + kobo_resources["image_url_template"] = unquote(calibre_web_url + url_for("kobo.HandleCoverImageRequest", + auth_token = kobo_auth.get_auth_token(), + book_uuid="{ImageId}")) + + return make_response(store_response_json, store_response.status_code) + else: + resources = NATIVE_KOBO_RESOURCES(calibre_web_url) + response = make_response(jsonify({"Resources": resources})) + response.headers["x-kobo-apitoken"] = "e30=" + return response - calibre_web_url = url_for("web.index", _external=True).strip("/") - kobo_resources["image_host"] = calibre_web_url - kobo_resources["image_url_quality_template"] = unquote(url_for("kobo.HandleCoverImageRequest", _external=True, - auth_token = kobo_auth.get_auth_token(), - book_uuid="{ImageId}")) - kobo_resources["image_url_template"] = unquote(url_for("kobo.HandleCoverImageRequest", _external=True, - auth_token = kobo_auth.get_auth_token(), - book_uuid="{ImageId}")) - - return make_response(store_response_json, store_response.status_code) +def NATIVE_KOBO_RESOURCES(calibre_web_url): + return { + "account_page": "https://secure.kobobooks.com/profile", + "account_page_rakuten": "https://my.rakuten.co.jp/", + "add_entitlement": "https://storeapi.kobo.com/v1/library/{RevisionIds}", + "affiliaterequest": "https://storeapi.kobo.com/v1/affiliate", + "audiobook_subscription_orange_deal_inclusion_url": "https://authorize.kobo.com/inclusion", + "authorproduct_recommendations": "https://storeapi.kobo.com/v1/products/books/authors/recommendations", + "autocomplete": "https://storeapi.kobo.com/v1/products/autocomplete", + "blackstone_header": {"key": "x-amz-request-payer", "value": "requester"}, + "book": "https://storeapi.kobo.com/v1/products/books/{ProductId}", + "book_detail_page": "https://store.kobobooks.com/{culture}/ebook/{slug}", + "book_detail_page_rakuten": "http://books.rakuten.co.jp/rk/{crossrevisionid}", + "book_landing_page": "https://store.kobobooks.com/ebooks", + "book_subscription": "https://storeapi.kobo.com/v1/products/books/subscriptions", + "categories": "https://storeapi.kobo.com/v1/categories", + "categories_page": "https://store.kobobooks.com/ebooks/categories", + "category": "https://storeapi.kobo.com/v1/categories/{CategoryId}", + "category_featured_lists": "https://storeapi.kobo.com/v1/categories/{CategoryId}/featured", + "category_products": "https://storeapi.kobo.com/v1/categories/{CategoryId}/products", + "checkout_borrowed_book": "https://storeapi.kobo.com/v1/library/borrow", + "configuration_data": "https://storeapi.kobo.com/v1/configuration", + "content_access_book": "https://storeapi.kobo.com/v1/products/books/{ProductId}/access", + "customer_care_live_chat": "https://v2.zopim.com/widget/livechat.html?key=Y6gwUmnu4OATxN3Tli4Av9bYN319BTdO", + "daily_deal": "https://storeapi.kobo.com/v1/products/dailydeal", + "deals": "https://storeapi.kobo.com/v1/deals", + "delete_entitlement": "https://storeapi.kobo.com/v1/library/{Ids}", + "delete_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "delete_tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/items/delete", + "device_auth": "https://storeapi.kobo.com/v1/auth/device", + "device_refresh": "https://storeapi.kobo.com/v1/auth/refresh", + "dictionary_host": "https://kbdownload1-a.akamaihd.net", + "discovery_host": "https://discovery.kobobooks.com", + "eula_page": "https://www.kobo.com/termsofuse?style=onestore", + "exchange_auth": "https://storeapi.kobo.com/v1/auth/exchange", + "external_book": "https://storeapi.kobo.com/v1/products/books/external/{Ids}", + "facebook_sso_page": "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/", + "featured_list": "https://storeapi.kobo.com/v1/products/featured/{FeaturedListId}", + "featured_lists": "https://storeapi.kobo.com/v1/products/featured", + "free_books_page": { + "EN": "https://www.kobo.com/{region}/{language}/p/free-ebooks", + "FR": "https://www.kobo.com/{region}/{language}/p/livres-gratuits", + "IT": "https://www.kobo.com/{region}/{language}/p/libri-gratuiti", + "NL": "https://www.kobo.com/{region}/{language}/List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg", + "PT": "https://www.kobo.com/{region}/{language}/p/livros-gratis", + }, + "fte_feedback": "https://storeapi.kobo.com/v1/products/ftefeedback", + "get_tests_request": "https://storeapi.kobo.com/v1/analytics/gettests", + "giftcard_epd_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem-ereader", + "giftcard_redeem_url": "https://www.kobo.com/{storefront}/{language}/redeem", + "help_page": "http://www.kobo.com/help", + "image_host": calibre_web_url, + "image_url_quality_template": unquote(calibre_web_url + url_for("kobo.HandleCoverImageRequest", + auth_token = kobo_auth.get_auth_token(), + book_uuid="{ImageId}")), + "image_url_template": unquote(calibre_web_url + url_for("kobo.HandleCoverImageRequest", + auth_token = kobo_auth.get_auth_token(), + book_uuid="{ImageId}")), + "kobo_audiobooks_enabled": "False", + "kobo_audiobooks_orange_deal_enabled": "False", + "kobo_audiobooks_subscriptions_enabled": "False", + "kobo_nativeborrow_enabled": "True", + "kobo_onestorelibrary_enabled": "False", + "kobo_redeem_enabled": "True", + "kobo_shelfie_enabled": "False", + "kobo_subscriptions_enabled": "False", + "kobo_superpoints_enabled": "False", + "kobo_wishlist_enabled": "True", + "library_book": "https://storeapi.kobo.com/v1/user/library/books/{LibraryItemId}", + "library_items": "https://storeapi.kobo.com/v1/user/library", + "library_metadata": "https://storeapi.kobo.com/v1/library/{Ids}/metadata", + "library_prices": "https://storeapi.kobo.com/v1/user/library/previews/prices", + "library_stack": "https://storeapi.kobo.com/v1/user/library/stacks/{LibraryItemId}", + "library_sync": "https://storeapi.kobo.com/v1/library/sync", + "love_dashboard_page": "https://store.kobobooks.com/{culture}/kobosuperpoints", + "love_points_redemption_page": "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}", + "magazine_landing_page": "https://store.kobobooks.com/emagazines", + "notifications_registration_issue": "https://storeapi.kobo.com/v1/notifications/registration", + "oauth_host": "https://oauth.kobo.com", + "overdrive_account": "https://auth.overdrive.com/account", + "overdrive_library": "https://{libraryKey}.auth.overdrive.com/library", + "overdrive_library_finder_host": "https://libraryfinder.api.overdrive.com", + "overdrive_thunder_host": "https://thunder.api.overdrive.com", + "password_retrieval_page": "https://www.kobobooks.com/passwordretrieval.html", + "post_analytics_event": "https://storeapi.kobo.com/v1/analytics/event", + "privacy_page": "https://www.kobo.com/privacypolicy?style=onestore", + "product_nextread": "https://storeapi.kobo.com/v1/products/{ProductIds}/nextread", + "product_prices": "https://storeapi.kobo.com/v1/products/{ProductIds}/prices", + "product_recommendations": "https://storeapi.kobo.com/v1/products/{ProductId}/recommendations", + "product_reviews": "https://storeapi.kobo.com/v1/products/{ProductIds}/reviews", + "products": "https://storeapi.kobo.com/v1/products", + "provider_external_sign_in_page": "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/", + "purchase_buy": "https://www.kobo.com/checkout/createpurchase/", + "purchase_buy_templated": "https://www.kobo.com/{culture}/checkout/createpurchase/{ProductId}", + "quickbuy_checkout": "https://storeapi.kobo.com/v1/store/quickbuy/{PurchaseId}/checkout", + "quickbuy_create": "https://storeapi.kobo.com/v1/store/quickbuy/purchase", + "rating": "https://storeapi.kobo.com/v1/products/{ProductId}/rating/{Rating}", + "reading_state": "https://storeapi.kobo.com/v1/library/{Ids}/state", + "redeem_interstitial_page": "https://store.kobobooks.com", + "registration_page": "https://authorize.kobo.com/signup?returnUrl=http://store.kobobooks.com/", + "related_items": "https://storeapi.kobo.com/v1/products/{Id}/related", + "remaining_book_series": "https://storeapi.kobo.com/v1/products/books/series/{SeriesId}", + "rename_tag": "https://storeapi.kobo.com/v1/library/tags/{TagId}", + "review": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}", + "review_sentiment": "https://storeapi.kobo.com/v1/products/reviews/{ReviewId}/sentiment/{Sentiment}", + "shelfie_recommendations": "https://storeapi.kobo.com/v1/user/recommendations/shelfie", + "sign_in_page": "https://authorize.kobo.com/signin?returnUrl=http://store.kobobooks.com/", + "social_authorization_host": "https://social.kobobooks.com:8443", + "social_host": "https://social.kobobooks.com", + "stacks_host_productId": "https://store.kobobooks.com/collections/byproductid/", + "store_home": "www.kobo.com/{region}/{language}", + "store_host": "store.kobobooks.com", + "store_newreleases": "https://store.kobobooks.com/{culture}/List/new-releases/961XUjtsU0qxkFItWOutGA", + "store_search": "https://store.kobobooks.com/{culture}/Search?Query={query}", + "store_top50": "https://store.kobobooks.com/{culture}/ebooks/Top", + "tag_items": "https://storeapi.kobo.com/v1/library/tags/{TagId}/Items", + "tags": "https://storeapi.kobo.com/v1/library/tags", + "taste_profile": "https://storeapi.kobo.com/v1/products/tasteprofile", + "update_accessibility_to_preview": "https://storeapi.kobo.com/v1/library/{EntitlementIds}/preview", + "use_one_store": "False", + "user_loyalty_benefits": "https://storeapi.kobo.com/v1/user/loyalty/benefits", + "user_platform": "https://storeapi.kobo.com/v1/user/platform", + "user_profile": "https://storeapi.kobo.com/v1/user/profile", + "user_ratings": "https://storeapi.kobo.com/v1/user/ratings", + "user_recommendations": "https://storeapi.kobo.com/v1/user/recommendations", + "user_reviews": "https://storeapi.kobo.com/v1/user/reviews", + "user_wishlist": "https://storeapi.kobo.com/v1/user/wishlist", + "userguide_host": "https://kbdownload1-a.akamaihd.net", + "wishlist_page": "https://store.kobobooks.com/{region}/{language}/account/wishlist", + } diff --git a/cps/kobo_auth.py b/cps/kobo_auth.py index 60f3ea5f..edb110ff 100644 --- a/cps/kobo_auth.py +++ b/cps/kobo_auth.py @@ -62,7 +62,7 @@ from datetime import datetime from os import urandom from flask import g, Blueprint, url_for -from flask_login import login_user, current_user, login_required +from flask_login import login_user, login_required from flask_babel import gettext as _ from . import logger, ub, lm @@ -102,8 +102,7 @@ def load_user_from_kobo_request(request): login_user(user) return user log.info("Received Kobo request without a recognizable auth token.") - return None - + return kobo_auth = Blueprint("kobo_auth", __name__, url_prefix="/kobo_auth") diff --git a/cps/reverseproxy.py b/cps/reverseproxy.py index 25bbe77b..30af5dbf 100644 --- a/cps/reverseproxy.py +++ b/cps/reverseproxy.py @@ -60,10 +60,13 @@ class ReverseProxied(object): def __init__(self, application): self.app = application + self.proxied = False def __call__(self, environ, start_response): + self.proxied = False script_name = environ.get('HTTP_X_SCRIPT_NAME', '') if script_name: + self.proxied = True environ['SCRIPT_NAME'] = script_name path_info = environ.get('PATH_INFO', '') if path_info and path_info.startswith(script_name): @@ -76,3 +79,7 @@ class ReverseProxied(object): if servr: environ['HTTP_HOST'] = servr return self.app(environ, start_response) + + @property + def is_proxied(self): + return self.proxied diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index b0ad49f7..7fcf44a2 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -1,6 +1,6 @@ {% extends "layout.html" %} {% block body %} -
+

{{title}}

@@ -169,6 +169,18 @@
+ {% if feature_support['kobo'] %} +
+ + +
+
+
+ + +
+
+ {% endif %} {% if feature_support['goodreads'] %}
diff --git a/cps/templates/user_edit.html b/cps/templates/user_edit.html index 9ef11979..4b57b110 100644 --- a/cps/templates/user_edit.html +++ b/cps/templates/user_edit.html @@ -26,6 +26,7 @@
+ {% if not content.role_anonymous() %}
+ {% endif %} +