From 6d30382ae02faa6db2c0302d50f82660ebb2734c Mon Sep 17 00:00:00 2001 From: Jack Darlington Date: Mon, 20 Feb 2017 18:34:37 +0000 Subject: [PATCH 1/2] Initial gdrive commit Work on watching metadata More efficient storing folder keys to database Nearly completed. Need to do final touches to callback for when metadata.db updated on real server, as cannot test locally Changed callback for file changes from being hard coded to mine used url_for in template as apposed to hard coded links Fix to drive template First attempt at redownload metadata.db Fixed incorrect call to downloadFile Added logging Fixed call to copy file Added exception logging to gdriveutils + fixed string long concat Fix file download Fix backup metadata Added slashes to paths Removed threading temporarily Fix for reloading database Fix reinitialising of variables Fix check to see if custom column already setup Update to showing authenticate google drive callback + fix for reinitialising database Fixed logic for showing authenticate with google drive --- cps/db.py | 53 +++--- cps/gdriveutils.py | 313 +++++++++++++++++++++++++++++++++ cps/templates/config_edit.html | 58 ++++++ cps/ub.py | 28 +++ cps/web.py | 263 +++++++++++++++++++++++++-- gdrive_template.yaml | 14 ++ 6 files changed, 690 insertions(+), 39 deletions(-) create mode 100644 cps/gdriveutils.py create mode 100644 gdrive_template.yaml diff --git a/cps/db.py b/cps/db.py index f6ee790e..56594de2 100755 --- a/cps/db.py +++ b/cps/db.py @@ -12,9 +12,9 @@ import ub session = None cc_exceptions = None -cc_classes = None -cc_ids = None -books_custom_column_links = None +cc_classes = {} +cc_ids = [] +books_custom_column_links = {} engine = None @@ -293,41 +293,40 @@ def setup_db(): cc = conn.execute("SELECT id, datatype FROM custom_columns") - cc_ids = [] cc_exceptions = ['datetime', 'int', 'comments', 'float', 'composite', 'series'] - books_custom_column_links = {} - cc_classes = {} for row in cc: if row.datatype not in cc_exceptions: - books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata, + if row.id not in books_custom_column_links: + books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata, Column('book', Integer, ForeignKey('books.id'), primary_key=True), Column('value', Integer, ForeignKey('custom_column_' + str(row.id) + '.id'), primary_key=True) ) - cc_ids.append([row.id, row.datatype]) - if row.datatype == 'bool': - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'book': Column(Integer, ForeignKey('books.id')), - 'value': Column(Boolean)} - else: - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'value': Column(String)} - cc_classes[row.id] = type('Custom_Column_' + str(row.id), (Base,), ccdict) + cc_ids.append([row.id, row.datatype]) + if row.datatype == 'bool': + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'book': Column(Integer, ForeignKey('books.id')), + 'value': Column(Boolean)} + else: + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'value': Column(String)} + cc_classes[row.id] = type('Custom_Column_' + str(row.id), (Base,), ccdict) for id in cc_ids: - if id[1] == 'bool': - setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]], - primaryjoin=( - Books.id == cc_classes[id[0]].book), - backref='books')) - else: - setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]], - secondary=books_custom_column_links[id[0]], - backref='books')) + if not hasattr(Books, 'custom_column_' + str(id[0])): + if id[1] == 'bool': + setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]], + primaryjoin=( + Books.id == cc_classes[id[0]].book), + backref='books')) + else: + setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]], + secondary=books_custom_column_links[id[0]], + backref='books')) # Base.metadata.create_all(engine) Session = sessionmaker() diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py new file mode 100644 index 00000000..a0d1fa2f --- /dev/null +++ b/cps/gdriveutils.py @@ -0,0 +1,313 @@ +from pydrive.auth import GoogleAuth +from pydrive.drive import GoogleDrive +import os, time + +from ub import config + +from sqlalchemy import * +from sqlalchemy import exc +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import * + +from apiclient import errors + +import web + + +dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "gdrive.db") +engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False) +Base = declarative_base() + +# Open session for database connection +Session = sessionmaker() +Session.configure(bind=engine) +session = Session() + +class GdriveId(Base): + __tablename__='gdrive_ids' + + id = Column(Integer, primary_key=True) + gdrive_id = Column(Integer, unique=True) + path = Column(String) + + def __repr__(self): + return str(self.path) + +if not os.path.exists(dbpath): + try: + Base.metadata.create_all(engine) + except Exception: + raise + +def getDrive(gauth=None): + if not gauth: + gauth=GoogleAuth(settings_file='settings.yaml') + # Try to load saved client credentials + gauth.LoadCredentialsFile("gdrive_credentials") + if gauth.access_token_expired: + # Refresh them if expired + gauth.Refresh() + else: + # Initialize the saved creds + gauth.Authorize() + # Save the current credentials to a file + return GoogleDrive(gauth) + +def getEbooksFolder(drive=None): + if not drive: + drive = getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + ebooksFolder= "title = '%s' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % config.config_google_drive_folder + + fileList = drive.ListFile({'q': ebooksFolder}).GetList() + return fileList[0] + +def getEbooksFolderId(drive=None): + storedPathName=session.query(GdriveId).filter(GdriveId.path == '/').first() + if storedPathName: + return storedPathName.gdrive_id + else: + gDriveId=GdriveId() + gDriveId.gdrive_id=getEbooksFolder(drive)['id'] + gDriveId.path='/' + session.merge(gDriveId) + session.commit() + return + +def getFolderInFolder(parentId, folderName, drive=None): + if not drive: + drive = getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + folder= "title = '%s' and '%s' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % (folderName.replace("'", "\\'"), parentId) + fileList = drive.ListFile({'q': folder}).GetList() + return fileList[0] + +def getFile(pathId, fileName, drive=None): + if not drive: + drive = getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + metaDataFile="'%s' in parents and trashed = false and title = '%s'" % (pathId, fileName.replace("'", "\\'")) + + fileList = drive.ListFile({'q': metaDataFile}).GetList() + return fileList[0] + +def getFolderId(path, drive=None): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + currentFolderId=getEbooksFolderId(drive) + sqlCheckPath=path if path[-1] =='/' else path + '/' + storedPathName=session.query(GdriveId).filter(GdriveId.path == sqlCheckPath).first() + + if not storedPathName: + dbChange=False + s=path.split('/') + for i, x in enumerate(s): + if len(x) > 0: + currentPath="/".join(s[:i+1]) + if currentPath[-1] != '/': + currentPath = currentPath + '/' + storedPathName=session.query(GdriveId).filter(GdriveId.path == currentPath).first() + if storedPathName: + currentFolderId=storedPathName.gdrive_id + else: + currentFolderId=getFolderInFolder(currentFolderId, x, drive)['id'] + gDriveId=GdriveId() + gDriveId.gdrive_id=currentFolderId + gDriveId.path=currentPath + session.merge(gDriveId) + dbChange=True + if dbChange: + session.commit() + else: + currentFolderId=storedPathName.gdrive_id + return currentFolderId + + +def getFileFromEbooksFolder(drive, path, fileName): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + if path: + sqlCheckPath=path if path[-1] =='/' else path + '/' + folderId=getFolderId(path, drive) + else: + folderId=getEbooksFolderId(drive) + + return getFile(folderId, fileName, drive) + +def copyDriveFileRemote(drive, origin_file_id, copy_title): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + copied_file = {'title': copy_title} + try: + file_data = drive.auth.service.files().copy( + fileId=origin_file_id, body=copied_file).execute() + return drive.CreateFile({'id': file_data['id']}) + except errors.HttpError as error: + print ('An error occurred: %s' % error) + return None + +def downloadFile(drive, path, filename, output): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + f=getFileFromEbooksFolder(drive, path, filename) + f.GetContentFile(output) + +def backupCalibreDbAndOptionalDownload(drive, f=None): + pass + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + metaDataFile="'%s' in parents and title = 'metadata.db' and trashed = false" % getEbooksFolderId() + + fileList = drive.ListFile({'q': metaDataFile}).GetList() + + databaseFile=fileList[0] + + if f: + databaseFile.GetContentFile(f) + +def copyToDrive(drive, uploadFile, createRoot, replaceFiles, + ignoreFiles=[], + parent=None, prevDir=''): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + isInitial=not bool(parent) + if not parent: + parent=getEbooksFolder(drive) + if os.path.isdir(os.path.join(prevDir,uploadFile)): + existingFolder=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList() + if len(existingFolder) == 0 and (not isInitial or createRoot): + parent = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], + "mimeType": "application/vnd.google-apps.folder" }) + parent.Upload() + else: + if (not isInitial or createRoot) and len(existingFolder) > 0: + parent=existingFolder[0] + for f in os.listdir(os.path.join(prevDir,uploadFile)): + if f not in ignoreFiles: + copyToDrive(drive, f, True, replaceFiles, ignoreFiles, parent, os.path.join(prevDir,uploadFile)) + else: + if os.path.basename(uploadFile) not in ignoreFiles: + existingFiles=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList() + if len(existingFiles) > 0: + driveFile = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], }) + else: + driveFile=existingFiles[0] + driveFile.SetContentFile(os.path.join(prevDir,uploadFile)) + driveFile.Upload() + +def watchChange(drive, channel_id, channel_type, channel_address, + channel_token=None, expiration=None): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + """Watch for all changes to a user's Drive. + Args: + service: Drive API service instance. + channel_id: Unique string that identifies this channel. + channel_type: Type of delivery mechanism used for this channel. + channel_address: Address where notifications are delivered. + channel_token: An arbitrary string delivered to the target address with + each notification delivered over this channel. Optional. + channel_address: Address where notifications are delivered. Optional. + Returns: + The created channel if successful + Raises: + apiclient.errors.HttpError: if http request to create channel fails. + """ + body = { + 'id': channel_id, + 'type': channel_type, + 'address': channel_address + } + if channel_token: + body['token'] = channel_token + if expiration: + body['expiration'] = expiration + return drive.auth.service.changes().watch(body=body).execute() + +def watchFile(drive, file_id, channel_id, channel_type, channel_address, + channel_token=None, expiration=None): + """Watch for any changes to a specific file. + Args: + service: Drive API service instance. + file_id: ID of the file to watch. + channel_id: Unique string that identifies this channel. + channel_type: Type of delivery mechanism used for this channel. + channel_address: Address where notifications are delivered. + channel_token: An arbitrary string delivered to the target address with + each notification delivered over this channel. Optional. + channel_address: Address where notifications are delivered. Optional. + Returns: + The created channel if successful + Raises: + apiclient.errors.HttpError: if http request to create channel fails. + """ + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + + body = { + 'id': channel_id, + 'type': channel_type, + 'address': channel_address + } + if channel_token: + body['token'] = channel_token + if expiration: + body['expiration'] = expiration + return drive.auth.service.files().watch(fileId=file_id, body=body).execute() + +def stopChannel(drive, channel_id, resource_id): + """Stop watching to a specific channel. + Args: + service: Drive API service instance. + channel_id: ID of the channel to stop. + resource_id: Resource ID of the channel to stop. + Raises: + apiclient.errors.HttpError: if http request to create channel fails. + """ + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + service=drive.auth.service + body = { + 'id': channel_id, + 'resourceId': resource_id + } + return drive.auth.service.channels().stop(body=body).execute() + +def getChangeById (drive, change_id): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + """Print a single Change resource information. + + Args: + service: Drive API service instance. + change_id: ID of the Change resource to retrieve. + """ + try: + change = drive.auth.service.changes().get(changeId=change_id).execute() + return change + except errors.HttpError, error: + web.app.logger.exception(error) + return None diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 7824ddd0..9d107cb2 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -7,6 +7,45 @@ +
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ {% if show_authenticate_google_drive %} +
+ Authenticate Google Drive +
+ {% else %} + {% if content.config_google_drive_watch_changes_response %} + +
+ + + Revoke + + {% else %} + Enable watch of metadata.db + {% endif %} +
+ {% endif %} +
@@ -80,3 +119,22 @@
{% endblock %} +{% block js %} + +{% endblock %} diff --git a/cps/ub.py b/cps/ub.py index 19f85880..1addddcb 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -11,6 +11,7 @@ import traceback import logging from werkzeug.security import generate_password_hash from flask_babel import gettext as _ +import json dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "app.db") engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False) @@ -269,6 +270,12 @@ class Settings(Base): config_anonbrowse = Column(SmallInteger, default=0) config_public_reg = Column(SmallInteger, default=0) config_default_role = Column(SmallInteger, default=0) + config_use_google_drive = Column(Boolean) + config_google_drive_client_id = Column(String) + config_google_drive_client_secret = Column(String) + config_google_drive_folder = Column(String) + config_google_drive_calibre_url_base = Column(String) + config_google_drive_watch_changes_response = Column(String) def __repr__(self): pass @@ -295,6 +302,16 @@ class Config: self.config_anonbrowse = data.config_anonbrowse self.config_public_reg = data.config_public_reg self.config_default_role = data.config_default_role + self.config_use_google_drive = data.config_use_google_drive + self.config_google_drive_client_id = data.config_google_drive_client_id + self.config_google_drive_client_secret = data.config_google_drive_client_secret + self.config_google_drive_calibre_url_base = data.config_google_drive_calibre_url_base + self.config_google_drive_folder = data.config_google_drive_folder + if data.config_google_drive_watch_changes_response: + self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response) + else: + self.config_google_drive_watch_changes_response=None + if self.config_calibre_dir is not None: self.db_configured = True else: @@ -379,6 +396,17 @@ def migrate_Database(): conn.execute("ALTER TABLE Settings ADD column `config_anonbrowse` SmallInteger DEFAULT 0") conn.execute("ALTER TABLE Settings ADD column `config_public_reg` SmallInteger DEFAULT 0") session.commit() + + try: + session.query(exists().where(Settings.config_use_google_drive)).scalar() + except exc.OperationalError: + conn = engine.connect() + conn.execute("ALTER TABLE Settings ADD column `config_use_google_drive` INTEGER DEFAULT 0") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_id` String DEFAULT ''") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_secret` String DEFAULT ''") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_calibre_url_base` INTEGER DEFAULT 0") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_folder` String DEFAULT ''") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_watch_changes_response` String DEFAULT ''") try: session.query(exists().where(Settings.config_default_role)).scalar() session.commit() diff --git a/cps/web.py b/cps/web.py index 4e644cc1..9a49ddf2 100755 --- a/cps/web.py +++ b/cps/web.py @@ -1,12 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from pydrive.auth import GoogleAuth + import mimetypes import logging from logging.handlers import RotatingFileHandler from tempfile import gettempdir import textwrap from flask import Flask, render_template, session, request, Response, redirect, url_for, send_from_directory, \ - make_response, g, flash, abort + make_response, g, flash, abort, send_file import ub from ub import config import helper @@ -42,6 +44,15 @@ import db from shutil import move, copyfile from tornado.ioloop import IOLoop import StringIO +from shutil import move +import gdriveutils +import io +import hashlib +import threading + +import time + +current_milli_time = lambda: int(round(time.time() * 1000)) try: from wand.image import Image @@ -52,13 +63,67 @@ except ImportError, e: from cgi import escape # Global variables +gdrive_watch_callback_token='target=calibreweb-watch_files' global_task = None +def md5(fname): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + +class Singleton: + """ + A non-thread-safe helper class to ease implementing singletons. + This should be used as a decorator -- not a metaclass -- to the + class that should be a singleton. + + The decorated class can define one `__init__` function that + takes only the `self` argument. Also, the decorated class cannot be + inherited from. Other than that, there are no restrictions that apply + to the decorated class. + + To get the singleton instance, use the `Instance` method. Trying + to use `__call__` will result in a `TypeError` being raised. + + """ + + def __init__(self, decorated): + self._decorated = decorated + + def Instance(self): + """ + Returns the singleton instance. Upon its first call, it creates a + new instance of the decorated class and calls its `__init__` method. + On all subsequent calls, the already created instance is returned. + + """ + try: + return self._instance + except AttributeError: + self._instance = self._decorated() + return self._instance + + def __call__(self): + raise TypeError('Singletons must be accessed through `Instance()`.') + + def __instancecheck__(self, inst): + return isinstance(inst, self._decorated) + +@Singleton +class Gauth: + def __init__(self): + self.auth=GoogleAuth(settings_file='settings.yaml') + +@Singleton +class Gdrive: + def __init__(self): + self.drive=gdriveutils.getDrive(Gauth.Instance().auth) -# Proxy Helper class class ReverseProxied(object): """Wrap the application in this middleware and configure the - front-end server to add these headers, to let you quietly bind + front-end server to add these headers, to let you quietly bind this to a URL other than / and to an HTTP scheme that is different than what is used locally. @@ -187,6 +252,12 @@ def authenticate(): 'You have to login with proper credentials', 401, {'WWW-Authenticate': 'Basic realm="Login Required"'}) +def updateGdriveCalibreFromLocal(): + gdriveutils.backupCalibreDbAndOptionalDownload(Gdrive.Instance().drive) + gdriveutils.copyToDrive(Gdrive.Instance().drive, config.config_calibre_dir, False, True) + for x in os.listdir(config.config_calibre_dir): + if os.path.isdir(os.path.join(config.config_calibre_dir,x)): + shutil.rmtree(os.path.join(config.config_calibre_dir,x)) def requires_basic_auth_if_no_ano(f): @wraps(f) @@ -286,6 +357,17 @@ def formatdate(val): formatdate = datetime.datetime.strptime(conformed_timestamp[:15], "%Y%m%d %H%M%S") return format_date(formatdate, format='medium',locale=get_locale()) +@app.template_filter('strftime') +def timestamptodate(date, fmt=None): + date=datetime.datetime.fromtimestamp( + int(date)/1000 + ) + native = date.replace(tzinfo=None) + if fmt: + format=fmt + else: + format='%d %m %Y - %H:%S' + return native.strftime(format) def admin_required(f): """ @@ -668,8 +750,15 @@ def get_opds_download_link(book_id, format): file_name = book.title if len(book.authors) > 0: file_name = book.authors[0].name + '-' + file_name - file_name = helper.get_valid_filename(file_name) - response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format)) + + if config.config_use_google_drive: + df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format)) + download_url = df.metadata.get('downloadUrl') + resp, content = df.auth.Get_Http_Object().request(download_url) + response=send_file(io.BytesIO(content)) + else: + file_name = helper.get_valid_filename(file_name) + response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format)) response.headers["Content-Disposition"] = "attachment; filename=\"%s.%s\"" % (data.name, format) return response @@ -802,7 +891,9 @@ def hot_books(page): hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() for book in hot_books: - entries.append(db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first()) + entry=db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first() + if entry: + entries.append(entry) numBooks = entries.__len__() pagination = Pagination(page, config.config_books_per_page, numBooks) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, @@ -1037,6 +1128,99 @@ def stats(): categorycounter=categorys, seriecounter=series, title=_(u"Statistics")) +#@app.route("/load_gdrive") +#@login_required +#@admin_required +#def load_all_gdrive_folder_ids(): +# books=db.session.query(db.Books).all() +# for book in books: +# gdriveutils.getFolderId(book.path, Gdrive.Instance().drive) +# return + +@app.route("/gdrive/authenticate") +@login_required +@admin_required +def authenticate_google_drive(): + authUrl=Gauth.Instance().auth.GetAuthUrl() + return redirect(authUrl) + +@app.route("/gdrive/callback") +def google_drive_callback(): + auth_code = request.args.get('code') + credentials = Gauth.Instance().auth.flow.step2_exchange(auth_code) + with open('gdrive_credentials' ,'w') as f: + f.write(credentials.to_json()) + return redirect(url_for('configuration')) + +@app.route("/gdrive/watch/subscribe") +@login_required +@admin_required +def watch_gdrive(): + if not config.config_google_drive_watch_changes_response: + address = '%scalibre-web/gdrive/watch/callback' % config.config_google_drive_calibre_url_base + notification_id=str(uuid4()) + result = gdriveutils.watchChange(Gdrive.Instance().drive, notification_id, + 'web_hook', address, gdrive_watch_callback_token, current_milli_time() + 604800*1000) + print (result) + settings = ub.session.query(ub.Settings).first() + settings.config_google_drive_watch_changes_response=json.dumps(result) + ub.session.merge(settings) + ub.session.commit() + settings = ub.session.query(ub.Settings).first() + config.loadSettings() + + print (settings.config_google_drive_watch_changes_response) + + return redirect(url_for('configuration')) + +@app.route("/gdrive/watch/revoke") +@login_required +@admin_required +def revoke_watch_gdrive(): + last_watch_response=config.config_google_drive_watch_changes_response + if last_watch_response: + response=gdriveutils.stopChannel(Gdrive.Instance().drive, last_watch_response['id'], last_watch_response['resourceId']) + settings = ub.session.query(ub.Settings).first() + settings.config_google_drive_watch_changes_response=None + ub.session.merge(settings) + ub.session.commit() + config.loadSettings() + return redirect(url_for('configuration')) + +@app.route("/gdrive/watch/callback", methods=['GET', 'POST']) +def on_received_watch_confirmation(): + app.logger.info (request.headers) + if request.headers.get('X-Goog-Channel-Token') == gdrive_watch_callback_token \ + and request.headers.get('X-Goog-Resource-State') == 'change' \ + and request.data: + + data=request.data + + def updateMetaData(): + app.logger.info ('Change received from gdrive') + app.logger.info (data) + try: + j=json.loads(data) + app.logger.info ('Getting change details') + response=gdriveutils.getChangeById(Gdrive.Instance().drive, j['id']) + app.logger.info (response) + if response: + dbpath = os.path.join(config.config_calibre_dir, "metadata.db") + if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != md5(dbpath): + app.logger.info ('Database file updated') + copyfile (dbpath, config.config_calibre_dir + "/metadata.db_" + str(current_milli_time())) + app.logger.info ('Backing up existing and downloading updated metadata.db') + gdriveutils.downloadFile(Gdrive.Instance().drive, None, "metadata.db", config.config_calibre_dir + "/tmp_metadata.db") + app.logger.info ('Setting up new DB') + os.rename(config.config_calibre_dir + "/tmp_metadata.db", dbpath) + db.setup_db() + except Exception, e: + app.logger.exception(e) + + updateMetaData() + return '' + + @app.route("/shutdown") @login_required @admin_required @@ -1173,8 +1357,15 @@ def advanced_search(): @app.route("/cover/") @login_required_if_no_ano def get_cover(cover_path): - return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg") + if config.config_use_google_drive: + df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg') + download_url = df.metadata.get('webContentLink') + return redirect(download_url) + else: + return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg") + resp.headers['Content-Type']='image/jpeg' + return resp @app.route("/opds/thumb_240_240/") @app.route("/opds/cover_240_240/") @@ -1183,7 +1374,12 @@ def get_cover(cover_path): @requires_basic_auth_if_no_ano def feed_get_cover(book_id): book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - return send_from_directory(os.path.join(config.config_calibre_dir, book.path), "cover.jpg") + if config.config_use_google_drive: + df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg') + download_url = df.metadata.get('webContentLink') + return redirect(download_url) + else: + return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg") def render_read_books(page, are_read, as_xml=False): readBooks=ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id)).filter(ub.ReadBook.is_read == True).all() @@ -1308,8 +1504,13 @@ def get_download_link(book_id, format): if len(book.authors) > 0: file_name = book.authors[0].name + '-' + file_name file_name = helper.get_valid_filename(file_name) - response = make_response( - send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format)) + if config.config_use_google_drive: + df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format)) + download_url = df.metadata.get('downloadUrl') + resp, content = df.auth.Get_Http_Object().request(download_url) + response=send_file(io.BytesIO(content)) + else: + response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format)) try: response.headers["Content-Type"] = mimetypes.types_map['.' + format] except: @@ -1682,6 +1883,38 @@ def configuration_helper(origin): if content.config_calibre_dir != to_save["config_calibre_dir"]: content.config_calibre_dir = to_save["config_calibre_dir"] db_change = True + ##Google drive setup + create_new_yaml=False + if "config_google_drive_client_id" in to_save: + if content.config_google_drive_client_id != to_save["config_google_drive_client_id"]: + content.config_google_drive_client_id = to_save["config_google_drive_client_id"] + create_new_yaml=True + db_change = True + if "config_google_drive_client_secret" in to_save: + if content.config_google_drive_client_secret != to_save["config_google_drive_client_secret"]: + content.config_google_drive_client_secret = to_save["config_google_drive_client_secret"] + create_new_yaml=True + db_change = True + if "config_google_drive_calibre_url_base" in to_save: + if content.config_google_drive_calibre_url_base != to_save["config_google_drive_calibre_url_base"]: + content.config_google_drive_calibre_url_base = to_save["config_google_drive_calibre_url_base"] + create_new_yaml=True + db_change = True + if ("config_use_google_drive" in to_save and not content.config_use_google_drive) or ("config_use_google_drive" not in to_save and content.config_use_google_drive): + content.config_use_google_drive = "config_use_google_drive" in to_save + db_change = True + if not content.config_use_google_drive: + create_new_yaml=False + if create_new_yaml: + with open('settings.yaml', 'w') as f: + with open('gdrive_template.yaml' ,'r') as t: + f.write(t.read() % {'client_id' : content.config_google_drive_client_id, 'client_secret' : content.config_google_drive_client_secret, + "redirect_uri" : content.config_google_drive_calibre_url_base + 'gdrive/callback'}) + if "config_google_drive_folder" in to_save: + if content.config_google_drive_folder != to_save["config_google_drive_folder"]: + content.config_google_drive_folder = to_save["config_google_drive_folder"] + db_change = True + ## if "config_port" in to_save: if content.config_port != int(to_save["config_port"]): content.config_port = int(to_save["config_port"]) @@ -1751,6 +1984,7 @@ def configuration_helper(origin): if origin: success = True return render_title_template("config_edit.html", origin=origin, success=success, content=config, + show_authenticate_google_drive=not os.path.exists('settings.yaml') or not os.path.exists('gdrive_credentials'), title=_(u"Basic Configuration")) @@ -1999,7 +2233,7 @@ def edit_book(book_id): modify_database_object(input_authors, book.authors, db.Authors, db.session, 'author') if author0_before_edit != book.authors[0].name: edited_books_id.add(book.id) - book.author_sort=helper.get_sorted_author(input_authors[0]) + book.author_sort=helper.get_sorted_author(input_authors[0]) if to_save["cover_url"] and os.path.splitext(to_save["cover_url"])[1].lower() == ".jpg": img = requests.get(to_save["cover_url"]) @@ -2163,6 +2397,8 @@ def edit_book(book_id): author_names.append(author.name) for b in edited_books_id: helper.update_dir_stucture(b, config.config_calibre_dir) + if config.config_use_google_drive: + updateGdriveCalibreFromLocal() if "detail_view" in to_save: return redirect(url_for('show_book', id=book.id)) else: @@ -2227,7 +2463,7 @@ def upload(): if is_author: db_author = is_author else: - db_author = db.Authors(author, helper.get_sorted_author(author), "") + db_author = db.Authors(author, helper.get_sorted_author(author), "") db.session.add(db_author) # combine path and normalize path from windows systems path = os.path.join(author_dir, title_dir).replace('\\','/') @@ -2242,6 +2478,9 @@ def upload(): author_names = [] for author in db_book.authors: author_names.append(author.name) + if config.config_use_google_drive: + if not current_user.role_edit() and not current_user.role_admin(): + updateGdriveCalibreFromLocal() cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() if current_user.role_edit() or current_user.role_admin(): return render_title_template('book_edit.html', book=db_book, authors=author_names, cc=cc, diff --git a/gdrive_template.yaml b/gdrive_template.yaml new file mode 100644 index 00000000..a6d77eee --- /dev/null +++ b/gdrive_template.yaml @@ -0,0 +1,14 @@ +client_config_backend: settings +client_config: + client_id: %(client_id)s + client_secret: %(client_secret)s + redirect_uri: %(redirect_uri)s + +save_credentials: True +save_credentials_backend: file +save_credentials_file: gdrive_credentials + +get_refresh_token: True + +oauth_scope: + - https://www.googleapis.com/auth/drive From 7b709c15c26b4bce1ce42d56389110eb9f2bf26b Mon Sep 17 00:00:00 2001 From: Jack Darlington Date: Wed, 22 Feb 2017 22:06:59 +0000 Subject: [PATCH 2/2] GitIgnore update + fixes for initial setup --- .gitignore | 3 +++ cps/db.py | 2 ++ cps/gdriveutils.py | 4 ++-- cps/ub.py | 2 +- cps/web.py | 11 +++++++---- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 4ed21c61..a8166349 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ cps/static/[0-9]* *.bak *.log.* tags + +settings.yaml +gdrive_credentials \ No newline at end of file diff --git a/cps/db.py b/cps/db.py index 56594de2..c64f286b 100755 --- a/cps/db.py +++ b/cps/db.py @@ -274,6 +274,8 @@ def setup_db(): return False dbpath = os.path.join(config.config_calibre_dir, "metadata.db") + if not os.path.exists(dbpath): + return False engine = create_engine('sqlite:///{0}'.format(dbpath.encode('utf-8')), echo=False) try: conn = engine.connect() diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index a0d1fa2f..66afed71 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -204,9 +204,9 @@ def copyToDrive(drive, uploadFile, createRoot, replaceFiles, if os.path.basename(uploadFile) not in ignoreFiles: existingFiles=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList() if len(existingFiles) > 0: - driveFile = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], }) - else: driveFile=existingFiles[0] + else: + driveFile = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], }) driveFile.SetContentFile(os.path.join(prevDir,uploadFile)) driveFile.Upload() diff --git a/cps/ub.py b/cps/ub.py index 1addddcb..d5063866 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -312,7 +312,7 @@ class Config: else: self.config_google_drive_watch_changes_response=None - if self.config_calibre_dir is not None: + if (self.config_calibre_dir is not None and not self.config_use_google_drive) or os.path.exists(self.config_calibre_dir + '/metadata.db'): self.db_configured = True else: self.db_configured = False diff --git a/cps/web.py b/cps/web.py index 9a49ddf2..53a5b1a5 100755 --- a/cps/web.py +++ b/cps/web.py @@ -43,6 +43,7 @@ import re import db from shutil import move, copyfile from tornado.ioloop import IOLoop +import shutil import StringIO from shutil import move import gdriveutils @@ -198,6 +199,9 @@ lm.anonymous_user = ub.Anonymous app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT' db.setup_db() +def is_gdrive_ready(): + return os.path.exists('settings.yaml') and os.path.exists('gdrive_credentials') + @babel.localeselector def get_locale(): # if a user is logged in, use the locale from the user settings @@ -1889,17 +1893,14 @@ def configuration_helper(origin): if content.config_google_drive_client_id != to_save["config_google_drive_client_id"]: content.config_google_drive_client_id = to_save["config_google_drive_client_id"] create_new_yaml=True - db_change = True if "config_google_drive_client_secret" in to_save: if content.config_google_drive_client_secret != to_save["config_google_drive_client_secret"]: content.config_google_drive_client_secret = to_save["config_google_drive_client_secret"] create_new_yaml=True - db_change = True if "config_google_drive_calibre_url_base" in to_save: if content.config_google_drive_calibre_url_base != to_save["config_google_drive_calibre_url_base"]: content.config_google_drive_calibre_url_base = to_save["config_google_drive_calibre_url_base"] create_new_yaml=True - db_change = True if ("config_use_google_drive" in to_save and not content.config_use_google_drive) or ("config_use_google_drive" not in to_save and content.config_use_google_drive): content.config_use_google_drive = "config_use_google_drive" in to_save db_change = True @@ -1953,6 +1954,8 @@ def configuration_helper(origin): if "passwd_role" in to_save: content.config_default_role = content.config_default_role + ub.ROLE_PASSWD try: + if content.config_use_google_drive and is_gdrive_ready() and not os.path.exists(config.config_calibre_dir + "/metadata.db"): + gdriveutils.downloadFile(Gdrive.Instance().drive, None, "metadata.db", config.config_calibre_dir + "/metadata.db") if db_change: if config.db_configured: db.session.close() @@ -1984,7 +1987,7 @@ def configuration_helper(origin): if origin: success = True return render_title_template("config_edit.html", origin=origin, success=success, content=config, - show_authenticate_google_drive=not os.path.exists('settings.yaml') or not os.path.exists('gdrive_credentials'), + show_authenticate_google_drive=not is_gdrive_ready(), title=_(u"Basic Configuration"))