Merge branch 'feature/google_drive' into develop

.gitignore

@@ -23,3 +23,6 @@ cps/static/[0-9]*
 *.bak
 *.log.*
 tags
+
+settings.yaml
+gdrive_credentials

cps/db.py

@@ -12,9 +12,9 @@ import ub
 
 session = None
 cc_exceptions = None
-cc_classes = None
+cc_classes = {}
-cc_ids = None
+cc_ids = []
-books_custom_column_links = None
+books_custom_column_links = {}
 engine = None
 
 
@@ -274,6 +274,8 @@ def setup_db():
         return False
 
     dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
+    if not os.path.exists(dbpath):
+        return False
     engine = create_engine('sqlite:///{0}'.format(dbpath.encode('utf-8')), echo=False)
     try:
         conn = engine.connect()
@@ -293,12 +295,10 @@ def setup_db():
 
     cc = conn.execute("SELECT id, datatype FROM custom_columns")
 
-    cc_ids = []
     cc_exceptions = ['datetime', 'int', 'comments', 'float', 'composite', 'series']
-    books_custom_column_links = {}
-    cc_classes = {}
     for row in cc:
         if row.datatype not in cc_exceptions:
+            if row.id not in books_custom_column_links:
                 books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata,
                                                       Column('book', Integer, ForeignKey('books.id'),
                                                              primary_key=True),
@@ -319,6 +319,7 @@ def setup_db():
                 cc_classes[row.id] = type('Custom_Column_' + str(row.id), (Base,), ccdict)
 
     for id in cc_ids:
+        if not hasattr(Books, 'custom_column_' + str(id[0])):
             if id[1] == 'bool':
                 setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]],
                                                                            primaryjoin=(

cps/gdriveutils.py

@@ -0,0 +1,313 @@
+from pydrive.auth import GoogleAuth
+from pydrive.drive import GoogleDrive
+import os, time
+
+from ub import config
+
+from sqlalchemy import *
+from sqlalchemy import exc
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import *
+
+from apiclient import errors
+
+import web
+
+
+dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "gdrive.db")
+engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False)
+Base = declarative_base()
+
+# Open session for database connection
+Session = sessionmaker()
+Session.configure(bind=engine)
+session = Session()
+
+class GdriveId(Base):
+    __tablename__='gdrive_ids'
+
+    id = Column(Integer, primary_key=True)
+    gdrive_id = Column(Integer, unique=True)
+    path = Column(String)
+
+    def __repr__(self):
+        return str(self.path)
+
+if not os.path.exists(dbpath):
+    try:
+        Base.metadata.create_all(engine)
+    except Exception:
+        raise
+
+def getDrive(gauth=None):
+    if not gauth:
+        gauth=GoogleAuth(settings_file='settings.yaml')
+    # Try to load saved client credentials
+    gauth.LoadCredentialsFile("gdrive_credentials")
+    if gauth.access_token_expired:
+        # Refresh them if expired
+        gauth.Refresh()
+    else:
+        # Initialize the saved creds
+        gauth.Authorize()
+    # Save the current credentials to a file
+    return GoogleDrive(gauth)
+
+def getEbooksFolder(drive=None):
+    if not drive:
+        drive = getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    ebooksFolder= "title = '%s' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % config.config_google_drive_folder
+
+    fileList = drive.ListFile({'q': ebooksFolder}).GetList()
+    return fileList[0]
+
+def getEbooksFolderId(drive=None):
+    storedPathName=session.query(GdriveId).filter(GdriveId.path == '/').first()
+    if storedPathName:
+        return storedPathName.gdrive_id
+    else:
+        gDriveId=GdriveId()
+        gDriveId.gdrive_id=getEbooksFolder(drive)['id']
+        gDriveId.path='/'
+        session.merge(gDriveId)
+        session.commit()
+        return 
+
+def getFolderInFolder(parentId, folderName, drive=None):
+    if not drive:
+        drive = getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    folder= "title = '%s' and '%s' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false"  % (folderName.replace("'", "\\'"), parentId)
+    fileList = drive.ListFile({'q': folder}).GetList()
+    return fileList[0]    
+
+def getFile(pathId, fileName, drive=None):
+    if not drive:
+        drive = getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    metaDataFile="'%s' in parents and trashed = false and title = '%s'" % (pathId, fileName.replace("'", "\\'"))
+
+    fileList = drive.ListFile({'q': metaDataFile}).GetList()
+    return fileList[0]
+
+def getFolderId(path, drive=None):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    currentFolderId=getEbooksFolderId(drive)
+    sqlCheckPath=path if path[-1] =='/' else path + '/'
+    storedPathName=session.query(GdriveId).filter(GdriveId.path == sqlCheckPath).first()
+
+    if not storedPathName:
+        dbChange=False
+        s=path.split('/')
+        for i, x in enumerate(s):
+            if len(x) > 0:
+                currentPath="/".join(s[:i+1])
+                if currentPath[-1] != '/':
+                    currentPath = currentPath + '/'
+                storedPathName=session.query(GdriveId).filter(GdriveId.path == currentPath).first()
+                if storedPathName:
+                    currentFolderId=storedPathName.gdrive_id
+                else:
+                    currentFolderId=getFolderInFolder(currentFolderId, x, drive)['id']
+                    gDriveId=GdriveId()
+                    gDriveId.gdrive_id=currentFolderId
+                    gDriveId.path=currentPath
+                    session.merge(gDriveId)
+                    dbChange=True
+        if dbChange:
+            session.commit()
+    else:
+        currentFolderId=storedPathName.gdrive_id
+    return currentFolderId
+
+
+def getFileFromEbooksFolder(drive, path, fileName):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    if path:
+        sqlCheckPath=path if path[-1] =='/' else path + '/'
+        folderId=getFolderId(path, drive)
+    else:
+        folderId=getEbooksFolderId(drive)
+    
+    return getFile(folderId, fileName, drive)
+
+def copyDriveFileRemote(drive, origin_file_id, copy_title):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    copied_file = {'title': copy_title}
+    try:
+        file_data = drive.auth.service.files().copy(
+        fileId=origin_file_id, body=copied_file).execute()
+        return drive.CreateFile({'id': file_data['id']})
+    except errors.HttpError as error:
+        print ('An error occurred: %s' % error)
+    return None
+
+def downloadFile(drive, path, filename, output):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    f=getFileFromEbooksFolder(drive, path, filename)
+    f.GetContentFile(output)
+
+def backupCalibreDbAndOptionalDownload(drive, f=None):
+    pass
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    metaDataFile="'%s' in parents and title = 'metadata.db' and trashed = false" % getEbooksFolderId()
+
+    fileList = drive.ListFile({'q': metaDataFile}).GetList()
+ 
+    databaseFile=fileList[0]
+
+    if f:
+        databaseFile.GetContentFile(f)
+
+def copyToDrive(drive, uploadFile, createRoot, replaceFiles,
+    ignoreFiles=[],
+    parent=None, prevDir=''):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    isInitial=not bool(parent)
+    if not parent:
+        parent=getEbooksFolder(drive)
+    if os.path.isdir(os.path.join(prevDir,uploadFile)):
+        existingFolder=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList()
+        if len(existingFolder) == 0 and (not isInitial or createRoot):
+            parent = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], 
+                "mimeType": "application/vnd.google-apps.folder" })
+            parent.Upload()
+        else:
+            if (not isInitial or createRoot) and len(existingFolder) > 0:
+                parent=existingFolder[0]
+        for f in os.listdir(os.path.join(prevDir,uploadFile)):
+            if f not in ignoreFiles:
+                copyToDrive(drive, f, True, replaceFiles, ignoreFiles, parent, os.path.join(prevDir,uploadFile))
+    else:
+        if os.path.basename(uploadFile) not in ignoreFiles:
+            existingFiles=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList()
+            if len(existingFiles) > 0:
+                driveFile=existingFiles[0]
+            else:
+                driveFile = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], })
+            driveFile.SetContentFile(os.path.join(prevDir,uploadFile))
+            driveFile.Upload()
+
+def watchChange(drive, channel_id, channel_type, channel_address,
+              channel_token=None, expiration=None):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+    """Watch for all changes to a user's Drive.
+    Args:
+    service: Drive API service instance.
+    channel_id: Unique string that identifies this channel.
+    channel_type: Type of delivery mechanism used for this channel.
+    channel_address: Address where notifications are delivered.
+    channel_token: An arbitrary string delivered to the target address with
+                   each notification delivered over this channel. Optional.
+    channel_address: Address where notifications are delivered. Optional.
+    Returns:
+    The created channel if successful
+    Raises:
+    apiclient.errors.HttpError: if http request to create channel fails.
+    """
+    body = {
+    'id': channel_id,
+    'type': channel_type,
+    'address': channel_address
+    }
+    if channel_token:
+        body['token'] = channel_token
+    if expiration:
+        body['expiration'] = expiration
+    return  drive.auth.service.changes().watch(body=body).execute()
+
+def watchFile(drive, file_id, channel_id, channel_type, channel_address,
+              channel_token=None, expiration=None):
+    """Watch for any changes to a specific file.
+    Args:
+    service: Drive API service instance.
+    file_id: ID of the file to watch.
+    channel_id: Unique string that identifies this channel.
+    channel_type: Type of delivery mechanism used for this channel.
+    channel_address: Address where notifications are delivered.
+    channel_token: An arbitrary string delivered to the target address with
+                   each notification delivered over this channel. Optional.
+    channel_address: Address where notifications are delivered. Optional.
+    Returns:
+    The created channel if successful
+    Raises:
+    apiclient.errors.HttpError: if http request to create channel fails.
+    """
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh()
+
+    body = {
+        'id': channel_id,
+        'type': channel_type,
+        'address': channel_address
+    }
+    if channel_token:
+        body['token'] = channel_token
+    if expiration:
+        body['expiration'] = expiration
+    return drive.auth.service.files().watch(fileId=file_id, body=body).execute()
+
+def stopChannel(drive, channel_id, resource_id):
+    """Stop watching to a specific channel.
+    Args:
+    service: Drive API service instance.
+    channel_id: ID of the channel to stop.
+    resource_id: Resource ID of the channel to stop.
+    Raises:
+    apiclient.errors.HttpError: if http request to create channel fails.
+    """
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh() 
+    service=drive.auth.service
+    body = {
+    'id': channel_id,
+    'resourceId': resource_id
+    }
+    return drive.auth.service.channels().stop(body=body).execute()
+
+def getChangeById (drive, change_id):
+    if not drive:
+        drive=getDrive()
+    if drive.auth.access_token_expired:
+        drive.auth.Refresh() 
+    """Print a single Change resource information.
+
+    Args:
+    service: Drive API service instance.
+    change_id: ID of the Change resource to retrieve.
+    """
+    try:
+        change = drive.auth.service.changes().get(changeId=change_id).execute()
+        return change
+    except errors.HttpError, error:
+        web.app.logger.exception(error) 
+        return None

cps/templates/config_edit.html

@@ -7,6 +7,45 @@
       <label for="config_calibre_dir">{{_('Location of Calibre database')}}</label>
       <input type="text" class="form-control" name="config_calibre_dir" id="config_calibre_dir" value="{% if content.config_calibre_dir != None %}{{ content.config_calibre_dir }}{% endif %}" autocomplete="off">
     </div>
+    <div class="form-group required">
+      <label for="config_use_google_drive">{{_('Use google drive?')}}</label>
+      <input type="checkbox" id="config_use_google_drive" class="form-control" name="config_use_google_drive" id="config_use_google_drive" {% if content.config_use_google_drive %}checked{% endif %} >
+    </div>
+    <div id="gdrive_settings">
+      <div class="form-group required">
+        <label for="config_google_drive_client_id">{{_('Client id')}}</label>
+        <input type="text" class="form-control" name="config_google_drive_client_id" id="config_google_client_id" value="{% if content.config_google_drive_client_id %}{{content.config_google_drive_client_id}}{% endif %}" autocomplete="off">
+      </div>
+      <div class="form-group required">
+        <label for="config_google_drive_client_secret">{{_('Client secret')}}</label>
+        <input type="text" class="form-control" name="config_google_drive_client_secret" id="config_google_drive_client_secret" value="{% if content.config_google_drive_client_secret %}{{content.config_google_drive_client_secret}}{% endif %}" autocomplete="off">
+      </div>
+      <div class="form-group required">
+        <label for="config_google_drive_calibre_url_base">{{_('Calibre Base URL')}}</label>
+        <input type="text" class="form-control" name="config_google_drive_calibre_url_base" id="config_google_drive_calibre_url_base" value="{% if content.config_google_drive_calibre_url_base %}{{content.config_google_drive_calibre_url_base}}{% endif %}" autocomplete="off">
+      </div>
+      <div class="form-group required">
+        <label for="config_google_drive_folder">{{_('Google drive Calibre folder')}}</label>
+        <input type="text" class="form-control" name="config_google_drive_folder" id="config_google_drive_folder" value="{% if content.config_google_drive_folder %}{{content.config_google_drive_folder}}{% endif %}" autocomplete="off" required>
+      </div>
+      {% if show_authenticate_google_drive %}
+        <div class="form-group required">
+          <a href="{{ url_for('authenticate_google_drive') }}" class="btn btn-primary">Authenticate Google Drive</a>
+        </div>
+      {% else %}
+        {% if content.config_google_drive_watch_changes_response %}
+        <label for="config_google_drive_watch_changes_response">{{_('Metadata Watch Channel ID')}}</label>
+        <div class="form-group input-group required">
+          <input type="text" class="form-control" name="config_google_drive_watch_changes_response" id="config_google_drive_watch_changes_response" value="{{ content.config_google_drive_watch_changes_response['id'] }} expires on {{ content.config_google_drive_watch_changes_response['expiration'] |  strftime }}" autocomplete="off" disabled="">
+          <span class="input-group-btn">
+          <a href="{{ url_for('revoke_watch_gdrive') }}" class="btn btn-primary">Revoke</a>
+          </span>
+          {% else %}
+          <a href="{{ url_for('watch_gdrive') }}" class="btn btn-primary">Enable watch of metadata.db</a>
+          {% endif %}
+        </div>
+        {% endif %}
+    </div>
     <div class="form-group">
       <label for="config_port">{{_('Server Port')}}</label>
       <input type="number" min="1" max="65535" class="form-control" name="config_port" id="config_port" value="{% if content.config_port != None %}{{ content.config_port }}{% endif %}" autocomplete="off" required>
@@ -80,3 +119,22 @@
   </form>
 </div>
 {% endblock %}
+{% block js %}
+<script>
+$(document).ready(function() {
+  $('#config_use_google_drive').trigger("change");
+});
+$('#config_use_google_drive').change(function(){
+    formInputs=$("#gdrive_settings :input");
+    isChecked=this.checked;
+    formInputs.each(function(formInput) {
+      $(this).prop('required',isChecked);
+    });
+    if (this.checked) {
+      $('#gdrive_settings').show();
+    } else {
+      $('#gdrive_settings').hide();
+    }
+});
+</script>
+{% endblock %}

cps/ub.py

@@ -11,6 +11,7 @@ import traceback
 import logging
 from werkzeug.security import generate_password_hash
 from flask_babel import gettext as _
+import json
 
 dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "app.db")
 engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False)
@@ -269,6 +270,12 @@ class Settings(Base):
     config_anonbrowse = Column(SmallInteger, default=0)
     config_public_reg = Column(SmallInteger, default=0)
     config_default_role = Column(SmallInteger, default=0)
+    config_use_google_drive = Column(Boolean)
+    config_google_drive_client_id = Column(String)
+    config_google_drive_client_secret = Column(String)
+    config_google_drive_folder = Column(String)
+    config_google_drive_calibre_url_base = Column(String)
+    config_google_drive_watch_changes_response = Column(String)
 
     def __repr__(self):
         pass
@@ -295,7 +302,17 @@ class Config:
         self.config_anonbrowse = data.config_anonbrowse
         self.config_public_reg = data.config_public_reg
         self.config_default_role = data.config_default_role
-        if self.config_calibre_dir is not None:
+        self.config_use_google_drive = data.config_use_google_drive
+        self.config_google_drive_client_id = data.config_google_drive_client_id
+        self.config_google_drive_client_secret = data.config_google_drive_client_secret
+        self.config_google_drive_calibre_url_base = data.config_google_drive_calibre_url_base
+        self.config_google_drive_folder = data.config_google_drive_folder
+        if data.config_google_drive_watch_changes_response:
+            self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response)
+        else:
+            self.config_google_drive_watch_changes_response=None
+
+        if (self.config_calibre_dir is not None and not self.config_use_google_drive) or os.path.exists(self.config_calibre_dir + '/metadata.db'):
             self.db_configured = True
         else:
             self.db_configured = False
@@ -379,6 +396,17 @@ def migrate_Database():
         conn.execute("ALTER TABLE Settings ADD column `config_anonbrowse` SmallInteger DEFAULT 0")
         conn.execute("ALTER TABLE Settings ADD column `config_public_reg` SmallInteger DEFAULT 0")
         session.commit()
+
+    try:
+        session.query(exists().where(Settings.config_use_google_drive)).scalar()
+    except exc.OperationalError:
+        conn = engine.connect()
+        conn.execute("ALTER TABLE Settings ADD column `config_use_google_drive` INTEGER DEFAULT 0")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_id` String DEFAULT ''")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_secret` String DEFAULT ''")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_calibre_url_base` INTEGER DEFAULT 0")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_folder` String DEFAULT ''")
+        conn.execute("ALTER TABLE Settings ADD column `config_google_drive_watch_changes_response` String DEFAULT ''")
     try:
         session.query(exists().where(Settings.config_default_role)).scalar()
         session.commit()

cps/web.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
+from pydrive.auth import GoogleAuth
+
 import mimetypes
 import logging
 from logging.handlers import RotatingFileHandler
 from tempfile import gettempdir
 import textwrap
 from flask import Flask, render_template, session, request, Response, redirect, url_for, send_from_directory, \
-    make_response, g, flash, abort
+        make_response, g, flash, abort, send_file
 import ub
 from ub import config
 import helper
@@ -41,7 +43,17 @@ import re
 import db
 from shutil import move, copyfile
 from tornado.ioloop import IOLoop
+import shutil
 import StringIO
+from shutil import move
+import gdriveutils
+import io
+import hashlib
+import threading
+
+import time
+
+current_milli_time = lambda: int(round(time.time() * 1000))
 
 try:
     from wand.image import Image
@@ -52,10 +64,64 @@ except ImportError, e:
 from cgi import escape
 
 # Global variables
+gdrive_watch_callback_token='target=calibreweb-watch_files'
 global_task = None
 
+def md5(fname):
+    hash_md5 = hashlib.md5()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+class Singleton:
+    """
+    A non-thread-safe helper class to ease implementing singletons.
+    This should be used as a decorator -- not a metaclass -- to the
+    class that should be a singleton.
+
+    The decorated class can define one `__init__` function that
+    takes only the `self` argument. Also, the decorated class cannot be
+    inherited from. Other than that, there are no restrictions that apply
+    to the decorated class.
+
+    To get the singleton instance, use the `Instance` method. Trying
+    to use `__call__` will result in a `TypeError` being raised.
+
+    """
+
+    def __init__(self, decorated):
+        self._decorated = decorated
+
+    def Instance(self):
+        """
+        Returns the singleton instance. Upon its first call, it creates a
+        new instance of the decorated class and calls its `__init__` method.
+        On all subsequent calls, the already created instance is returned.
+
+        """
+        try:
+            return self._instance
+        except AttributeError:
+            self._instance = self._decorated()
+            return self._instance
+
+    def __call__(self):
+        raise TypeError('Singletons must be accessed through `Instance()`.')
+
+    def __instancecheck__(self, inst):
+        return isinstance(inst, self._decorated)
+
+@Singleton
+class Gauth:
+    def __init__(self):
+        self.auth=GoogleAuth(settings_file='settings.yaml')
+
+@Singleton
+class Gdrive:
+    def __init__(self):
+        self.drive=gdriveutils.getDrive(Gauth.Instance().auth)
 
-# Proxy Helper class
 class ReverseProxied(object):
     """Wrap the application in this middleware and configure the
     front-end server to add these headers, to let you quietly bind
@@ -133,6 +199,9 @@ lm.anonymous_user = ub.Anonymous
 app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT'
 db.setup_db()
 
+def is_gdrive_ready():
+    return os.path.exists('settings.yaml') and os.path.exists('gdrive_credentials')
+
 @babel.localeselector
 def get_locale():
     # if a user is logged in, use the locale from the user settings
@@ -187,6 +256,12 @@ def authenticate():
         'You have to login with proper credentials', 401,
         {'WWW-Authenticate': 'Basic realm="Login Required"'})
 
+def updateGdriveCalibreFromLocal():
+    gdriveutils.backupCalibreDbAndOptionalDownload(Gdrive.Instance().drive)
+    gdriveutils.copyToDrive(Gdrive.Instance().drive, config.config_calibre_dir, False, True)
+    for x in os.listdir(config.config_calibre_dir):
+        if os.path.isdir(os.path.join(config.config_calibre_dir,x)):
+            shutil.rmtree(os.path.join(config.config_calibre_dir,x))
 
 def requires_basic_auth_if_no_ano(f):
     @wraps(f)
@@ -286,6 +361,17 @@ def formatdate(val):
     formatdate = datetime.datetime.strptime(conformed_timestamp[:15], "%Y%m%d %H%M%S")
     return format_date(formatdate, format='medium',locale=get_locale())
 
+@app.template_filter('strftime')
+def timestamptodate(date, fmt=None):
+    date=datetime.datetime.fromtimestamp(
+        int(date)/1000
+    )
+    native = date.replace(tzinfo=None)
+    if fmt:
+        format=fmt
+    else:
+        format='%d %m %Y - %H:%S'
+    return native.strftime(format)
 
 def admin_required(f):
     """
@@ -668,6 +754,13 @@ def get_opds_download_link(book_id, format):
     file_name = book.title
     if len(book.authors) > 0:
         file_name = book.authors[0].name + '-' + file_name
+
+    if config.config_use_google_drive:
+        df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format))
+        download_url = df.metadata.get('downloadUrl')
+        resp, content = df.auth.Get_Http_Object().request(download_url)
+        response=send_file(io.BytesIO(content))
+    else:
         file_name = helper.get_valid_filename(file_name)
         response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
     response.headers["Content-Disposition"] = "attachment; filename=\"%s.%s\"" % (data.name, format)
@@ -802,7 +895,9 @@ def hot_books(page):
     hot_books = all_books.offset(off).limit(config.config_books_per_page)
     entries = list()
     for book in hot_books:
-        entries.append(db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first())
+        entry=db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first()
+        if entry:
+            entries.append(entry)
     numBooks = entries.__len__()
     pagination = Pagination(page, config.config_books_per_page, numBooks)
     return render_title_template('index.html', random=random, entries=entries, pagination=pagination,
@@ -1037,6 +1132,99 @@ def stats():
                                  categorycounter=categorys, seriecounter=series, title=_(u"Statistics"))
 
 
+#@app.route("/load_gdrive")
+#@login_required
+#@admin_required
+#def load_all_gdrive_folder_ids():
+#    books=db.session.query(db.Books).all()
+#    for book in books:
+#        gdriveutils.getFolderId(book.path, Gdrive.Instance().drive)
+#    return 
+
+@app.route("/gdrive/authenticate")
+@login_required
+@admin_required
+def authenticate_google_drive():
+    authUrl=Gauth.Instance().auth.GetAuthUrl()
+    return redirect(authUrl)
+
+@app.route("/gdrive/callback")
+def google_drive_callback():
+    auth_code = request.args.get('code')
+    credentials = Gauth.Instance().auth.flow.step2_exchange(auth_code)
+    with open('gdrive_credentials' ,'w') as f:
+        f.write(credentials.to_json())
+    return redirect(url_for('configuration'))
+
+@app.route("/gdrive/watch/subscribe")
+@login_required
+@admin_required
+def watch_gdrive():
+    if not config.config_google_drive_watch_changes_response:
+        address = '%scalibre-web/gdrive/watch/callback' % config.config_google_drive_calibre_url_base
+        notification_id=str(uuid4())
+        result = gdriveutils.watchChange(Gdrive.Instance().drive, notification_id,
+                               'web_hook', address, gdrive_watch_callback_token, current_milli_time() + 604800*1000)
+        print (result)
+        settings = ub.session.query(ub.Settings).first()
+        settings.config_google_drive_watch_changes_response=json.dumps(result)
+        ub.session.merge(settings)
+        ub.session.commit()
+        settings = ub.session.query(ub.Settings).first()
+        config.loadSettings()
+
+        print (settings.config_google_drive_watch_changes_response)
+
+    return redirect(url_for('configuration'))
+
+@app.route("/gdrive/watch/revoke")
+@login_required
+@admin_required
+def revoke_watch_gdrive():
+    last_watch_response=config.config_google_drive_watch_changes_response
+    if last_watch_response:
+        response=gdriveutils.stopChannel(Gdrive.Instance().drive, last_watch_response['id'], last_watch_response['resourceId'])
+        settings = ub.session.query(ub.Settings).first()
+        settings.config_google_drive_watch_changes_response=None
+        ub.session.merge(settings)
+        ub.session.commit()
+        config.loadSettings()
+    return redirect(url_for('configuration'))
+
+@app.route("/gdrive/watch/callback", methods=['GET', 'POST'])
+def on_received_watch_confirmation():
+    app.logger.info (request.headers)
+    if request.headers.get('X-Goog-Channel-Token') == gdrive_watch_callback_token \
+        and request.headers.get('X-Goog-Resource-State') == 'change' \
+        and request.data:
+
+        data=request.data
+
+        def updateMetaData():
+            app.logger.info ('Change received from gdrive')
+            app.logger.info (data) 
+            try:
+                j=json.loads(data)
+                app.logger.info ('Getting change details') 
+                response=gdriveutils.getChangeById(Gdrive.Instance().drive, j['id'])
+                app.logger.info (response)
+                if response:
+                    dbpath = os.path.join(config.config_calibre_dir, "metadata.db")
+                    if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != md5(dbpath):
+                        app.logger.info ('Database file updated')
+                        copyfile (dbpath, config.config_calibre_dir + "/metadata.db_" + str(current_milli_time()))
+                        app.logger.info ('Backing up existing and downloading updated metadata.db')
+                        gdriveutils.downloadFile(Gdrive.Instance().drive, None, "metadata.db", config.config_calibre_dir + "/tmp_metadata.db")
+                        app.logger.info ('Setting up new DB')
+                        os.rename(config.config_calibre_dir + "/tmp_metadata.db", dbpath)
+                        db.setup_db()
+            except Exception, e:
+                app.logger.exception(e)
+
+        updateMetaData()
+    return ''
+
+
 @app.route("/shutdown")
 @login_required
 @admin_required
@@ -1173,8 +1361,15 @@ def advanced_search():
 @app.route("/cover/<path:cover_path>")
 @login_required_if_no_ano
 def get_cover(cover_path):
+    if config.config_use_google_drive:
+        df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg')
+        download_url = df.metadata.get('webContentLink')
+        return redirect(download_url)
+    else:
         return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg")
+    resp.headers['Content-Type']='image/jpeg'
 
+    return resp
 
 @app.route("/opds/thumb_240_240/<path:book_id>")
 @app.route("/opds/cover_240_240/<path:book_id>")
@@ -1183,7 +1378,12 @@ def get_cover(cover_path):
 @requires_basic_auth_if_no_ano
 def feed_get_cover(book_id):
     book = db.session.query(db.Books).filter(db.Books.id == book_id).first()
-    return send_from_directory(os.path.join(config.config_calibre_dir, book.path), "cover.jpg")
+    if config.config_use_google_drive:
+        df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg')
+        download_url = df.metadata.get('webContentLink')
+        return redirect(download_url)
+    else:
+        return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg")
 
 def render_read_books(page, are_read, as_xml=False):
     readBooks=ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id)).filter(ub.ReadBook.is_read == True).all()
@@ -1308,8 +1508,13 @@ def get_download_link(book_id, format):
         if len(book.authors) > 0:
             file_name = book.authors[0].name + '-' + file_name
         file_name = helper.get_valid_filename(file_name)
-        response = make_response(
+        if config.config_use_google_drive:
-            send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
+            df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format))
+            download_url = df.metadata.get('downloadUrl')
+            resp, content = df.auth.Get_Http_Object().request(download_url)
+            response=send_file(io.BytesIO(content))
+        else:
+            response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format))
         try:
             response.headers["Content-Type"] = mimetypes.types_map['.' + format]
         except:
@@ -1682,6 +1887,35 @@ def configuration_helper(origin):
             if content.config_calibre_dir != to_save["config_calibre_dir"]:
                 content.config_calibre_dir = to_save["config_calibre_dir"]
                 db_change = True
+        ##Google drive setup
+        create_new_yaml=False
+        if "config_google_drive_client_id" in to_save:
+            if content.config_google_drive_client_id != to_save["config_google_drive_client_id"]:
+                content.config_google_drive_client_id = to_save["config_google_drive_client_id"]
+                create_new_yaml=True
+        if "config_google_drive_client_secret" in to_save:
+            if content.config_google_drive_client_secret != to_save["config_google_drive_client_secret"]:
+                content.config_google_drive_client_secret = to_save["config_google_drive_client_secret"]
+                create_new_yaml=True
+        if "config_google_drive_calibre_url_base" in to_save:
+            if content.config_google_drive_calibre_url_base != to_save["config_google_drive_calibre_url_base"]:
+                content.config_google_drive_calibre_url_base = to_save["config_google_drive_calibre_url_base"]
+                create_new_yaml=True
+        if ("config_use_google_drive" in to_save and not content.config_use_google_drive) or ("config_use_google_drive" not in to_save and content.config_use_google_drive):
+            content.config_use_google_drive = "config_use_google_drive" in to_save
+            db_change = True
+            if not content.config_use_google_drive:
+                create_new_yaml=False
+        if create_new_yaml:
+            with open('settings.yaml', 'w') as f:
+                with open('gdrive_template.yaml' ,'r') as t:
+                    f.write(t.read() % {'client_id' : content.config_google_drive_client_id, 'client_secret' : content.config_google_drive_client_secret,
+                     "redirect_uri" : content.config_google_drive_calibre_url_base + 'gdrive/callback'})
+        if "config_google_drive_folder" in to_save:
+            if content.config_google_drive_folder != to_save["config_google_drive_folder"]:
+                content.config_google_drive_folder = to_save["config_google_drive_folder"]
+                db_change = True
+        ##
         if "config_port" in to_save:
             if content.config_port != int(to_save["config_port"]):
                 content.config_port = int(to_save["config_port"])
@@ -1720,6 +1954,8 @@ def configuration_helper(origin):
         if "passwd_role" in to_save:
             content.config_default_role = content.config_default_role + ub.ROLE_PASSWD
         try:
+            if content.config_use_google_drive and is_gdrive_ready() and not os.path.exists(config.config_calibre_dir + "/metadata.db"): 
+                    gdriveutils.downloadFile(Gdrive.Instance().drive, None, "metadata.db", config.config_calibre_dir + "/metadata.db")
             if db_change:
                 if config.db_configured:
                     db.session.close()
@@ -1751,6 +1987,7 @@ def configuration_helper(origin):
         if origin:
             success = True
     return render_title_template("config_edit.html", origin=origin, success=success, content=config,
+                                 show_authenticate_google_drive=not is_gdrive_ready(),
                                  title=_(u"Basic Configuration"))
 
 
@@ -2163,6 +2400,8 @@ def edit_book(book_id):
                 author_names.append(author.name)
             for b in edited_books_id:
                 helper.update_dir_stucture(b, config.config_calibre_dir)
+            if config.config_use_google_drive:
+                updateGdriveCalibreFromLocal()
             if "detail_view" in to_save:
                 return redirect(url_for('show_book', id=book.id))
             else:
@@ -2242,6 +2481,9 @@ def upload():
         author_names = []
         for author in db_book.authors:
             author_names.append(author.name)
+        if config.config_use_google_drive:
+            if not current_user.role_edit() and not current_user.role_admin():
+                updateGdriveCalibreFromLocal()
     cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all()
     if current_user.role_edit() or current_user.role_admin():
         return render_title_template('book_edit.html', book=db_book, authors=author_names, cc=cc,

gdrive_template.yaml

@@ -0,0 +1,14 @@
+client_config_backend: settings
+client_config:
+  client_id: %(client_id)s
+  client_secret: %(client_secret)s
+  redirect_uri: %(redirect_uri)s
+  
+save_credentials: True
+save_credentials_backend: file
+save_credentials_file: gdrive_credentials
+
+get_refresh_token: True
+
+oauth_scope:
+  - https://www.googleapis.com/auth/drive