""" Flask Documentation: http://flask.pocoo.org/docs/ Jinja2 Documentation: http://jinja.pocoo.org/2/documentation/ Werkzeug Documentation: http://werkzeug.pocoo.org/documentation/ This file creates your application. """ from app import app, db, socketio, DOMAIN from flask import Flask, Response, session, render_template, request, redirect, url_for, flash, send_from_directory, jsonify, abort import json from sqlalchemy.sql.expression import func, select from sqlalchemy.sql import except_ from app.forms import UploadForm, EditForm, SearchForm, ChatForm, StackForm, AddtoStackForm, EditStackForm from app.models import Book, BookSchema, Author, AuthorSchema, Stack, StackSchema, UserIns, Chat, ChatSchema, Instance, Potential from app.cover import get_cover from app.extractText import extract_text from os import environ from flask_socketio import SocketIO, emit from weasyprint import HTML import datetime import time from csv import DictWriter, DictReader import io from sqlalchemy.inspection import inspect #import autocomplete import sys import os from werkzeug.utils import secure_filename # import sqlite3 ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'epub', 'chm', 'mobi']) author_schema = AuthorSchema() authors_schema = AuthorSchema(many=True) book_schema = BookSchema() books_schema = BookSchema(many=True) stack_schema = StackSchema() stacks_schema = StackSchema(many=True) chat_schema = ChatSchema() chats_schema = ChatSchema(many=True) def allowed_file(filename): return '.' in filename and \ filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS ### # Routing for your application. ### @app.route('/', methods= ['POST','GET']) def home(): chat_form = ChatForm() chat_messages = db.session.query(Chat).all() username = 'librarian' # if request.method == 'POST': # if chat_form.validate_on_submit(): # message = chat_form.message.data # msg = Chat(message) # db.session.add(msg) # db.session.commit() #client = request.remote_addr server = request.host if request.environ.get('HTTP_X_FORWARDED_FOR') is None: client =request.environ['REMOTE_ADDR'] else: client = request.environ['HTTP_X_FORWARDED_FOR'] return render_template('home.html',domain=DOMAIN,chat=chat_messages, channel = 1, username=username, client=client, server=server) @app.route('/hello/') def hello(name): return "Hello " + name @app.route('/about/') def about(): """Render the website's about page.""" return render_template('about.html', name="Mary Jane") @app.route('/uploads/') def uploaded_file(filename): book = Book.query.filter_by(file=filename).first() i = Instance(request.host, "download") existing_ip = db.session.query(Instance).filter_by(ip=request.host).first() if existing_ip: i.name = existing_ip.name book.instances.append(i) db.session.commit() return send_from_directory(app.config['UPLOAD_FOLDER'], filename) @app.route('/uploads/cover/') def uploaded_file_cover(filename): return send_from_directory(app.config['UPLOAD_FOLDER_COVER'], filename) @app.route('/updates', methods=['POST', 'GET']) def get_updates(): userin = UserIns.query.filter_by(title="lastViewed").first() allbooks = db.session.query(Book).all() id = len(allbooks) latest_upload = allbooks[-1] return "This is the XPPL ~ Library XPUB ~ Updates / / / / / / / Last viewed: " + userin.info + " / / / / / / / " + str(len(allbooks)) + " Books online "+ " / / / / / / / " + "Latest upload: " + latest_upload.title @app.route('/scape', methods=['POST', 'GET']) def scape(): if request.method == 'POST': data = request.form book = Book.query.get(data['id']) print(book.scapeX) book.scapeX = data['x'] book.scapeY = data['y'] db.session.commit() books = db.session.query(Book).all() # or you could have used User.query.all() all_instances = db.session.query(Instance).all() instances = [] for instance in all_instances: exists = False for existing_inst in instances: if existing_inst.name == instance.name: exists = True break else: exists = False if not exists: instances.append(instance) return render_template('scape.html', books=books, instances=instances) @app.route('/books_grid') def show_books_grid(): books = db.session.query(Book).all() # or you could have used User.query.all() return render_template('show_books_grid.html', books=books) @app.route('/books/') def show_book_by_id(id): book = Book.query.get(id) all_instances = db.session.query(Instance).all() previousbook = Book.query.filter_by(id=id - 1).first() nextbook = Book.query.filter_by(id=id + 1).first() allbooks = db.session.query(Book).all() edge = len(allbooks) if id == 1: previousbook = None if id == edge: nextbook = None userin = UserIns.query.filter_by(title="lastViewed").first() if userin != None: userin.info = book.title db.session.commit() else: user_info = UserIns("lastViewed", book.title) db.session.add(user_info) db.session.commit() if not book: return render_template('red_link.html', id=id) else: return render_template('show_book_detail.html', book=book, previousbook = previousbook, nextbook = nextbook, all_instances=all_instances) @app.route('/books//delete', methods=['POST', 'GET']) def remove_book_by_id(id): book_to_edit = Book.query.filter_by(id=id).first() title = book_to_edit.title Book.query.filter_by(id=id).delete() #author_table = Author.query.filter_by(book_id=book_to_edit.id).delete() db.session.commit() flash("%s deleted from library" % (title)) return redirect(url_for('show_books')) @app.route('/books//edit', methods=['POST', 'GET']) def edit_book_by_id(id): book_to_edit = Book.query.filter_by(id=id).first() user_form = EditForm(title = book_to_edit.title, author =book_to_edit.authors, category = book_to_edit.category, year_published= book_to_edit.year_published, message= book_to_edit.message, sameness=book_to_edit.sameness, gender=book_to_edit.gender, diversity=book_to_edit.diversity, who=book_to_edit.who) if request.method == 'POST': if user_form.validate_on_submit(): # on submit, check fields title = user_form.title.data input_authors = user_form.author.data category = user_form.category.data year_published = user_form.year_published.data message = user_form.message.data sameness = user_form.sameness.data gender = user_form.gender.data diversity = user_form.diversity.data who = user_form.who.data if year_published=="": year_published = None book = Book.query.filter_by(id=id).first() book.title = title book.category = category book.year_published = year_published book.message = message book.sameness = sameness book.gender = gender book.diversity = diversity book.who = who #authors update book.authors.clear() for i, author in enumerate(input_authors): author_name = author.get("author_name") if author_name: a = db.session.query(Author).filter_by(author_name=author_name).first() if a == None: a = Author(author_name=author_name) db.session.add(a) book.authors.append(a) i = Instance(request.host, "edit") existing_ip = db.session.query(Instance).filter_by(ip=request.host).first() if existing_ip: i.name = existing_ip.name book.instances.append(i) # editing / uploading new file if user_form.file.data: file = request.files['file'] if file.filename == '': flash('No selected file') return redirect(request.url) if file and allowed_file(file.filename): allbooks = db.session.query(Book).all() id = book.id filename = str(id) + "_" + secure_filename(file.filename) fullpath = os.path.join(app.config['UPLOAD_FOLDER'], filename) name, file_extension = os.path.splitext(filename) file.save(fullpath) book.cover = get_cover(fullpath, name) book.file = filename else: flash('allowed file formats: %s' % ALLOWED_EXTENSIONS) db.session.commit() flash("%s updated" % (title)) return redirect(url_for('show_book_by_id', id=id)) return render_template('edit_book_detail.html', book=book_to_edit, form=user_form) @app.route('/add-book', methods=['POST', 'GET']) def add_book(): upload_form = UploadForm() allbooks = db.session.query(Book).all() books_all = len(allbooks) allauthors = db.session.query(Author).all() authors_all = len(allauthors) stacks_all = [s.stack_name for s in db.session.query(Stack.stack_name)] categories = [r.category for r in db.session.query(Book.category).distinct()] allpotential = db.session.query(Book).filter(Book.file.contains('potential.pdf')).all() books_potential = len(allpotential) earliest = db.session.query(func.min(Book.year_published)).scalar() latest = db.session.query(func.max(Book.year_published)).scalar() if request.method == 'POST': if upload_form.validate_on_submit(): #get data from form title = upload_form.title.data authors = upload_form.author.data category = upload_form.category.data message = upload_form.message.data year_published = upload_form.year_published.data sameness = upload_form.sameness.data gender = upload_form.gender.data diversity = upload_form.diversity.data who = upload_form.who.data if year_published=="": year_published = None #if upload with file if upload_form.upload.data: # check if the post request has the file part if 'file' not in request.files: flash('No file part') return redirect(request.url) file = request.files['file'] # if user does not select file, browser also # submit a empty part without filename if file.filename == '': flash('No selected file') return redirect(request.url) if file and allowed_file(file.filename): allbooks = db.session.query(Book).all() id = len(allbooks)+1 filename = str(id) + "_" + secure_filename(file.filename) fullpath = os.path.join(app.config['UPLOAD_FOLDER'], filename) name, file_extension = os.path.splitext(filename) file.save(fullpath) try: cover = get_cover(fullpath, name) except: cover = '' extract_text(fullpath, name) else: flash('allowed file formats: %s' % ALLOWED_EXTENSIONS) #if upload without file -> wishform, with potential PDF if upload_form.wish.data: #pdf generator filename = 'potential.pdf' file_extension = '.pdf' cover= '' ptitle = upload_form.title.data pbook = Potential(ptitle) db.session.add(pbook) db.session.commit() pbooks = Potential.query.all() template = 'app/templates/potential_pdf.html' html_string = render_template('potential_pdf.html', pbooks = pbooks) html = HTML(string=html_string) html.write_pdf(target='app/uploads/potential.pdf'); book = Book(title, filename, cover, file_extension, category, year_published, message, sameness, diversity, gender, who) db.session.add(book) for author in authors: author_name = author.get("author_name") if author_name: a = db.session.query(Author).filter_by(author_name=author_name).first() if a == None: a = Author(author_name=author_name) db.session.add(a) book.authors.append(a) i = Instance(request.host, "add") existing_ip = db.session.query(Instance).filter_by(ip=request.host).first() if existing_ip: i.name = existing_ip.name book.instances.append(i) db.session.commit() flash("%s added to the library" % (title)) return redirect(url_for('show_books')) flash_errors(upload_form) return render_template('add_book.html', form=upload_form, books_all=books_all, authors_all=authors_all, categories=categories, stacks_all=stacks_all, books_potential=books_potential, earliest=earliest, latest=latest) # Flash errors from the form if validation fails def flash_errors(form): for field, errors in form.errors.items(): for error in errors: flash(u"Error in the %s field - %s" % ( getattr(form, field).label.text, error )) #Authors @app.route('/authors/') def show_author_by_id(id): author = Author.query.get(id) if not author: abort (404) else: return render_template('show_author_detail.html', author=author) @app.route('/authors//edit', methods=['POST', 'GET']) def edit_author_by_id(id): return "Ask the programmer." ##stacks @app.route('/stacks') def show_stacks(): stacks = db.session.query(Stack).all() return render_template('show_stacks.html', stacks=stacks) @app.route('/stacks/add_stack', methods=['POST', 'GET']) def add_stack(): form = StackForm() stacks = db.session.query(Stack).all() if form.validate_on_submit(): stack_name = form.stack_name.data stack_description = form.stack_description.data stack_author = form.stack_author.data stack = Stack(stack_name, stack_description, stack_author) if form.stack_name.data: stack = Stack(stack_name, stack_description, stack_author) db.session.add(stack) stacks = db.session.query(Stack).all() return redirect(url_for('show_stacks')) flash("%s stack created" % (stack_name)) return render_template('add_stack.html', stacks=stacks, form=form) @app.route('/stacks/tab/', methods=['POST', 'GET']) def show_stack_in_tab(id): return show_stack_by_id(id, is_tab=True) @app.route('/stacks/', methods=['POST', 'GET']) def show_stack_by_id(id, is_tab=False): stack = Stack.query.get(id) if not stack: return render_template('add_stack.html', stacks=stacks, form=form) else: if is_tab == False: return render_template('show_stack_detail.html', stack=stack) else: return render_template('show_stack_detail_tab.html', stack=stack) @app.route('/stacks//delete', methods=['POST', 'GET']) def remove_stack_by_id(id): Stack.query.filter_by(id=id).delete() db.session.commit() return redirect(url_for('show_stacks')) @app.route('/stacks//edit', methods=['POST', 'GET']) def edit_stack_by_id(id): stack = Stack.query.filter_by(id=id).first() form = EditStackForm(edit_stack_name = stack.stack_name, edit_stack_description = stack.stack_description) if request.method == 'POST': if form.validate_on_submit(): stack_name = form.edit_stack_name.data stack_description = form.edit_stack_description.data stack.stack_name = stack_name stack.stack_description = stack_description db.session.commit() return redirect(url_for('show_stack_by_id', id=id)) return render_template('edit_stack_detail.html', stack=stack, form=form) @app.route('/instances', methods=['POST', 'GET']) def show_instances(): all_instances = db.session.query(Instance).all() instances = [] for instance in all_instances: exists = False for existing_inst in instances: if existing_inst.name == instance.name: exists = True break else: exists = False if not exists: instances.append(instance) if request.method == 'POST': for item in request.form.items(): for i, itm in enumerate(item): if i == 0: oldname = itm if i == 1: name = itm all_instances = db.session.query(Instance).filter_by(name=oldname).all() for instance in all_instances: instance.name = name print(oldname) print(name) db.session.commit() return render_template('show_instances.html', instances=instances) @app.route('/stacks//remove/', methods=['POST', 'GET']) def remove_from_stack(bookid, stackid): book = Book.query.get(bookid) stack = Stack.query.get(stackid) if book not in stack.books: return render_template('show_book_detail.html', book=book) stack.books.remove(book) db.session.commit() return render_template('show_book_detail.html', book=book) ## search view = ['1'] @app.route('/books', methods= ['POST','GET']) def show_books(): books = db.session.query(Book).order_by(Book.title) search = SearchForm(request.form) view.append('1') viewby = '1' if search.grid.data: viewby = '2' view.append('2') return render_template ('show_books_grid.html', books=books, form=search) if search.listview.data: viewby = '1' view.append('1') return render_template ('show_books.html', books=books, form=search) if request.method == 'POST': newmsg = 'searched for: ' + search.search.data socketio.emit('channel-' + str(1), { 'username': 'Search form', 'text': search.search.data, 'time': str(datetime.datetime.utcnow().strftime("%d.%m.%Y %H:%M"))}, broadcast=True) # Save message my_new_chat = Chat(message=newmsg) db.session.add(my_new_chat) try: db.session.commit() except: db.session.rollback() return redirect((url_for('search_results', searchtype=search.select.data, query=search.search.data, viewby=viewby))) return render_template('show_books.html', books=books, form=search) @app.route('/search///', methods=['POST', 'GET']) def search_results(searchtype, query, viewby): search = SearchForm(request.form, search=query) results=Book.query.filter(Book.title.contains(query)).order_by(Book.title) allbooks = set(Book.query.all()) viewby = view[-1] if searchtype == 'Title': results=Book.query.filter(Book.title.contains(query)).order_by(Book.title) if searchtype == 'Category': results=Book.query.filter(Book.category.contains(query)).order_by(Book.title) if searchtype== 'Author': results=db.session.query(Book).join(Book.authors).filter(Author.author_name.contains(query)).order_by(Book.title) if searchtype== 'Stack': results=db.session.query(Book).join(Book.stacks).filter(Stack.stack_name.contains(query)).order_by(Book.title) if searchtype== 'Outliers': results=Book.query.filter(Book.sameness > 50).order_by(Book.title) if searchtype== 'All': # results=Book.query.whoosh_search(query) results=Book.query.filter(Book.title.contains(query)) results=results.union(Book.query.filter(Book.category.contains(query))) results=results.union(Book.query.filter(Book.year_published.contains(query))) results=results.union(db.session.query(Book).join(Book.authors).filter(Author.author_name.contains(query))) results=results.union(db.session.query(Book).join(Book.stacks).filter(Stack.stack_name.contains(query))).order_by(Book.title) if results.count() == 0: books = Book.query.filter(Book.file.like('potential.pdf')) upload_form = UploadForm(title= query, author='') return render_template('red_link.html', form=upload_form, title=query, books=books) count = results.count() whole = Book.query.count() percentage = float(count / whole * 100) fbooks = set(results) books_all = allbooks - fbooks if search.listview.data: view.append('1') return render_template('results.html', books=results, form=search, query=query, books_all=books_all, searchtype=search.select.data, count = count, whole = whole, percentage = percentage) if search.grid.data: view.append('2') return render_template('results_grid.html', books=results, form=search, query=query, books_all=books_all, searchtype=search.select.data, count = count, whole = whole, percentage = percentage) if request.method == 'POST': newmsg = 'searched for: ' + search.search.data socketio.emit('channel-' + str(1), { 'username': 'Search form', 'text': search.search.data, 'time': str(datetime.datetime.utcnow().strftime("%d.%m.%Y %H:%M"))}, broadcast=True) # Save message my_new_chat = Chat(message=newmsg) db.session.add(my_new_chat) try: db.session.commit() except: db.session.rollback() query = search.search.data results = [] if viewby == '1': print (view[-1]) return redirect((url_for('search_results', searchtype=search.select.data, query=search.search.data, viewby=viewby))) else: return redirect((url_for('search_results', searchtype=search.select.data, query=search.search.data, viewby=viewby))) if viewby == '2': return render_template('results_grid.html', form=search, books=results, books_all=books_all, searchtype=search.select.data, query=query, count = count, whole = whole, percentage = percentage) else: return render_template('results.html', form=search, books=results, books_all=books_all, searchtype=search.select.data, query=query, count = count, whole = whole, percentage = percentage) # ## Search - autocomplete # autocomplete_suggestions = [] # @app.route('/autocomplete_suggestions', methods=['GET', 'POST']) # def test1(): # if request.method == 'POST': # autocomplete.load() # query = request.form['search'] # query_tokenized = query.lower().split() # print(query_tokenized) # word_1 = query_tokenized[-2] # word_2 = query_tokenized[-1] # #print(word_1) # autocomplete_output = autocomplete.predict(word_1 , word_2) # autocomplete_suggestions.clear() # for suggestion, score in autocomplete_output: # autocomplete_suggestions.append(suggestion) # # session['autocomplete_suggestions'] = str(autocomplete_suggestions) # # print(session['autocomplete_suggestions']) # # return Response(json.dumps(session['autocomplete_suggestions']), mimetype='application/json') ## STACKS! @app.route('/add_to_stack/', methods=['GET', 'POST']) def add_to_stack(id): stacks = db.session.query(Stack).all() add_form = AddtoStackForm(request.form) add_form.select_stack.choices = [(stack.id, stack.stack_name) for stack in stacks] if request.method == 'GET': book = Book.query.get(id) return render_template('add_to_stacks.html', id=id, stacks=stacks, book=book, add_form=add_form) else: stack = Stack.query.get(int(add_form.select_stack.data)) book = Book.query.get(id) stack.books.append(book) db.session.commit() return render_template('show_stack_detail.html', stack=stack) @app.route('/export/csv', methods=['GET']) def export_csv(): output = io.StringIO() #fieldnames = ['title', 'authors', 'file', 'fileformat', 'category', 'year_published', 'description' ] fields = Book.__mapper__.columns fieldnames = [] for columns in fields: fieldnames.append(columns.name) i = inspect(Book) referred_classes = [r.mapper.class_ for r in i.relationships] referred_classes_tablenames = [r.mapper.class_.__tablename__ for r in i.relationships] print(fieldnames+referred_classes_tablenames) csv = DictWriter(output,fieldnames+referred_classes_tablenames) csv.writeheader() for book in Book.query.order_by("title"): row = {} for col in fieldnames: print(getattr(book, col)) row[col] = getattr(book, col) for col in referred_classes: subattr = [] for subcol in getattr(book, col.__tablename__): for metacol in subcol.__mapper__.columns: query = metacol.name if query != "id": this = getattr(subcol, query) subattr.append(this) row[col.__tablename__] = " | ".join(subattr) csv.writerow(row) #print(row) resp = Response(output.getvalue(), mimetype="text/csv") resp.headers["Content-Disposition"] = "attachment;filename=export.csv" return resp import codecs @app.route('/import/csv', methods= ['POST','GET']) def import_csv(): if request.method == 'POST': if 'file' not in request.files: flash('No file part') return redirect(request.url) else: file = request.files['file'] for row in DictReader(codecs.iterdecode(file, 'utf-8')): numberadded = 0; book = Book.query.filter_by(title=row['title']).first() if book: print("allreadyexists") else: cover = '' if row['file'] == '': file = 'potential.pdf' file_extension = '.pdf' ptitle = row['title'] pbook = Potential(ptitle) db.session.add(pbook) db.session.commit() pbooks = Potential.query.all() template = 'app/templates/potential_pdf.html' html_string = render_template('potential_pdf.html', pbooks = pbooks) html = HTML(string=html_string) html.write_pdf(target='app/uploads/potential.pdf') print ('writing to potential_pdf') else: fullpath = os.path.join(app.config['UPLOAD_FOLDER'], row['file']) name, file_extension = os.path.splitext(row['file']) print ('get_cover', fullpath, name) cover = get_cover(fullpath, name) file = str(id) + "_" + row['file'] if row['year_published']: year_published = int(row['year_published']) else: year_published = None; book = Book(row['title'], file, cover, row['fileformat'], row['category'],year_published, None, None, None, None, None) book.scapeX = float(row['scapeX']) book.scapeY = float(row['scapeY']) db.session.add(book) numberadded = numberadded+1 authors = row['authors'].split('|') authors = [x.strip() for x in authors] for author in authors: if author: a = db.session.query(Author).filter_by(author_name=author).first() if a == None: a = Author(author_name=author) db.session.add(a) book.authors.append(a) db.session.commit() return render_template('import_csv.html', numberadded=numberadded) @app.route('/empty_catalogue487352698237465', methods= ['POST','GET']) def empty_catalogue(): meta = db.metadata for table in reversed(meta.sorted_tables): if str(table) == "books" or str(table) == "authors" or str(table) == "books_authors": print('Clear table %s' % table) db.session.execute(table.delete()) db.create_all() db.session.commit() return "ALL CLEARED" ### # The API ### @app.route('/api/books', methods=['GET']) def get_books(): books = Book.query.all() data = books_schema.dump(books) #print(errors) return jsonify({'books': data}) @app.route('/api/books/', methods=['GET']) def get_book_by_id(id): book = Book.query.get(id) data = book_schema.dump(book) if not data: return jsonify({"message": "Book could not be found."}), 400 else: return jsonify({'book': data }) @app.route('/api/chats', methods=['GET']) def get_chat(): chats = Chat.query.all() data = chats_schema.dump(chats) #print(errors) return jsonify({'chat': data}) ### # The functions below should be applicable to all Flask apps. ### @app.route('/.txt') def send_text_file(file_name): """Send your static text file.""" file_dot_text = file_name + '.txt' return app.send_static_file(file_dot_text) @app.after_request def add_header(response): """ Add headers to both force latest IE rendering engine or Chrome Frame, and also to cache the rendered page for 10 minutes. """ response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1' response.headers['Cache-Control'] = 'public, max-age=600' return response @app.errorhandler(404) def page_not_found(error): """Custom 404 page.""" return render_template('404.html'), 404 ### SOCKET for the chat @socketio.on('new_message') def new_message(message): # Send message to all users # print("new message") # channel is always 1 now, but might be interesting for further development emit('channel-' + str(message['channel']), { 'username': message['username'], 'text': message['text'], 'time': str(datetime.datetime.utcnow().strftime("%d.%m.%Y %H:%M")) }, broadcast=True ) # Save message my_new_chat = Chat( message=message['text'] ) db.session.add(my_new_chat) try: db.session.commit() except: db.session.rollback() if __name__ == '__main__': socketio.run(app) #app.run(debug=True,host="0.0.0.0",port="8080")