You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
570 lines
17 KiB
Python
570 lines
17 KiB
Python
# IMPORT
|
|
# to work with files in folder
|
|
import os
|
|
|
|
# to work with json files
|
|
import json
|
|
from urllib.request import urlopen
|
|
|
|
# to create the Flask app
|
|
from flask import Flask, render_template, request, url_for, redirect, jsonify, abort
|
|
|
|
# to import text contents and metadata from markdown files
|
|
from flaskext.markdown import Markdown
|
|
import frontmatter
|
|
|
|
# to cast string arguments into the required types
|
|
from pydoc import locate
|
|
|
|
# to work with notebooks
|
|
#
|
|
# to import notebook files
|
|
import nbimporter
|
|
nbimporter.options['only_defs'] = False
|
|
import importlib
|
|
|
|
# to read and execute the content of notebooks
|
|
import nbformat
|
|
from nbconvert import HTMLExporter, MarkdownExporter
|
|
from nbconvert.preprocessors import ExecutePreprocessor
|
|
|
|
# not sure about this is in the nbconvert documentation
|
|
from traitlets.config import Config
|
|
|
|
# to work with placeholder descriptions
|
|
from random import choice
|
|
|
|
# Subgroup Projects import
|
|
|
|
# not really happy of this routing for the module but
|
|
ac = importlib.import_module("projects.annotation-compass.annotation_compass")
|
|
|
|
|
|
# FUNCTIONS
|
|
|
|
def filenames(folder, remove_ext = False):
|
|
''' Read all the functions in a folder '''
|
|
names = []
|
|
for entry in os.scandir(folder):
|
|
# add to the list only proper files
|
|
if entry.is_file(follow_symlinks=False):
|
|
# remove the extension from the filename
|
|
n = os.path.splitext(entry.name)[0]
|
|
if remove_ext:
|
|
n = entry.name
|
|
names.append(n)
|
|
return names
|
|
|
|
|
|
def dirnames(folder):
|
|
''' Return all the folders in a folder '''
|
|
names = []
|
|
for entry in os.scandir(folder):
|
|
# add to the list only proper files
|
|
if not entry.name.startswith('.') and entry.is_dir():
|
|
# remove the extension from the filename
|
|
names.append(entry.name)
|
|
return names
|
|
|
|
|
|
# not really sure about this file -> module -> function thing!
|
|
# could someone help pls ?
|
|
def get_function(name, folder):
|
|
''' Dynamic import a function from a folder '''
|
|
file = __import__(f'{folder}.{name}')
|
|
module = getattr(file, name)
|
|
function = getattr(module, name)
|
|
return function
|
|
|
|
|
|
def get_function_info(function):
|
|
''' Extract info from a function '''
|
|
name = function.__name__
|
|
description = function.__doc__
|
|
parameters = []
|
|
output = ''
|
|
|
|
# TODO: default values
|
|
|
|
# populate a list of tuple with patameter, type
|
|
for param in function.__annotations__.keys():
|
|
if param == 'return':
|
|
output = function.__annotations__[param].__name__
|
|
if param != 'return':
|
|
parameters.append((param, function.__annotations__[param].__name__))
|
|
|
|
return(name, description, parameters, output)
|
|
|
|
def print_info(function):
|
|
''' Print the info of a function nicely '''
|
|
name, description, parameters, output = get_function_info(function)
|
|
|
|
# very important feature
|
|
from kaomoji.kaomoji import Kaomoji
|
|
kao = Kaomoji()
|
|
|
|
header = f'----------{kao.create()}'
|
|
footer = '-' * len(header)
|
|
|
|
print(header)
|
|
print(name)
|
|
print(description)
|
|
print('Input:')
|
|
for param, tp in parameters:
|
|
print(f' {param}, of type {tp}')
|
|
print(f'Returns a {output}')
|
|
print(footer)
|
|
|
|
|
|
def generate_function_list():
|
|
''' Build a list of the function '''
|
|
functions = []
|
|
for function in filenames("./notebooks"):
|
|
try:
|
|
fx = get_function(function, notebooks)
|
|
name, description, parameters, output = get_function_info(fx)
|
|
# print(f'{function} function is ok')
|
|
# print(f'Description is: {description}')
|
|
if description == None:
|
|
description = placeholder_description('function')
|
|
f = {
|
|
"title": name,
|
|
"description": description
|
|
}
|
|
except:
|
|
# print('--->error so placeholder fx')
|
|
f = {
|
|
"title": function,
|
|
"description": placeholder_description('function')
|
|
}
|
|
functions.append(f)
|
|
# print(functions)
|
|
with open('./static/functions.json', 'w') as file:
|
|
json.dump(functions, file)
|
|
|
|
|
|
def placeholder_description(subject):
|
|
''' Return a placeholder description of a subject '''
|
|
# ok this is a meme but is to make the notebooks a bit more waterproof
|
|
adjs = ['useful', 'nice', 'intresting', 'incredible', 'wow']
|
|
placeholders = [f'A super', f'A {choice(adjs)} and', f'You wont believe to this', f'Yet another' ]
|
|
return f'{choice(placeholders)} {choice(adjs)} {subject}'
|
|
|
|
|
|
# optionally you can pass a boolean argument to execute the notebook before the export
|
|
# but it is a slow process so by default is not active
|
|
# TODO: markdown export instead of HTML ?
|
|
# TODO: extract images from base64
|
|
def get_notebook_contents(filename, execute = False):
|
|
''' Export notebook contents as HTML. '''
|
|
with open(filename) as f:
|
|
nb = nbformat.read(f, as_version=4)
|
|
|
|
if execute:
|
|
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
|
|
ep.preprocess(nb, {'metadata':{'path':'notebooks/'}})
|
|
|
|
html_exporter = HTMLExporter()
|
|
html_exporter.template_name = 'basic'
|
|
(body, resources) = html_exporter.from_notebook_node(nb)
|
|
|
|
return body
|
|
|
|
# EXPORT AS MARKDOWN
|
|
def get_notebook_md(filename, execute = False):
|
|
''' Export notebook contents as Markdown. '''
|
|
with open(filename) as f:
|
|
nb = nbformat.read(f, as_version=4)
|
|
|
|
if execute:
|
|
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
|
|
ep.preprocess(nb, {'metadata':{'path':'notebooks/'}})
|
|
|
|
md_exporter = MarkdownExporter()
|
|
|
|
(body, resources) = md_exporter.from_notebook_node(nb)
|
|
|
|
return body
|
|
|
|
def get_contents(filename, directory = './contents'):
|
|
''' Return contents from a filename as frontmatter handler '''
|
|
with open(f"{directory}/{filename}", "r") as f:
|
|
content = frontmatter.load(f)
|
|
return content
|
|
|
|
def get_meta(filename, directory = './contents'):
|
|
''' Return contents from a filename as frontmatter handler '''
|
|
with open(f"{directory}/{filename}", "r") as f:
|
|
metadata, content = frontmatter.parse(f.read())
|
|
return metadata
|
|
|
|
# EVENT MODE
|
|
|
|
def is_event_mode():
|
|
return False
|
|
|
|
|
|
|
|
# FLASK APP
|
|
|
|
base_url = "si16"
|
|
notebooks = "notebooks"
|
|
projects = "projects"
|
|
|
|
|
|
# create flask application
|
|
app = Flask(__name__,
|
|
static_url_path=f'/soupboat/{base_url}/static',
|
|
static_folder=f'/soupboat/{base_url}/static')
|
|
Markdown(app, extensions=['extra'])
|
|
|
|
|
|
|
|
# add the base_url variable to all the flask templates
|
|
@app.context_processor
|
|
def set_base_url():
|
|
return dict(base_url = base_url)
|
|
|
|
# Generate functions list
|
|
generate_function_list()
|
|
|
|
# For specific pages we can build and link dedicated templates
|
|
|
|
|
|
# Homepage
|
|
@app.route(f"/{base_url}/")
|
|
def home_page():
|
|
|
|
# get the basic info of the website from the /contents/si16_info.md file
|
|
info = get_contents("si16_info.md").to_dict()
|
|
|
|
|
|
projects_list = []
|
|
for project in dirnames("./projects"):
|
|
page = get_contents("documentation.md", f"./{projects}/{project}").to_dict()
|
|
page['slug'] = project
|
|
projects_list.append(page)
|
|
|
|
|
|
|
|
# get the list of the projects, the functions, and the corpora
|
|
home = {
|
|
**info,
|
|
"projects": projects_list,
|
|
"functions": filenames("./notebooks"),
|
|
"corpora": dirnames("./static/corpora")
|
|
}
|
|
return render_template("home.html", **home)
|
|
|
|
|
|
# # About Page
|
|
# @app.route(f"/{base_url}/about/")
|
|
# def about_page():
|
|
# about = get_contents('about.md')
|
|
# return render_template("about.html", title = about['title'], description = about['description'], contents=about.content)
|
|
|
|
# For generic pages we can include a common template and change only the contents
|
|
@app.route(f"/{base_url}/<slug>/")
|
|
def dynamic_page(slug = None):
|
|
try:
|
|
page = get_contents(f"{slug}.md").to_dict()
|
|
# page is a dictionary that contains:
|
|
# - all the attributes in the markdown file (ex: title, description, soup, etc)
|
|
# - the content of the md in the property 'content'
|
|
# in this way we can access those frontmatter attributes in jinja simply using the variables title, description, soup, etc
|
|
return render_template("page.html", **page)
|
|
except FileNotFoundError:
|
|
# TODO: a proper not found page
|
|
return render_template('404.html')
|
|
|
|
|
|
|
|
# List of projects
|
|
@app.route(f"/{base_url}/projects/")
|
|
def projects_list():
|
|
# get a list of the functions from the notebooks folder
|
|
projects_list = []
|
|
for project in dirnames("./projects"):
|
|
page = get_contents("documentation.md", f"./{projects}/{project}").to_dict()
|
|
page['slug'] = project
|
|
projects_list.append(page)
|
|
|
|
info = get_contents("projects.md").to_dict()
|
|
|
|
# generate a link to each function
|
|
return render_template("projects.html", projects=projects_list, **info)
|
|
|
|
|
|
# Single project
|
|
@app.route(f"/{base_url}/projects/<project>/")
|
|
def p_info(project = None):
|
|
try:
|
|
|
|
page = get_contents("documentation.md", f"./{projects}/{project}").to_dict()
|
|
|
|
if 'showcases' in dirnames(f"./{projects}/{project}"):
|
|
page['showcases'] = []
|
|
showcases_list = filenames(f"./{projects}/{project}/showcases/")
|
|
for showcase in showcases_list:
|
|
print(showcase)
|
|
info = get_meta(showcase + '.md', f"./{projects}/{project}/showcases/")
|
|
info['slug'] = showcase
|
|
page['showcases'].append(info)
|
|
|
|
return render_template("project.html", **page)
|
|
except FileNotFoundError:
|
|
return render_template('404.html')
|
|
|
|
|
|
@app.route(f"/{base_url}/projects/<project>/<showcase>/")
|
|
def p_showcase(project = None, showcase = None):
|
|
print(project)
|
|
print(showcase)
|
|
try:
|
|
page = get_contents(f"{showcase}.md", f"./{projects}/{project}/showcases").to_dict()
|
|
return render_template('page.html', **page)
|
|
except FileNotFoundError:
|
|
return render_template('404.html')
|
|
return render_template('404.html')
|
|
|
|
# List of functions
|
|
@app.route(f"/{base_url}/functions/")
|
|
def functions_list():
|
|
|
|
|
|
info = get_contents("functions.md").to_dict()
|
|
|
|
# get a list of the functions from the notebooks folder
|
|
functions = filenames("./notebooks")
|
|
|
|
with open('./static/functions.json', 'r') as file:
|
|
functions = json.load(file)
|
|
|
|
|
|
# generate a link to each function
|
|
return render_template("functions.html", functions=functions, **info)
|
|
|
|
|
|
# Single Function page
|
|
@app.route(f"/{base_url}/functions/<function>/")
|
|
def f_info(function=None):
|
|
if function in filenames(f"./{notebooks}"):
|
|
fx = get_function(function, notebooks)
|
|
name, description, parameters, output = get_function_info(fx)
|
|
|
|
# executing a notebook takes a lot of time mmm should we just send them without the results of the examples? or save it executed?
|
|
# documentation = get_notebook_contents(f"./{notebooks}/{function}.ipynb")
|
|
|
|
documentation = get_notebook_md(f"./{notebooks}/{function}.ipynb")
|
|
|
|
return render_template("function.html", title=name, description=description, parameters=parameters, output=output, documentation=documentation)
|
|
|
|
# TODO: meaningful error code return
|
|
else:
|
|
return render_template('404.html')
|
|
|
|
|
|
|
|
# Function API page
|
|
|
|
@app.route(f"/{base_url}/api/<function>/")
|
|
def f_api(function=None):
|
|
if function in filenames(f"./{notebooks}"):
|
|
|
|
fx = get_function(function, notebooks)
|
|
name, description, parameters, output = get_function_info(fx)
|
|
|
|
query_params = []
|
|
for param, tp in parameters:
|
|
a = request.args.get(param)
|
|
# cast the type of the argument to the type that the function requires
|
|
if tp == "list":
|
|
a = request.args.getlist(param, type=str)
|
|
else:
|
|
tp = locate(tp)
|
|
a = tp(a)
|
|
# print(a)
|
|
query_params.append(a)
|
|
return fx(*query_params)
|
|
|
|
# TODO: meaningful error code return
|
|
return "mmmm there is no function with this name sorry"
|
|
|
|
|
|
|
|
# TODO: maybe the event mode could be a decorator ? to avoid the request from the client every time and instead just do it in the server ?
|
|
# EVENT MODE
|
|
import mimetypes
|
|
|
|
@app.route(f"/{base_url}/api/is-event-mode/")
|
|
def event_mode():
|
|
if is_event_mode():
|
|
event_path = '/static/event'
|
|
event = dirnames("./static/event")
|
|
page = request.values.get('page', '')
|
|
snippets = []
|
|
if page in event:
|
|
snippets = []
|
|
for entry in os.scandir(f".{event_path}/{page}"):
|
|
# add to the list only proper files
|
|
if entry.is_file(follow_symlinks=False):
|
|
snippet = {
|
|
"name": entry.name
|
|
}
|
|
|
|
ty, enc = mimetypes.guess_type(entry.name)
|
|
|
|
if 'text' in ty:
|
|
snippet['type'] = 'text'
|
|
with open(f".{event_path}/{page}/{entry.name}", 'r') as f:
|
|
content = f.read()
|
|
snippet['content'] = content
|
|
elif 'image' in ty:
|
|
snippet['type'] = 'image'
|
|
snippet['content'] = f"{event_path}/{page}/{entry.name}"
|
|
elif 'video' in ty:
|
|
snippet['type'] = 'video'
|
|
snippet['content'] = f"{event_path}/{page}/{entry.name}"
|
|
elif 'audio' in ty:
|
|
snippet['type'] = 'audio'
|
|
snippet['content'] = f"{event_path}/{page}/{entry.name}"
|
|
|
|
snippets.append(snippet)
|
|
return {"event": True, "snippets": snippets}
|
|
return {"event": True}
|
|
return {"event": False}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# PROJECTS API pages
|
|
# custom section ===========
|
|
# warning mothership reporting
|
|
# we will add here the custom endpoints for each subgroup project
|
|
|
|
|
|
|
|
# 2MB max for file upload (we can change this)
|
|
app.config['MAX_CONTENT_LENGTH'] = 2 * 1000 * 1000
|
|
|
|
@app.route(f"/{base_url}/annotation-compass/", methods=['GET','POST'])
|
|
def annotation_compass():
|
|
if request.method == 'POST':
|
|
# Upload new file
|
|
ac.upload_file(request)
|
|
images = ac.list_images()
|
|
return render_template('annotation-compass.html', title='Annotation Compass', images=images)
|
|
|
|
@app.route(f"/{base_url}/annotation-compass/annotate/<image>/")
|
|
def annotate_image(image=None):
|
|
print(app.static_url_path, app.static_folder)
|
|
description = ''
|
|
try:
|
|
description = ac.get_image_description(image)
|
|
except:
|
|
print('there is no description')
|
|
return render_template('annotate_image.html', image=image, description=description)
|
|
|
|
@app.route(f"/{base_url}/annotation-compass/add-label/", methods=['GET','POST'])
|
|
def insert_label():
|
|
if request.method == 'POST':
|
|
return ac.insert_label(request)
|
|
|
|
@app.route(f"/{base_url}/annotation-compass/get-labels/<image>/")
|
|
def get_labels_list(image=None):
|
|
return ac.get_labels_list(image)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# JIAN API
|
|
|
|
def generate_urls(elements):
|
|
a_list=''
|
|
for element in elements:
|
|
url=f'https://hub.xpub.nl/soupboat/individual-maps/{element}/'
|
|
a = f'<a href="{url}">{element}</a><br />'
|
|
a_list = a_list + a
|
|
return a_list
|
|
|
|
@app.route(f"/{base_url}/api/label/rejection/individual_map/")
|
|
def individual_map():
|
|
|
|
image = 'xpub1_rotterdam_map_1400px.jpg'
|
|
url = f"https://hub.xpub.nl/soupboat/generic-labels/get-labels/?image={image}"
|
|
response = urlopen(url)
|
|
data_json = json.loads(response.read())
|
|
|
|
all_userIDs = []
|
|
for label in data_json['labels']:
|
|
all_userIDs.append(label['userID'])
|
|
set_userIDs = set(all_userIDs)
|
|
|
|
im = get_function('individual_map', notebooks)
|
|
|
|
individual_labels = im(data_json['labels'], ['5058763759','5941298752'])
|
|
|
|
to_html = get_function('html_tag', notebooks)
|
|
result = to_html(individual_labels, True, True, True, True)
|
|
|
|
return result
|
|
|
|
|
|
@app.route(f"/{base_url}/projects/map/rejection_map/")
|
|
def rejection_map():
|
|
return render_template('rejection_map.html')
|
|
|
|
|
|
|
|
# ===============
|
|
# Error handlers!
|
|
@app.errorhandler(400)
|
|
def error_400(e):
|
|
# bad request or invalid url
|
|
return render_template('400.html'), 400
|
|
|
|
@app.errorhandler(403)
|
|
def error_403(e):
|
|
# forbidden (for invalid key) for evemt mode when it's not the 17th of the month
|
|
return render_template('403.html'), 403
|
|
|
|
@app.errorhandler(404)
|
|
def error_404(e):
|
|
# page not found
|
|
return render_template('404.html'), 404
|
|
|
|
@app.errorhandler(500)
|
|
def error_500(e):
|
|
# internal server error
|
|
return render_template('500.html'), 500
|
|
|
|
@app.errorhandler(502)
|
|
def error_502(e):
|
|
# bad gateaway
|
|
return render_template('502.html'), 502
|
|
|
|
@app.errorhandler(503)
|
|
def error_503(e):
|
|
# service temporarily unavailable, for secret breaks
|
|
return render_template('503.html'), 503
|
|
|
|
@app.errorhandler(504)
|
|
def error_503(e):
|
|
# ???????? gateway timeout shall we put it?????
|
|
return render_template('504.html'), 504
|
|
|
|
|
|
|
|
# RUN
|
|
|
|
app.run(port="3131")
|