You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

129 lines
4.9 KiB
Python

#! /usr/bin/env python
# -*- coding: utf-8 -*-
###########
# prototyping downloading and converting mw page content to html
###########
# OVERVIEW:
# * creating one single html page
# * replace {{youtube/vimeo}} with video tags
# * replace galleries with rows of images
# request all the pages
# **BUILD INDEX**
# build all pages
import xml.etree.ElementTree as ET
import html5lib, pprint
from mmdc_modules import api_page, pandoc2html, parse_work, api_file_url, replace_gallery, replace_video, index_addwork, write_html_file, mw_cats
from argparse import ArgumentParser
p = ArgumentParser()
p.add_argument("--host", default="pzwiki.wdka.nl")
p.add_argument("--path", default="/mw-mediadesign/", help="nb: should end with /")
p.add_argument("--category", "-c", nargs="*", default=[], action="append", help="category to query, use -c foo -c bar to intersect multiple categories")
args = p.parse_args()
print args
########
# QUERY API
########
sid = '1234'
useragent = "Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101"
endpoint = "http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&"
########
# CREATE INDEX
########
memberpages=mw_cats(args)
print 'memberpages', memberpages
#memberpages = api_pagesincategories('Graduation work', '2015') #list, containing dictionary of all pages ids. Example: [{u'ns': 0, u'pageid': 15974, u'title': u'Ahhhh'}, {u'ns': 0, u'pageid': 16005, u'title': u'Artyom-graduation-work'}]
#memberpages = [{u'ns': 0, u'pageid': 15982, u'title': u'The Aesthetics of Ethics'}]
#memberpages = [{u'ns': 0, u'pageid': 16005, u'title': u'Artyom-graduation-work'}]
#memberpages = [{u'ns': 0, u'pageid': 16007, u'title': u'U ntitled'}]
#memberpages = [{u'ns': 0, u'pageid': 15965, u'title': u'Qq'}]
## output: memberpages [{u'ns': 0, u'pageid': 15982, u'title': u'The Aesthetics of Ethics'}]
########
# Templates
########
page_template = open("web/page-template.html", "r")
page_template = page_template.read()
index_file = 'web/index-template.html'
index_file = open(index_file, 'r')
index_tree = html5lib.parse(index_file, namespaceHTMLElements=False)
index_container = index_tree.find(".//div[@class='isotope']") #maybe id is important, to destinguish it
########
# CREATE PAGE
########
for member in memberpages:
#print member
# download mw work page
# pageid=member['pageid']
# pagetitle=(member['title'].encode('utf-8'))
workpage_mw = api_page(member, 'content')
# parse workpage_mw
workpage_mw = replace_gallery(workpage_mw)
workpage_mw = replace_video(workpage_mw)
workdict = parse_work(member, workpage_mw) # create dictionary workpage_mw template
for key in workdict.keys(): # convert Extra, Description, Bio to HTML
if key in ['Extra', 'Description', 'Bio'] and workdict[key]:
workdict[key] = pandoc2html( (workdict[key].decode('utf-8')) )
# fill template with dictionary/mw_page values
workpage_html = page_template.format(title=(workdict['Title']),
creator=(workdict['Creator']),
date=workdict['Date'],
website=workdict['Website'],
thumbnail=workdict['Thumbnail'],
bio=(workdict['Bio']),
description=(workdict['Description']),
extra=(workdict['Extra']) )
# parse workpage_html # process html: img full url
tree = html5lib.parse(workpage_html, namespaceHTMLElements=False)
imgs = tree.findall('.//img')
for img in imgs:
if img.get('id') is not 'logo':
src = img.get('src')
newsrc = api_file_url(src) ## MOVE FULL URl OPERATION TO MW CONTENT
if newsrc:
img.set('src', newsrc)
website = tree.find('.//p[@class="hightlightSidebar"]/a')
if not website.get('href'): # if no website link.
#remove empty <a> .//p[@class="hightlightSidebar"]/a
website_parent = tree.find('.//p[@class="hightlightSidebar"]')
website_parent.remove(website)
# save workpage_html
workpage_html = ET.tostring(tree)
creator = workdict['Creator'].decode('ascii', 'ignore')
creator = creator.replace(' ','_')
work_filename = 'web/{}-{}.html'.format(workdict['Date'], creator)
work_file = open(work_filename, "w")
work_file.write(workpage_html)
work_file.close()
# insert work to index
index_addwork( parent=index_container,
workid=key,
href=work_filename.replace('web/',''),
title=(workdict['Title']).decode('utf-8'),
creator=workdict['Creator'].decode('utf-8'),
date=workdict['Date'],
thumbnail=workdict['Thumbnail'],
)
write_html_file(index_tree, 'web/index.html')
print
print