You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
174 lines
7.3 KiB
Python
174 lines
7.3 KiB
Python
#! /usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import urllib2, json, pprint, re
|
|
import xml.etree.ElementTree as ET
|
|
import subprocess, shlex
|
|
sid = '1234'
|
|
useragent = "Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101"
|
|
endpoint = "http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&"
|
|
|
|
# API MODULES
|
|
def api_request(action, pagename): #get page: content, metadata, images, imageifnp
|
|
url = endpoint + action.format(pagename)
|
|
request = urllib2.urlopen(url)
|
|
jsonp = json.loads(request.read() )
|
|
json_dic= (jsonp.get('query').get('pages'))
|
|
# pprint.pprint( json_dic )
|
|
page_id = json_dic.keys()[0]
|
|
page_content = json_dic.get(page_id)
|
|
return page_content
|
|
|
|
def api_page(pageid, query):
|
|
if query == 'content':
|
|
api_response = api_request('action=query&pageids={}&prop=revisions&rvprop=content', pageid)
|
|
response = ((api_response.get('revisions'))[0])['*']
|
|
elif query == 'metadata':
|
|
response = api_request('action=query&pageids={}&prop=info', pageid)
|
|
elif query == 'articleimgs':
|
|
response = api_request('action=query&pageids={}&prop=images', pageid)
|
|
elif query == 'file':
|
|
response = api_request('action=query&titles=File:{}&prop=imageinfo&iiprop=url',pageid)
|
|
pprint.pprint( response )
|
|
elif query == 'imageinfo':
|
|
pagename = pageid # in imageinfo titles are used instead of id
|
|
response = api_request('action=query&titles=File:{}&prop=imageinfo&iiprop=url&iiurlwidth=500', pagename) # iiurlwidht dermines with of thumbnail
|
|
return response
|
|
|
|
def api_file_url(filename): # get full urls
|
|
page_content_dict = api_page(filename, 'file')
|
|
if 'imageinfo' in page_content_dict.keys():
|
|
imgurl = ((page_content_dict.get('imageinfo'))[0].get('url'))
|
|
return imgurl
|
|
else:
|
|
return None
|
|
|
|
def api_thumb_url(filename):
|
|
'''get thumbnail url of image'''
|
|
page_content_dict = api_page(filename, 'imageinfo')
|
|
if 'imageinfo' in page_content_dict.keys():
|
|
thumburl = ((page_content_dict.get('imageinfo'))[0].get('thumburl'))
|
|
return thumburl
|
|
|
|
|
|
# http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&action=query&titles=File:2x2 905.jpg&prop=imageinfo&iiprop=url&iiurlwidth=300
|
|
# http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&action=query&pageids=10603&prop=revisions&rvprop=content
|
|
# http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&action=query&titles=Graduation_Website_Braindump&prop=revisions&rvprop=content
|
|
|
|
# XML MODULES
|
|
|
|
def write_html_file(html_tree, filename):
|
|
doctype = "<!DOCTYPE HTML>"
|
|
html = doctype + ET.tostring(html_tree, encoding='utf-8', method='html')
|
|
edited = open(filename, 'w') #write
|
|
edited.write(html)
|
|
edited.close()
|
|
|
|
# mw article modules
|
|
|
|
def parse_work_page(title, content):
|
|
# content = content.encode('utf-8')
|
|
if re.match('\{\{\Graduation work', content):
|
|
work_dict = {}
|
|
work_dict['Title']=title
|
|
template, extra = (re.findall('\{\{Graduation work\n(.*?)\}\}(.*)', content, re.DOTALL))[0]
|
|
# template's key/value pair
|
|
keyval = re.findall('\|(.*?)\=(.*?\n)', template, re.DOTALL)
|
|
if extra:
|
|
extra = ('Extra', extra)
|
|
keyval.append(extra)
|
|
|
|
# checkkeys = [keyval[i] for i in range(len(keyval)) if keyval[i][0] in mainkeys and len(keyval[i][1])>3] #list mainkeys present, w/ values, in tuples [(key, val),(key, val)...]
|
|
# if len(checkkeys) == 3 : # checkkeys contains all mainkeys and values
|
|
for pair in keyval:
|
|
key = pair[0]
|
|
val = pair[1]
|
|
val = val.replace('\n','')
|
|
if 'Creator' in key:
|
|
val = val.replace(', ', '')
|
|
elif 'Thumbnail' in key:
|
|
thumburl = api_thumb_url(val)
|
|
work_dict['Thumbnail_url']=thumburl
|
|
print 'THUMB:', thumburl
|
|
work_dict[key]=val
|
|
return work_dict, extra
|
|
|
|
|
|
# Alternative to parse_work_page - W/out dictionary
|
|
def parse_work(title, content):
|
|
if re.match('\{\{\Graduation work', content):
|
|
template, extra = (re.findall('\{\{Graduation work\n(.*?)\}\}(.*)', content, re.DOTALL))[0]
|
|
# template's key/value pair
|
|
keyval = re.findall('\|(.*?)\=(.*?\n)', template, re.DOTALL)
|
|
for pair in keyval:
|
|
key = pair[0]
|
|
val = pair[1]
|
|
if 'Description' in key:
|
|
description = val
|
|
|
|
print keyval
|
|
|
|
return extra, description
|
|
|
|
|
|
|
|
|
|
# Conversion Modules
|
|
def pandoc2html(mw_content):
|
|
if mw_content:
|
|
mw_content = mw_content#.encode('utf-8')
|
|
# convert from mw to html
|
|
args_echo =shlex.split( ('echo "{}"'.format(mw_content)) )
|
|
args_pandoc = shlex.split( 'pandoc -f mediawiki -t html5' )
|
|
p1 = subprocess.Popen(args_echo, stdout=subprocess.PIPE)
|
|
p2 = subprocess.Popen(args_pandoc, stdin=p1.stdout, stdout=subprocess.PIPE)
|
|
html = (p2.communicate())[0]
|
|
return html
|
|
|
|
def pandoc(filename, title, creator, date, website, thumbnail, bio, description, extra, template) :
|
|
'''pandoc: convert mediawiki syntax to html'''
|
|
# mw_content = mw_content.encode('utf-8')
|
|
args_echo =shlex.split( ('echo "{}"'.format( extra )) )
|
|
args_pandoc = shlex.split( 'pandoc -s -f mediawiki -t html \
|
|
--template {template} --variable title="{title}" --variable creator="{creator}" --variable date="{date}" --variable website="{website}" --variable website="{website}" --variable thumbnail="{thumbnail}" --variable bio="""{bio}""" -o {filename}'.format(template=template, title=title, creator=creator, date=date, website=website, thumbnail=thumbnail, bio=bio, description=description, extra=extra, filename=filename) )
|
|
print args_pandoc
|
|
p1 = subprocess.Popen(args_echo, stdout=subprocess.PIPE)
|
|
p2 = subprocess.Popen(args_pandoc, stdin=p1.stdout, stdout=subprocess.PIPE)
|
|
html = (p2.communicate())[0]
|
|
# return html
|
|
|
|
# pandoc either reades input from stdin or through input file
|
|
# pandoc DOES NOT convert variables; it has to receive the input from stdin.
|
|
# to create html convert: bio, description, extra, of at time
|
|
# insert them into HTML template by:
|
|
## gerating html in python? and insertion sub elements ?
|
|
## ??
|
|
|
|
# pandoc = 'pandoc -s -f mediawiki -t html5 \
|
|
# --template template_article.html \
|
|
# --variable title="{title}" \
|
|
# --variable section="{section}" \
|
|
# --variable topics="{topics}" \
|
|
# --variable issueName="{iname}" \
|
|
# --variable issueNumber="{inum}" \
|
|
# "articles/tmp_content.mw" -o "{articlepath}/{htmlfile}.html"'.format(articlepath=path, title=(pagename).replace("_"," "), section=in_section, topics=in_topic, iname=in_issuename, inum=in_issue, htmlfile=pagename)
|
|
# subprocess.call(pandoc, shell=True) # saved in tmp_content.html html
|
|
# html = open('tmp_content.html', 'r') #write mediawiki content to html in tmp_content.html
|
|
# html = html.read()
|
|
# return html
|
|
|
|
|
|
|
|
def img_fullurl(parent):
|
|
imgs = parent.findall('.//img')
|
|
print 'len IMG', len(imgs)
|
|
for img in imgs:
|
|
src = img.get('src')
|
|
fullurl = api_thumb_url(src)
|
|
print '----- IMG', ET.tostring(img ), src, fullurl
|
|
if fullurl != None:
|
|
img.set('src', fullurl)
|
|
|
|
# fileurl = api_request(src, endpoint)# find url of file
|
|
|