You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
114 lines
4.2 KiB
Python
114 lines
4.2 KiB
Python
#! /usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
##############
|
|
# CREATE JSON DICTIONARY WITH AN ENTRY FOR EACH WORK
|
|
#####
|
|
import urllib2, json, pprint, re
|
|
from mmdc_modules import api_request, api_page, api_thumb_url
|
|
|
|
sid = '1234'
|
|
useragent = "Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101"
|
|
endpoint = "http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&"
|
|
allworks = {}
|
|
mainkeys = ['Thumbnail','Date','Creator']
|
|
|
|
# http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&action=query&titles=File:2x2 905.jpg&prop=imageinfo&iiprop=url&iiurlwidth=300
|
|
# http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&action=query&pageids=10603&prop=revisions&rvprop=content
|
|
# http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&action=query&titles=Graduation_Website_Braindump&prop=revisions&rvprop=content
|
|
|
|
def parse_work_page(title, content):
|
|
content = content.encode('utf-8')
|
|
if re.match('\{\{\Graduation work', content):
|
|
work_dict = {}
|
|
work_dict['Title']=title
|
|
template, extra = (re.findall('\{\{Graduation work\n(.*?)\}\}(.*)', content, re.DOTALL))[0]
|
|
# template's key/value pair
|
|
keyval = re.findall('\|(.*?)\=(.*?\n)', template, re.DOTALL)
|
|
if extra:
|
|
extra = ('Extra', extra)
|
|
keyval.append(extra)
|
|
|
|
checkkeys = [keyval[i] for i in range(len(keyval)) if keyval[i][0] in mainkeys and len(keyval[i][1])>3] #list mainkeys present, w/ values, in tuples [(key, val),(key, val)...]
|
|
if len(checkkeys) == 3 : # checkkeys contains all mainkeys and values
|
|
for pair in keyval:
|
|
key = pair[0]
|
|
val = pair[1]
|
|
val = val.replace('\n','')
|
|
if 'Creator' in key:
|
|
val = val.replace(', ', '')
|
|
elif 'Thumbnail' in key:
|
|
thumburl = api_thumb_url(val)
|
|
work_dict['Thumbnail_url']=thumburl
|
|
print 'THUMB:', thumburl
|
|
work_dict[key]=val
|
|
return work_dict
|
|
|
|
def api_PageCategories(pageid):
|
|
'''Find all the categories, and their parent category of a page '''
|
|
query = 'action=query&pageids={}&prop=categories'.format(pageid)
|
|
url = endpoint + query
|
|
request = urllib2.urlopen(url)
|
|
jsonp = json.loads(request.read())
|
|
json_dic = jsonp['query']['pages']
|
|
page_id = json_dic.keys()[0]
|
|
page_categories = json_dic[page_id][u'categories']
|
|
all_cats = [ entry[u'title'].encode('utf-8') for entry in page_categories ] #.replace('Category:', '')
|
|
return all_cats
|
|
|
|
|
|
def api_category(category, year): #Find all pages incategory and add to allworks dictionary
|
|
category = category.replace(' ', '_')
|
|
apiCatMembers = endpoint + 'action=query&list=categorymembers&cmlimit=1000&cmtitle=Category:{}'.format(category)
|
|
request = urllib2.urlopen(apiCatMembers)
|
|
jsonp = json.loads(request.read())
|
|
Graduation_work_Members = jsonp['query']['categorymembers']
|
|
intersectCatMembers = []
|
|
if year:
|
|
for member in Graduation_work_Members:
|
|
page_cats = api_PageCategories(member['pageid'])
|
|
if ('Category:{}'.format(year)) in page_cats:
|
|
print year, 'in', page_cats
|
|
intersectCatMembers.append(member)# add member to intersectCatMembers
|
|
else:
|
|
intersectCatMembers = Graduation_work_Members
|
|
|
|
for page in intersectCatMembers:
|
|
title = ((page['title']).encode('utf-8') ).replace(" ", "_") #snakecase for page titles
|
|
pageid = page['pageid']
|
|
article = api_page(pageid, 'content')
|
|
# print title
|
|
# pprint.pprint(article)
|
|
work = parse_work_page(title, article)
|
|
if work:
|
|
allworks[pageid] = work #dictionary(allworks) entry
|
|
print pprint.pprint( work )
|
|
# Create work page
|
|
else:
|
|
print 'WORK DOES NOT CONTAIN REQUIRED CONTENT'
|
|
print '-------------'
|
|
print
|
|
|
|
api_category('Graduation work', '2015')
|
|
json_allworks = open('allworks_mmdc.json', 'w') # save json
|
|
json.dump(allworks, json_allworks )
|
|
|
|
|
|
|
|
'''
|
|
Title
|
|
{{Graduation_work
|
|
|Description=
|
|
|Creator=
|
|
|Date=
|
|
|Bio=
|
|
|Thumbnail=
|
|
|Website=
|
|
}}
|
|
Description=
|
|
Extra=
|
|
'''
|
|
|
|
|
|
|