#! /usr/bin/env python # -*- coding: utf-8 -*- ############## # CREATE JSON DICTIONARY WITH AN ENTRY FOR EACH WORK ##### import urllib2, json, pprint, re from mmdc_modules import api_request, api_page, api_thumb_url sid = '1234' useragent = "Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101" endpoint = "http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&" allworks = {} mainkeys = ['Thumbnail','Date','Creator'] # http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&action=query&titles=File:2x2 905.jpg&prop=imageinfo&iiprop=url&iiurlwidth=300 # http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&action=query&pageids=10603&prop=revisions&rvprop=content # http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&action=query&titles=Graduation_Website_Braindump&prop=revisions&rvprop=content def parse_work_page(title, content): content = content.encode('utf-8') if re.match('\{\{\Graduation work', content): work_dict = {} work_dict['Title']=title template, extra = (re.findall('\{\{Graduation work\n(.*?)\}\}(.*)', content, re.DOTALL))[0] # template's key/value pair keyval = re.findall('\|(.*?)\=(.*?\n)', template, re.DOTALL) if extra: extra = ('Extra', extra) keyval.append(extra) checkkeys = [keyval[i] for i in range(len(keyval)) if keyval[i][0] in mainkeys and len(keyval[i][1])>3] #list mainkeys present, w/ values, in tuples [(key, val),(key, val)...] if len(checkkeys) == 3 : # checkkeys contains all mainkeys and values for pair in keyval: key = pair[0] val = pair[1] val = val.replace('\n','') if 'Creator' in key: val = val.replace(', ', '') elif 'Thumbnail' in key: thumburl = api_thumb_url(val) work_dict['Thumbnail_url']=thumburl print 'THUMB:', thumburl work_dict[key]=val return work_dict def api_PageCategories(pageid): '''Find all the categories, and their parent category of a page ''' query = 'action=query&pageids={}&prop=categories'.format(pageid) url = endpoint + query request = urllib2.urlopen(url) jsonp = json.loads(request.read()) json_dic = jsonp['query']['pages'] page_id = json_dic.keys()[0] page_categories = json_dic[page_id][u'categories'] all_cats = [ entry[u'title'].encode('utf-8') for entry in page_categories ] #.replace('Category:', '') return all_cats def api_category(category, year): #Find all pages incategory and add to allworks dictionary category = category.replace(' ', '_') apiCatMembers = endpoint + 'action=query&list=categorymembers&cmlimit=1000&cmtitle=Category:{}'.format(category) request = urllib2.urlopen(apiCatMembers) jsonp = json.loads(request.read()) Graduation_work_Members = jsonp['query']['categorymembers'] intersectCatMembers = [] if year: for member in Graduation_work_Members: page_cats = api_PageCategories(member['pageid']) if ('Category:{}'.format(year)) in page_cats: print year, 'in', page_cats intersectCatMembers.append(member)# add member to intersectCatMembers else: intersectCatMembers = Graduation_work_Members for page in intersectCatMembers: title = ((page['title']).encode('utf-8') ).replace(" ", "_") #snakecase for page titles pageid = page['pageid'] article = api_page(pageid, 'content') # print title # pprint.pprint(article) work = parse_work_page(title, article) if work: allworks[pageid] = work #dictionary(allworks) entry print pprint.pprint( work ) # Create work page else: print 'WORK DOES NOT CONTAIN REQUIRED CONTENT' print '-------------' print api_category('Graduation work', '2015') json_allworks = open('allworks_mmdc.json', 'w') # save json json.dump(allworks, json_allworks ) ''' Title {{Graduation_work |Description= |Creator= |Date= |Bio= |Thumbnail= |Website= }} Description= Extra= '''