md_index.py is creating json dictionary with all the fully documented (with Date, Creator, Thumbnail) works in Category:Graduation work

.gitignore

@@ -0,0 +1 @@
+*~

md_index.py

@@ -0,0 +1,148 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+##############
+# Edit the index sorting articles according to topic, section and issue
+#####
+
+#
+import xml.etree.ElementTree as ET
+import html5lib, urllib2, json, pprint, subprocess
+from urllib import quote as quote
+import re
+
+sid = '1234'
+useragent = "Mozilla/5.001 (windows; U; NT4.0; en-US; rv:1.0) Gecko/25250101"
+endpoint = "http://pzwiki.wdka.nl/mw-mediadesign/api.php?format=json&"
+allworks = {}
+mainkeys = ['Thumbnail','Date','Creator']
+
+def api_request(action, pagename): #get page: content, metadata, images, imageifnp 
+    url = endpoint + action.format(pagename)
+    request = urllib2.urlopen(url)
+    jsonp = json.loads(request.read() )
+    json_dic= (jsonp.get('query').get('pages'))
+    page_id =  json_dic.keys()[0]
+    page_content = json_dic.get(page_id)
+    print 'API URL:', url
+    return page_content
+
+
+def api_page(pageid, info):
+    if info == 'content':
+        api_response = api_request('action=query&pageids={}&prop=revisions&rvprop=content', pageid)
+        page_content = ((api_response.get('revisions'))[0])['*']
+    elif info == 'metadata':
+        page_content = api_request('action=query&pageids={}&prop=info', pageid)
+    elif info == 'articleimgs':
+        page_content = api_request('action=query&pageids={}&prop=images', pageid)
+    elif info == 'imageinfo':
+        page_content = api_request('action=query&pageids={}&prop=imageinfo&iiprop=url&iiurlwidth=300', pageid)  # iiurlwidht dermines with of thumbnail         
+    return page_content
+
+def api_page_content(pagename):
+    content = ((page.get('revisions'))[0])['*']
+    return content
+#    print json.dumps( revisions, sort_keys=True, indent=4) ## see response
+
+
+
+def api_thumb_url(filename):
+   '''get thumbnail url of image'''
+   page_content_dict = api_page(filename, 'imageinfo')   
+   if 'imageinfo' in page_content_dict.keys():
+       thumburl = ((page_content_dict.get('imageinfo'))[0].get('thumburl'))
+       return page_content_dict
+
+
+def find_imgs(article):
+    '''get all internal images from published articles:
+    * query all images in  a article
+    * for each image request a thumbnail
+    * get the thumburl    
+    '''
+    page_content_dict = api_page(article, 'articleimgs')
+    if 'images' in page_content_dict.keys():
+        images_list = page_content_dict.get('images')
+#        print 'images_list', images_list
+        thumbs_list = []
+        for img in images_list: # all images in article
+            title = img.get('title')
+            thumburl_json = api_thumb_url(title)
+            if thumburl_json != None:
+                thumburl = (thumburl_json.get('imageinfo')[0]).get('thumburl')
+                thumbs_list.append(thumburl)
+            else:
+                thumburl = None
+                
+        return thumbs_list 
+
+def parse_work_page(title, content):
+    content = content.encode('utf-8')
+    if re.match('\{\{\Graduation work', content):
+        work_dict = {}
+        work_dict['Title']=title
+        template, extra = (re.findall('\{\{Graduation work\n(.*?)\}\}(.*)', content, re.DOTALL))[0]
+        # template's key/value pair
+        keyval = re.findall('\|(.*?)\=(.*?\n)', template, re.DOTALL)
+        extra = ('Extra', extra)
+        keyval.append(extra)
+        for pair in keyval:
+            key = pair[0]
+            val = pair[1]
+            val = val.replace('\n','')            
+            if 'Creator' in key:
+                val = val.replace(', ', '')
+            work_dict[key]=val
+            
+        return work_dict
+
+'''
+TEMPLATE
+
+|Description=
+|Creator=
+|Date=
+|Thumbnail=
+|Website=
+
+Description=Based on her written thesis: The Web Cheated on Me, Marie is trying to figure out where her disappointment with the web comes from. She analyzed her webbrowser history for half a year to find out what kind of information she is looking up. Her graduation work is an audio installation based on this research.\n|Creator=Marie Wocher,\n|Date=2013\n|Thumbnail=4 FromHypertextToApplePie.jpg\n
+'''
+
+def api_category(category, year):
+    '''Finds all pages within  category and returns a dictionary with info on those pages'''
+    category =  category.replace(' ', '_')
+    if year:
+        api_url = endpoint + 'action=query&list=categorymembers&cmlimit=500&cmtitle=Category:{}&cmtitle=Category:{}'.format(category, year) #BUG: API only queries last cmtitle: YEAR
+    else:
+        api_url = endpoint + 'action=query&list=categorymembers&cmlimit=500&cmtitle=Category:{}'.format(category)
+    
+    request = urllib2.urlopen(api_url)
+    jsonp = json.loads(request.read())    
+#    dict_page = {}
+    for page in  jsonp['query']['categorymembers']:
+        print 'Page:', page
+        title = ((page['title']).encode('utf-8') ).replace(" ", "_") #snakecase for page titles
+        pageid = page['pageid']
+        print 'Pageid:', pageid
+        ## NOTE: instead of using page name to query page, use PAGE ID 
+        article = api_page(pageid, 'content')
+        print 'Content:'
+        pprint.pprint(article)
+        print 
+        work = parse_work_page(title, article)  #
+        if work and set(mainkeys).issubset(work.keys()) and len([ work[key] for key in mainkeys if work[key] ])==3: # work must exist, have mainkeys as keys w/ values
+            allworks[pageid] = work
+
+
+api_category('Graduation work', '2013')
+#pprint.pprint(allworks)
+
+# save json 
+json_allworks = open('md_allworks.json', 'w')
+json.dump(allworks, json_allworks )
+print "wrote json dictionary to:", 'md_allworks.json'
+
+## TO DO
+# How do handle work['Extra'] value?
+# some tiles  work['Extra'] contains: <gallery>, [[Pages]], text, etc

prototype_json.html

@@ -0,0 +1,26 @@
+<!DOCTYPE HTML>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <script type="text/javascript" src="jquery-1.10.2.js"></script>
+    <script type="text/javascript">
+
+var myjson;
+
+function readJSON(){
+    $.getJSON( "md_allworks.json", function(data){
+    	myjson=data;
+	console.log(data);
+    })
+    
+}
+
+      
+    </script>
+
+</head>
+
+  <body onload="javascript:readJSON();" >
+    Testing JSON é
+  </body>
+</html>