You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
512 lines
19 KiB
Python
512 lines
19 KiB
Python
from __future__ import print_function
|
|
|
|
import os, sys, re, urllib, urlparse, html5lib, json
|
|
from PIL import Image
|
|
from math import log
|
|
from argparse import ArgumentParser
|
|
from urllib2 import urlopen
|
|
|
|
from xml.etree import ElementTree as ET
|
|
|
|
# from wiki_get_html import page_html
|
|
from mwclient import Site
|
|
from mwclient.page import Page
|
|
from mwclient.errors import APIError
|
|
|
|
from leaflet import tiles_wrapper, recursiverender, gridrender, html
|
|
from imagetile2 import tile_image
|
|
from urllib import quote as urlquote
|
|
|
|
|
|
def wget (url, path, blocksize=4*1000):
|
|
if type(url) == unicode:
|
|
url = url.encode("utf-8")
|
|
count = 0
|
|
with open(path, "wb") as fout:
|
|
fin = urlopen(url)
|
|
while True:
|
|
data = fin.read(blocksize)
|
|
if not data:
|
|
break
|
|
fout.write(data)
|
|
count += len(data)
|
|
return count
|
|
|
|
def page_url (site, page):
|
|
# print ("[page_url]", page.name, file=sys.stderr)
|
|
base = os.path.split(site.site['base'])[0]
|
|
path = page.normalize_title(page.name)
|
|
if type(path) == unicode:
|
|
path = path.encode("utf-8")
|
|
path = urlquote(path)
|
|
uret = os.path.join(base, path)
|
|
# assert type(uret) == str
|
|
return uret
|
|
|
|
def wiki_url_to_title (url):
|
|
return urllib.unquote(url.split("/")[-1])
|
|
|
|
def parse_gallery(t):
|
|
""" returns [(imagepageurl, caption, articleurl), ...] """
|
|
galleryitems = t.findall(".//li[@class='gallerybox']")
|
|
items = []
|
|
for i in galleryitems:
|
|
image_link = i.find(".//a[@class='image']")
|
|
src = None
|
|
captiontext = None
|
|
article = None
|
|
|
|
if image_link != None:
|
|
src = image_link.attrib.get("href")
|
|
# src = src.split("/")[-1]
|
|
|
|
caption = i.find(".//*[@class='gallerytext']")
|
|
if caption:
|
|
captiontext = ET.tostring(caption, method="html")
|
|
articlelink = caption.find(".//a")
|
|
if articlelink != None:
|
|
article = articlelink.attrib.get("href")
|
|
|
|
# f = wiki.Pages[imgname]
|
|
# items.append((f.imageinfo['url'], captiontext))
|
|
items.append((src, captiontext, article))
|
|
return items
|
|
|
|
def mwfilepage_to_url (wiki, url):
|
|
filename = urllib.unquote(url.split("/")[-1])
|
|
page = wiki.Pages[filename]
|
|
return page, page.imageinfo['url']
|
|
|
|
def url_to_path (url):
|
|
""" https://pzwiki.wdka.nl/mediadesign/File:I-could-have-written-that_these-are-the-words_mb_300dpi.png """
|
|
path = urllib.unquote(urlparse.urlparse(url).path)
|
|
return "/".join(path.split("/")[3:])
|
|
|
|
def wiki_absurl (wiki, url):
|
|
ret = ''
|
|
if type(wiki.host) == tuple:
|
|
ret = wiki.host[0]+"://"+wiki.host[1]
|
|
else:
|
|
ret = "http://"+wiki.host
|
|
|
|
return urlparse.urljoin(ret, url)
|
|
|
|
def wiki_title_to_url (wiki, title):
|
|
""" relies on wiki.site['base'] being set to the public facing URL of the Main page """
|
|
ret = ''
|
|
parts = urlparse.urlparse(wiki.site['base'])
|
|
base, main_page = os.path.split(parts.path)
|
|
ret = parts.scheme+"://"+parts.netloc+base
|
|
p = wiki.pages[title]
|
|
ret += "/" + p.normalize_title(p.name)
|
|
return ret
|
|
|
|
def ensure_wiki_image_tiles (wiki, imagepageurl, text='', basepath="tiles", force=False, bgcolor=None, tilewidth=256, tileheight=256, zoom=3, margin_right=0, margin_bottom=0):
|
|
print ("ensure_wiki_image_tiles", imagepageurl, file=sys.stderr)
|
|
page, imageurl = mwfilepage_to_url(wiki, imagepageurl)
|
|
path = os.path.join(basepath, url_to_path(imageurl))
|
|
print ("imageurl, path", imageurl, path, file=sys.stderr)
|
|
ret = tiles_wrapper(path, imagepageurl, text=text)
|
|
tp = ret.get_tile_path(0, 0, 0)
|
|
if os.path.exists(tp) and not force:
|
|
return ret
|
|
|
|
try:
|
|
os.makedirs(path)
|
|
except OSError:
|
|
pass
|
|
im = Image.open(urlopen(imageurl))
|
|
tile_image(im, zoom, tilewidth, tileheight, path+"/", ret.tilename, bgcolor, margin_right, margin_bottom)
|
|
return ret
|
|
|
|
def textcell (paras):
|
|
node = {}
|
|
node['text'] = paras[:1]
|
|
moretext = paras[1:]
|
|
if moretext:
|
|
node['children'] = [textcell([x]) for x in moretext]
|
|
return node
|
|
|
|
def name_to_path (name):
|
|
return name.replace("/", "_")
|
|
|
|
|
|
def render_article (wiki, ref, basepath="tiles", depth=0, maxdepth=3, tilewidth=256, tileheight=256):
|
|
print ("render_article", ref, file=sys.stderr)
|
|
if type(ref) == Page:
|
|
page = ref
|
|
title = page.name
|
|
ref = wiki_title_to_url(wiki, page.name)
|
|
elif ref.startswith("http"):
|
|
title = wiki_url_to_title(ref)
|
|
page = wiki.pages[title]
|
|
else:
|
|
title = ref
|
|
page = wiki.pages[title]
|
|
ref = wiki_title_to_url(wiki, page.name)
|
|
# pagetext = page.text()
|
|
# print ("WIKI PARSE", title, file=sys.stderr)
|
|
parse = wiki.parse(page=title)
|
|
html = parse['text']['*']
|
|
# print ("GOT HTML ", html, file=sys.stderr)
|
|
tree = html5lib.parse(html, treebuilder="etree", namespaceHTMLElements=False)
|
|
body = tree.find("./body")
|
|
paras = []
|
|
images = []
|
|
imgsrcs = {}
|
|
|
|
for c in body:
|
|
if c.tag == "p":
|
|
# filter out paras like <p><br></p> but checking text-only render length
|
|
ptext = ET.tostring(c, encoding="utf-8", method="text").strip()
|
|
if len(ptext) > 0:
|
|
ptext = ET.tostring(c, encoding="utf-8", method="html").strip()
|
|
paras.append(ptext)
|
|
|
|
elif c.tag == "ul" and c.attrib.get("class") != None and "gallery" in c.attrib.get("class"):
|
|
# print ("GALLERY")
|
|
gallery = parse_gallery(c)
|
|
# Ensure image is downloaded ... at least the 00 image...
|
|
for src, caption, article in gallery:
|
|
src = wiki_absurl(wiki, src)
|
|
if src in imgsrcs:
|
|
continue
|
|
imgsrcs[src] = True
|
|
print ("GalleryImage", src, caption, article, file=sys.stderr)
|
|
# if article and depth < maxdepth:
|
|
# article = wiki_absurl(wiki, article)
|
|
# images.append(render_article(wiki, article, caption, basepath, depth+1, maxdepth))
|
|
# else:
|
|
images.append(ensure_wiki_image_tiles(wiki, src, caption, basepath, tilewidth=tilewidth, tileheight=tileheight).zoom())
|
|
|
|
for a in body.findall('.//a[@class="image"]'):
|
|
caption = a.attrib.get("title", '')
|
|
src = wiki_absurl(wiki, a.attrib.get("href"))
|
|
# OEI... skippin svg for the moment (can't go straight to PIL)
|
|
if src.endswith(".svg"):
|
|
continue
|
|
print (u"Image_link {0}:'{1}'".format(src, caption).encode("utf-8"), file=sys.stderr)
|
|
if src in imgsrcs:
|
|
continue
|
|
imgsrcs[src] = True
|
|
images.append(ensure_wiki_image_tiles(wiki, src, caption, basepath, tilewidth=tilewidth, tileheight=tileheight).zoom())
|
|
|
|
print ("{0} paras, {1} images".format(len(paras), len(images)), file=sys.stderr)
|
|
|
|
|
|
if title == None:
|
|
title = page.name
|
|
|
|
basename = "tiles/" + name_to_path(page.name)
|
|
|
|
# gallerynode = gridrender(images, basename)
|
|
# return gallerynode
|
|
cells = []
|
|
if len(paras) > 0:
|
|
cells.append(textcell(paras))
|
|
cells.extend(images)
|
|
|
|
ret = recursiverender(cells, basename, tilewidth=tilewidth, tileheight=tileheight)
|
|
ret['text'] = u"""<p class="caption"><a class="url" href="{1}">{0}</a></p>""".format(title, ref)
|
|
if images:
|
|
ret['image'] = images[0]['image']
|
|
return ret
|
|
|
|
# article = {}
|
|
# article['text'] = title
|
|
# article['children'] = children = []
|
|
# children.append(textcell(paras))
|
|
# for iz in images[:2]:
|
|
# if 'image' not in article and 'image' in iz:
|
|
# article['image'] = iz['image']
|
|
# children.append(iz)
|
|
# restimages = images[2:]
|
|
# if len(restimages) == 1:
|
|
# children.append(restimages[0])
|
|
# elif len(restimages) > 1:
|
|
# children.append(gridrender(restimages, basename))
|
|
# return article
|
|
|
|
def render_category (wiki, cat, output="tiles", tilewidth=256, tileheight=256):
|
|
print ("Render Category", cat, file=sys.stderr)
|
|
# if type(cat) == Page:
|
|
# page = ref
|
|
# title = page.name
|
|
# ref = wiki_title_to_url(wiki, page.name)
|
|
if cat.startswith("http"):
|
|
title = wiki_url_to_title(cat)
|
|
cat = wiki.pages[title]
|
|
else:
|
|
title = ref
|
|
cat = wiki.pages[cat]
|
|
# ref = wiki_title_to_url(wiki, cat.name)
|
|
print ("cat", cat, file=sys.stderr)
|
|
pages = []
|
|
for m in cat.members():
|
|
pages.append(m)
|
|
pages.sort(key=lambda x: x.name)
|
|
pagenodes = [render_article(wiki, x.name, tilewidth=tilewidth, tileheight=tileheight) for x in pages]
|
|
for page, node in zip(pages, pagenodes):
|
|
node['text'] = u"""<p class="caption"><a class="url" href="{1}">{0}</a></p>""".format(page.name, wiki_title_to_url(wiki, page.name))
|
|
ret = gridrender(pagenodes, output+"/"+cat.name.replace(":", "_"), tilewidth=tilewidth, tileheight=tileheight)
|
|
ret['text'] = u"""<p class="caption"><a class="url" href="{0}">{1}</a></p>""".format(wiki_title_to_url(wiki, cat.name), cat.name)
|
|
return ret
|
|
# for p in pages:
|
|
# print (p.name, wiki_title_to_url(wiki, p.name))
|
|
|
|
def make_category (args):
|
|
wiki = Site((args.wikiprotocol, args.wikihost), path=args.wikipath)
|
|
root_node = render_category(wiki, args.category)
|
|
if args.html:
|
|
print (html(root_node, ""))
|
|
else:
|
|
print (json.dumps(root_node, indent=2))
|
|
|
|
|
|
def make_article (args):
|
|
wiki = Site((args.wikiprotocol, args.wikihost), path=args.wikipath)
|
|
root_node = render_article(wiki, args.wikipage, tilewidth=args.tilewidth, tileheight=args.tileheight)
|
|
if args.html:
|
|
print (html(root_node, ""))
|
|
else:
|
|
print (json.dumps(root_node, indent=2))
|
|
|
|
def make_gallery(args):
|
|
wiki = Site((args.wikiprotocol, args.wikihost), path=args.wikipath)
|
|
# apiurl = args.wikiprotocol+"://"+args.wikihost+args.wikipath+"api.php"
|
|
if len(args.wikipage) == 1:
|
|
root_node = render_article(wiki, args.wikipage[0], tilewidth=args.tilewidth, tileheight=args.tileheight)
|
|
else:
|
|
children = []
|
|
for wikipage in args.wikipage:
|
|
print ("rendering", wikipage, file=sys.stderr)
|
|
if "Category:" in wikipage:
|
|
print ("rendering", wikipage, file=sys.stderr)
|
|
cnode = render_category(wiki, wikipage, args.output)
|
|
else:
|
|
cnode = render_article(wiki, wikipage, tilewidth=args.tilewidth, tileheight=args.tileheight)
|
|
children.append(cnode)
|
|
if args.recursive:
|
|
root_node = recursiverender(children, args.output+"/"+args.name, direction=1, tilewidth=args.tilewidth, tileheight=args.tileheight)
|
|
else:
|
|
root_node = gridrender(children, args.output+"/"+args.name, direction=1, tilewidth=args.tilewidth, tileheight=args.tileheight)
|
|
|
|
if args.html:
|
|
print (html(root_node, ""))
|
|
else:
|
|
print (json.dumps(root_node, indent=2))
|
|
|
|
from time import sleep
|
|
def testwiki (args):
|
|
site = Site((args.wikiprotocol, args.wikihost), path=args.wikipath)
|
|
return site
|
|
|
|
USER_NS = 2
|
|
|
|
def imageinfo_with_thumbnail (site, name):
|
|
d = site.api(
|
|
"query",
|
|
titles=name,
|
|
prop="imageinfo",
|
|
iiprop="url|mime",
|
|
iiurlwidth=1024
|
|
)
|
|
pp = d['query']['pages']
|
|
for key in pp:
|
|
return pp[key]['imageinfo'][0]
|
|
|
|
def recentfiles (args):
|
|
# open connection to wiki
|
|
wiki = Site((args.wikiprotocol, args.wikihost), path=args.wikipath)
|
|
|
|
# Prepare user list to filter (if args.usercategory)
|
|
filter_by_users = None
|
|
if args.usercategory:
|
|
filter_by_users = set()
|
|
usercategory = wiki.categories.get(args.usercategory)
|
|
for p in usercategory.members():
|
|
if p.namespace == USER_NS:
|
|
filter_by_users.add(p.page_title)
|
|
|
|
# Load args.json for oldest timestamp
|
|
last_date = None
|
|
if args.json:
|
|
try:
|
|
with open (args.json) as f:
|
|
print ("Reading {0}".format(args.json), file=sys.stderr)
|
|
for line in f:
|
|
data = json.loads(line)
|
|
if 'date' in data:
|
|
last_date = data['date']
|
|
except IOError as e:
|
|
pass
|
|
|
|
|
|
# Prepare the query arguments
|
|
qargs = {
|
|
'list': "allimages",
|
|
'ailimit': 50,
|
|
'aisort': 'timestamp',
|
|
'aidir': 'descending',
|
|
'aiprop': "timestamp|url|user|userid"
|
|
}
|
|
if args.oldest:
|
|
qargs['aiend'] = args.oldest
|
|
if last_date:
|
|
print ("Using aiend {0}".format(last_date), file=sys.stderr)
|
|
qargs['aiend'] = last_date
|
|
|
|
count = 0 # used to satisfy --limit when given
|
|
skipped_users = set() # nicety for outputting names only once when skipped
|
|
items_to_output = []
|
|
|
|
# LOOP for continuing queries as needed
|
|
while True:
|
|
qq = wiki.api('query', **qargs)
|
|
# print ("Got {0} results".format(len(qq['query']['allimages'])), file=sys.stderr)
|
|
results = qq['query']['allimages']
|
|
for r in results:
|
|
# Filter on user
|
|
if filter_by_users != None:
|
|
if r['user'] not in filter_by_users:
|
|
if r['user'] not in skipped_users:
|
|
print ("Skipping user {0}".format(r['user']), file=sys.stderr)
|
|
skipped_users.add(r['user'])
|
|
continue
|
|
|
|
try:
|
|
# Filter on mime type (image/*)
|
|
filepage = wiki.pages.get(r['title'])
|
|
# mwclient's imageinfo doesn't have mime (or thumbnail info)
|
|
# imageinfo = filepage.imageinfo
|
|
imageinfo = imageinfo_with_thumbnail(wiki, r['title'])
|
|
if not imageinfo['mime'].startswith("image/"):
|
|
print (u"Skipping non image ({0}) {1}".format(imageinfo['mime'], r['title']).encode("utf-8"), file=sys.stderr)
|
|
continue
|
|
if 'thumburl' not in imageinfo:
|
|
print (u"Skipping item with no thumburl {0}".format(r['title']).encode("utf-8"), file=sys.stderr)
|
|
continue
|
|
|
|
# Deal with edge case at items == aiend are returned
|
|
if last_date and r['timestamp'] == last_date:
|
|
# print ("SKIPPING AIEND item", file=sys.stderr)
|
|
break
|
|
|
|
# Construct an item for output
|
|
print (u"[{0}], date:{1}".format(filepage.page_title, r['timestamp']).encode("utf-8"), file=sys.stderr)
|
|
usagepage = None
|
|
for usagepage in filepage.imageusage():
|
|
break # just grab the first usage page
|
|
# url : local path to file
|
|
# imageurl = imageinfo['url']
|
|
imageurl = imageinfo['thumburl']
|
|
localpath = imageurl.replace("https://pzwiki.wdka.nl/mw-mediadesign/images/", "wiki/")
|
|
# wget image from wiki to local folder
|
|
if not os.path.exists(localpath):
|
|
try:
|
|
os.makedirs(os.path.split(localpath)[0])
|
|
except OSError:
|
|
pass
|
|
print (" downloading {0} to {1}".format(imageurl, localpath), file=sys.stderr)
|
|
wget(imageurl, localpath)
|
|
|
|
item = {}
|
|
item['url'] = localpath
|
|
item['date'] = r['timestamp']
|
|
item['src'] = page_url(wiki, filepage)
|
|
userpage = wiki.pages.get('User:'+r['user'])
|
|
if usagepage:
|
|
item['text'] = u'<a href="{0}">{1}</a><br>Uploaded by <a href="{2}">{3}</a>'.format(
|
|
page_url(wiki, usagepage),
|
|
usagepage.page_title,
|
|
page_url(wiki, userpage),
|
|
r['user']).encode("utf-8")
|
|
else:
|
|
item['text'] = u'<a href="{0}">{1}</a><br>Uploaded by <a href="{2}">{3}</a>'.format(
|
|
page_url(wiki, filepage),
|
|
filepage.page_title,
|
|
page_url(wiki, userpage),
|
|
r['user']).encode("utf-8")
|
|
|
|
# print (json.dumps(item))
|
|
items_to_output.append(item)
|
|
# honor --limit
|
|
count += 1
|
|
if args.limit and count == args.limit:
|
|
break
|
|
|
|
except APIError as e:
|
|
print ("Error {0}, skipping".format(e) ,file=sys.stderr)
|
|
|
|
if args.limit and count == args.limit:
|
|
break
|
|
# continue the query if possible (pre-loop)...
|
|
if 'continue' in qq:
|
|
qargs['aicontinue'] = qq['continue']['aicontinue']
|
|
else:
|
|
# we've reached the end of the query data
|
|
break
|
|
|
|
# OUTPUT RESULTS
|
|
# reverse to be chronological
|
|
items_to_output.reverse()
|
|
if args.json:
|
|
with open(args.json, "a") as f:
|
|
for x in items_to_output:
|
|
print (json.dumps(x), file=f)
|
|
else:
|
|
for x in items_to_output:
|
|
print (json.dumps(x))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
ap = ArgumentParser("")
|
|
ap.add_argument("--wikiprotocol", default="https")
|
|
ap.add_argument("--wikihost", default="pzwiki.wdka.nl")
|
|
ap.add_argument("--wikipath", default="/mw-mediadesign/")
|
|
ap.add_argument("--wikishortpath", default="/mediadesign/")
|
|
|
|
ap.add_argument("--tilewidth", type=int, default=256)
|
|
ap.add_argument("--tileheight", type=int, default=256)
|
|
# ap.add_argument("--zoom", type=int, default=3)
|
|
|
|
ap.add_argument("--output", default="tiles")
|
|
# ap.add_argument("--title", default="TITLE")
|
|
|
|
|
|
subparsers = ap.add_subparsers(help='sub-command help')
|
|
ap_article = subparsers.add_parser('article', help='Render an article')
|
|
ap_article.add_argument("wikipage")
|
|
ap_article.add_argument("--html", default=False, action="store_true")
|
|
ap_article.set_defaults(func=make_article)
|
|
|
|
ap_gallery = subparsers.add_parser('gallery', help='Render a gallery of articles')
|
|
ap_gallery.add_argument("wikipage", nargs="+")
|
|
ap_gallery.add_argument("--html", default=False, action="store_true")
|
|
ap_gallery.add_argument("--recursive", default=False, action="store_true")
|
|
ap_gallery.add_argument("--direction", type=int, default=3, help="cell to recursively expand into, 0-3, default: 3 (bottom-right)")
|
|
ap_gallery.add_argument("--name", default=None)
|
|
ap_gallery.set_defaults(func=make_gallery)
|
|
|
|
ap_gallery = subparsers.add_parser('testwiki', help='Render a gallery of articles')
|
|
ap_gallery.set_defaults(func=testwiki)
|
|
|
|
ap_article = subparsers.add_parser('category', help='Render an article')
|
|
ap_article.add_argument("category")
|
|
ap_article.add_argument("--html", default=False, action="store_true")
|
|
ap_article.set_defaults(func=make_category)
|
|
|
|
ap_recentfiles = subparsers.add_parser('recentfiles', help='Incrementally update a json file with information about recent files')
|
|
ap_recentfiles.add_argument("--usercategory", help="limit to activity by users that are members of this category")
|
|
ap_recentfiles.add_argument("--limit", type=int, help="limit")
|
|
ap_recentfiles.add_argument("--oldest", default=None, help="No results earlier than this timestamp (e.g. 2018-01-01T00:00:00Z)")
|
|
ap_recentfiles.add_argument("--json", default=None, help="Use this json file as both input (to check last timestampt) and output -- append results chronologically as json-stream.")
|
|
ap_recentfiles.set_defaults(func=recentfiles)
|
|
|
|
|
|
|
|
args = ap.parse_args()
|
|
ret = args.func(args)
|
|
|