import os, json, sys, urllib from mwclient import Site from pprint import pprint from jinja2 import Template from functions import pandoc, unpack_response, clean_dir, remove_nonwords from functions import Colors import argparse p = argparse.ArgumentParser(description="From smw ask string generate HTML pages with resulting results.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) p.add_argument("--host", metavar='', default="hub.xpub.nl/sandbox", help='wiki host') p.add_argument("--path", metavar='', default="/itchwiki/", help="Wiki path. Should end with /") p.add_argument("--conditions", "-c", metavar='', default='[[File:+]][[Title::+]][[Part::+]][[Date::+]]', help='The query conditions') p.add_argument("--printouts", "-p", metavar='', default='?Title|?Date|?Part|?Partof|?Creator|?Organization|?Format|?Event|?Topic|?Language', help='Selection of properties to printout') p.add_argument("--sort", "-s", metavar='', default='Date,Title,Part', help='Sorting according to conditions') p.add_argument("--order", "-o", metavar='', default='asc,asc,asc', help='Order of sorting conditions. Should same amount as the --sort properties') p.add_argument('--limit', '-l', help='(optional) Limit the number of returned ' 'items') # TODO: GET limit to work.Perhaps with a site.raw_api method p.add_argument('--dry', '-d', action='store_true', help='dry-run: will only show the query but not run it') args = p.parse_args() if len(args.sort.split(',')) != len(args.order.split(',')): print(Colors.FAIL, 'Invalid query:', Colors.WARNING, '--sort and --order do not have the same amount of elements', Colors.ENDC) print('Script exiting now') sys.exit() query = f'{args.conditions}|{args.printouts}|sort={args.sort}|order={args.order}' if args.limit: limit_str = f'|limit={args.limit}' query += limit_str print('query:', Colors.GREEN, query, Colors.ENDC) query_unquoted = urllib.parse.quote(query) query_url = f'https://{args.host}{args.path}api.php?action=ask&query={query_unquoted}&format=json' print('query URL:', query_url) if args.dry is True: sys.exit() # site and login site = Site(host=args.host, path=args.path) wd = os.path.dirname(os.path.abspath(__file__)) # working directory imgdir = os.path.join(wd, 'images') imgsjson_fn = os.path.join(wd, 'images.json') with open(imgsjson_fn, 'r') as imgsjson_file: images_info = json.load(imgsjson_file) static_html = os.path.join(wd, 'static_html') os.makedirs(static_html, exist_ok=True) # create static_html/ dir clean_dir(static_html) # if static_html exists and has files or dirs: clean it with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd loginlines = login.read() user, pwd = loginlines.split('\n') site.login(username=user, password=pwd) # login to wiki # read template files with open(os.path.join(wd, 'templates/index.html')) as document_html: index_template = Template(document_html.read()) with open(os.path.join(wd, 'templates/document.html')) as document_html: document_template = Template(document_html.read()) with open(os.path.join(wd, 'templates/document_part.html')) as document_html: document_part_template = Template(document_html.read()) all_document_parts = '' # to append all content documentslist = [] for answer in site.ask(query): # publication_title = '' # print(answer, answer.keys()) page, printout_dict, fullurl = unpack_response(answer) print(page) try: img_info = images_info[printout_dict['page']] # find corresponding image in images.json except KeyError: print(Colors.WARNING, f"{printout_dict['page']} is not is missing from the local downloaded images") print(Colors.GREEN, 'run python3 download_imgs.py to fix the issue', Colors.ENDC) sys.exit() page = site.pages[[printout_dict['page']]] # request that page from wiki pagetext = page.text() pagetext_html = pandoc(pwd=wd, content=pagetext, format_in='mediawiki', format_out='html') img_local = os.path.join(imgdir, img_info.get('filename')) # RENDER document part document_part_html = document_part_template.render( printout_dict=printout_dict, imgsrc=os.path.join(imgdir, img_info.get('filename')), text=pagetext_html, fullurl=fullurl,) all_document_parts += document_part_html # append resulting html from document part to the previous parts if printout_dict['Part'] == printout_dict['Partof']: # RENDER DOCUMENT # by passing all_document_parts html to document_template content document_html = document_template.render( title=printout_dict.get('Title'), date=printout_dict.get('Date'), content=all_document_parts) # render document template htmlpage_fn = "{}.html".format( remove_nonwords(printout_dict.get('Title')[0]) ) with open(os.path.join(static_html, htmlpage_fn), 'w') as htmlfile: htmlfile.write(document_html) all_document_parts = '' # Reset all_document_parts # add info to documentslist for index creation documentslist.append({'file': htmlpage_fn, 'title': printout_dict.get('Title'), 'date': printout_dict.get('Date'), 'creator': printout_dict.get('Creator') }) # RENDER index.html from documentslist index_html = index_template.render(index='Index', query=query, documentslist=documentslist) with open(os.path.join(static_html, 'index.html'), 'w') as htmlfile: htmlfile.write(index_html)