WIP:multi values properties

populat_orgs
Castro0o 5 years ago
parent 69fc3f2ec5
commit 0d9ed8a2d4

@ -43,23 +43,41 @@ def unpack_response(response):
printouts = response['printouts'] printouts = response['printouts']
page = response['fulltext'] page = response['fulltext']
fullurl = response['fullurl'] fullurl = response['fullurl']
d['page'] = page printouts_dumps = json.dumps(printouts)
for prop in printouts: printouts_loads = json.loads(printouts_dumps)
p_item = response['printouts'][prop] printouts_loads['page'] = page
for prop_val in p_item: # printouts_loads['Date'] = datetime.fromtimestamp(
if isinstance(prop_val, dict) is False: # int(printouts_loads['Date'][0]['timestamp']))
d[prop] = prop_val simplified_printouts = {}
else: for k, v in printouts_loads.items():
# if len(prop_val) > 0: if k == 'Date':
props = list(prop_val.keys()) simplified_printouts[k] = datetime.fromtimestamp(
if 'fulltext' in props: int(v[0]['timestamp']))
val = prop_val.get('fulltext') # elif k == 'Title':
elif 'timestamp' in props: # simplified_printouts[k] = v[0]['fulltext']
val = datetime.fromtimestamp(int(prop_val.get('timestamp'))) elif k in ['Part', 'Partof', 'page']: # only 1 value for each
else: simplified_printouts[k] = v
val = list(prop_val.values())[0] else: # Possibly more than 1 value for the rest of properties
d[prop] = val simplified_printouts[k] = []
return page, d, fullurl for listitem in v:
simplified_printouts[k].append(listitem['fulltext'])
# for prop in printouts:
# p_item = response['printouts'][prop]
# for prop_val in p_item:
# if isinstance(prop_val, dict) is False:
# d[prop] = prop_val
# else:
# # if len(prop_val) > 0:
# props = list(prop_val.keys())
# if 'fulltext' in props:
# val = prop_val.get('fulltext')
# elif 'timestamp' in props:
# val = datetime.fromtimestamp(int(prop_val.get('timestamp')))
# else:
# val = list(prop_val.values())[0]
# d[prop] = val
return page, simplified_printouts, fullurl
def update_json(imgsjson_fn, img_dict, img_fn): def update_json(imgsjson_fn, img_dict, img_fn):

@ -15,7 +15,7 @@ p.add_argument("--conditions", "-c", metavar='',
default='[[File:+]][[Title::+]][[Part::+]][[Date::+]]', default='[[File:+]][[Title::+]][[Part::+]][[Date::+]]',
help='The query conditions') help='The query conditions')
p.add_argument("--printouts", "-p", metavar='', p.add_argument("--printouts", "-p", metavar='',
default='?Title|?Date|?Part|?Partof|?Creator', default='?Title|?Date|?Part|?Partof|?Creator|?Organization|?Format|?Event|?Topic|?Language',
help='Selection of properties to printout') help='Selection of properties to printout')
p.add_argument("--sort", "-s", metavar='', p.add_argument("--sort", "-s", metavar='',
default='Date,Title,Part', default='Date,Title,Part',
@ -23,6 +23,8 @@ p.add_argument("--sort", "-s", metavar='',
p.add_argument("--order", "-o", metavar='', p.add_argument("--order", "-o", metavar='',
default='asc,asc,asc', default='asc,asc,asc',
help='Order of sorting conditions. Should same amount as the --sort properties') help='Order of sorting conditions. Should same amount as the --sort properties')
p.add_argument('--limit', '-l', help='(optional) Limit the number of returned '
'items')
p.add_argument('--dry', '-d', action='store_true', p.add_argument('--dry', '-d', action='store_true',
help='dry-run: will only show the query but not run it') help='dry-run: will only show the query but not run it')
@ -33,8 +35,10 @@ if len(args.sort.split(',')) != len(args.order.split(',')):
Colors.WARNING, '--sort and --order do not have the same amount of elements', Colors.ENDC) Colors.WARNING, '--sort and --order do not have the same amount of elements', Colors.ENDC)
print('Script exiting now') print('Script exiting now')
sys.exit() sys.exit()
query = f'{args.conditions}|{args.printouts}|sort={args.sort}|order={args.order}' query = f'{args.conditions}|{args.printouts}|sort={args.sort}|order={args.order}'
if args.limit:
limit_str = f'|limit={args.limit}'
query += limit_str
print('query:', Colors.GREEN, query, Colors.ENDC) print('query:', Colors.GREEN, query, Colors.ENDC)
query_unquoted = urllib.parse.quote(query) query_unquoted = urllib.parse.quote(query)
query_url = f'https://{args.host}{args.path}api.php?action=ask&query={query_unquoted}&format=json' query_url = f'https://{args.host}{args.path}api.php?action=ask&query={query_unquoted}&format=json'
@ -75,7 +79,7 @@ with open(os.path.join(wd, 'templates/document_part.html')) as document_html:
all_document_parts = '' # to append all content all_document_parts = '' # to append all content
documentslist = [] documentslist = []
for answer in site.ask(query): for answer in site.ask(query):
publication_title = '' # publication_title = ''
# print(answer, answer.keys()) # print(answer, answer.keys())
page, printout_dict, fullurl = unpack_response(answer) page, printout_dict, fullurl = unpack_response(answer)
print(page) print(page)
@ -85,6 +89,13 @@ for answer in site.ask(query):
print(Colors.WARNING, f"{printout_dict['page']} is not is missing from the local downloaded images") print(Colors.WARNING, f"{printout_dict['page']} is not is missing from the local downloaded images")
print(Colors.GREEN, 'run python3 download_imgs.py to fix the issue', Colors.ENDC) print(Colors.GREEN, 'run python3 download_imgs.py to fix the issue', Colors.ENDC)
sys.exit() sys.exit()
#
# # TODO: EXTRACT PROPERTIES THROUGH THE FOLLOWING ASK QUERY
# ask_page_props = f'[[File:{printout_dict["page"]}]]|?Title|?Date|?Part|?Partof|?Creator|?Organization|?Format|?Event|?Topic|?Language'
# print(ask_page_props)
# page_props = site.ask(ask_page_props)
# print(page_props)
# import pdb; pdb.set_trace()
page = site.pages[[printout_dict['page']]] # request that page from wiki page = site.pages[[printout_dict['page']]] # request that page from wiki
pagetext = page.text() pagetext = page.text()
pagetext_html = pandoc(pwd=wd, content=pagetext, format_in='mediawiki', format_out='html') pagetext_html = pandoc(pwd=wd, content=pagetext, format_in='mediawiki', format_out='html')
@ -101,10 +112,15 @@ for answer in site.ask(query):
if printout_dict['Part'] == printout_dict['Partof']: if printout_dict['Part'] == printout_dict['Partof']:
# RENDER DOCUMENT # RENDER DOCUMENT
# by passing all_document_parts html to document_template content # by passing all_document_parts html to document_template content
document_html = document_template.render(title=printout_dict.get('Title'),
# TODO: EXPAND PROPERTIES IN TEMPLATE
document_html = document_template.render(
title=printout_dict.get('Title'),
date=printout_dict.get('Date'), date=printout_dict.get('Date'),
content=all_document_parts) # render document template content=all_document_parts) # render document template
htmlpage_fn = "{}.html".format(printout_dict.get('Title').replace(" ", "")) htmlpage_fn = "{}.html".format(
printout_dict.get('Title')[0].replace(" ", ""))
with open(os.path.join(static_html, htmlpage_fn), 'w') as htmlfile: with open(os.path.join(static_html, htmlpage_fn), 'w') as htmlfile:
htmlfile.write(document_html) htmlfile.write(document_html)
all_document_parts = '' # Reset all_document_parts all_document_parts = '' # Reset all_document_parts

@ -3,10 +3,10 @@
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<link rel="stylesheet" href="../static/style.css" /> <link rel="stylesheet" href="../static/style.css" />
<title>{{title}}</title> <title>{{ title[0] }}</title>
</head> </head>
<body> <body>
<h1>{{ title }}</h1> <h1>{{ title[0] }}</h1>
<p><time datetime="{{date}}">{{ date.year }}.{{ date.month }}.{{ date.day }} </time></p> <p><time datetime="{{date}}">{{ date.year }}.{{ date.month }}.{{ date.day }} </time></p>
<div id="content"> <div id="content">
{{ content }} {{ content }}

@ -11,17 +11,32 @@
<div class="metadata"> <div class="metadata">
<h3>Metadata</h3> <h3>Metadata</h3>
{% for key, value in printout_dict.items() %} {% for key, valuelist in printout_dict.items() %}
{% if key == 'Date' %} {% if key == 'Date' %}
<div class="metadata_{{key}}"> <div class="metadata_{{key}}">
<span class="key">{{key}}</span> <span class="key">{{key}}</span>
<span class="value">{{value.year}} {{value.month}} {{value.day}}</span> <span class="value">{{valuelist.year}} {{valuelist.month}} {{valuelist.day}}</span>
{# Note: in Date valuelist is not a list but a dateitme object #}
</div> </div>
{% else %}
<div class="metadata_{{key}}"> <div class="metadata_{{key}}">
{% if key == 'Date' %}
<span class="key">{{key}}</span>
<span class="value">{{valuelist.year}} {{valuelist.month}} {{valuelist.day}}</span>
{% elif key == 'page' %}
<span class="key">{{key|upper}}</span>
<span class="key">{{valuelist}}</span>
{% else %}
<span class="key">{{key|upper}}</span>
{% for val in valuelist %}
<span class="key">{{key|upper}}</span> <span class="key">{{key|upper}}</span>
<span class="value">{{value}}</span> {% endfor %}
{% endif %}
</div> </div>
{# TODO: resolve all these metadata items #}
{% endif %} {% endif %}
{% endfor %} {% endfor %}
</div> </div>

@ -8,7 +8,7 @@
<h3>Results from query:<br/><code>{{query}}</code></h3> <h3>Results from query:<br/><code>{{query}}</code></h3>
<ul> <ul>
{% for doc in documentslist %} {% for doc in documentslist %}
<li><a href="./{{ doc['file'] }}">{{ doc['title'] }}</a> <li><a href="./{{ doc['file'] }}">{{ doc['title'][0] }}</a>
{{ doc['date'].year }}.{{ doc['date'].month }}.{{ doc['date'].day }} {{ doc['date'].year }}.{{ doc['date'].month }}.{{ doc['date'].day }}
{{doc['creator']}} {{doc['creator']}}
</li> </li>

Loading…
Cancel
Save