diff --git a/.gitignore b/.gitignore index 67798e6..4121c5d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ share/ __pycache__/ *.jpg *.jpeg +secrets.py diff --git a/dumpwiki.py b/dumpwiki.py new file mode 100644 index 0000000..03182d3 --- /dev/null +++ b/dumpwiki.py @@ -0,0 +1,67 @@ +import os, json, sys, urllib +from mwclient import Site +from pprint import pprint +from jinja2 import Template +from functions import unpack_response, clean_dir, remove_nonwords + +from functions import Colors +import argparse + +p = argparse.ArgumentParser(description="Dump wiki files to html", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +p.add_argument("--host", metavar='', default="hub.xpub.nl/sandbox", help='wiki host') +p.add_argument("--path", metavar='', default="/itchwiki/", help="Wiki path. Should end with /") +p.add_argument("--output", default="/var/www/html/archive", help="Output path for pages") +# p.add_argument("--conditions", "-c", metavar='', +# default='[[File:+]][[Title::+]][[Part::+]][[Date::+]]', +# help='The query conditions') +# p.add_argument("--printouts", "-p", metavar='', +# default='?Title|?Date|?Part|?Partof|?Creator|?Organization|?Format|?Event|?Topic|?Language', +# help='Selection of properties to printout') +# p.add_argument("--sort", "-s", metavar='', +# default='Date,Title,Part', +# help='Sorting according to conditions') +# p.add_argument("--order", "-o", metavar='', +# default='asc,asc,asc', +# help='Order of sorting conditions. Should same amount as the --sort properties') +# p.add_argument('--limit', '-l', help='(optional) Limit the number of returned ' +# 'items') +# # TODO: GET limit to work.Perhaps with a site.raw_api method +# p.add_argument('--dry', '-d', action='store_true', +# help='dry-run: will only show the query but not run it') + +args = p.parse_args() + + + +# site and login + +site = Site(host=args.host, path=args.path) + +with open('login.txt', 'r') as login: # read login user & pwd + loginlines = login.read() + user, pwd = loginlines.split('\n') + site.login(username=user, password=pwd) # login to wiki + +# read template files + +with open('templates/title.html') as title_html: + title_template = Template(title_html.read()) + +SLASH = "\u2044" + +def filenameforpage(p): + f=p.name.replace(' ','_').replace('/', SLASH) + '.html' + return f + +title=site.Categories['Title'] +for p in title.members(): + print(p) + htmlsrc = site.parse(page=p.name)['text']['*'] + with open(os.path.join(args.output, filenameforpage(p)), 'w') as f: + print(htmlsrc, file=f) + # break + + + + diff --git a/images2html.py b/images2html.py deleted file mode 100644 index b6de07a..0000000 --- a/images2html.py +++ /dev/null @@ -1,76 +0,0 @@ -import os, json -from mwclient import Site -from pprint import pprint -from jinja2 import Template -from functions import pandoc, page_props - -site = Site(host='hub.xpub.nl/sandbox', path='/itchwiki/') -wd = os.path.dirname(os.path.abspath(__file__)) # working directory -imgdir = os.path.join(wd, 'images') -imgsjson_fn = os.path.join(wd, 'images.json') -with open(imgsjson_fn, 'r') as imgsjson_file: - images_info = json.load(imgsjson_file) - -static_html = os.path.join(wd, 'static_html') -os.makedirs(static_html, exist_ok=True) # create images/ dir - -with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd - loginlines = login.read() - user, pwd = loginlines.split('\n') - site.login(username=user, password=pwd) # login to wiki - - -page_html_template = ''' - - - - - - {{title}} - - -

{{ title }}

-

-
- -
-
- {{ content }} -
- - - -''' -page_template = Template(page_html_template) - - -for img_info in images_info.values(): - print(img_info) - page_name = img_info['name'] - page = site.pages[page_name] - # print(page) - # pprint(page.__dict__) - # print(dir(page)) - pagetext = page.text() - pageproperties = page_props(wikicontent=pagetext) - print(pageproperties) - - if pageproperties.get('Title'): - pagetext_html = pandoc(pwd=wd ,content=pagetext, format_in='mediawiki', format_out='html') - # print('pagetext', pagetext) - # print('pagetext_html', pagetext_html) - page_html = page_template.render(title=pageproperties.get('Title'), - date=pageproperties.get('Date'), - imgsrc=os.path.join(imgdir, img_info.get('filename')), - content=pagetext_html, - part=pageproperties.get('Part'), - partof=pageproperties.get('Partof')) - htmlpage_fn = "{}_{}.html".format( - pageproperties.get('Title').replace(" ", ""), - pageproperties.get('Part').zfill(3) - ) - print(htmlpage_fn) - with open(os.path.join(static_html, htmlpage_fn), 'w') as htmlfile: - htmlfile.write(page_html) diff --git a/query2html.py b/query2html.py index 0d51119..c288fb7 100644 --- a/query2html.py +++ b/query2html.py @@ -2,7 +2,7 @@ import os, json, sys, urllib from mwclient import Site from pprint import pprint from jinja2 import Template -from functions import pandoc, unpack_response, clean_dir, remove_nonwords +from functions import unpack_response, clean_dir, remove_nonwords from functions import Colors import argparse @@ -54,7 +54,6 @@ if args.dry is True: site = Site(host=args.host, path=args.path) wd = os.path.dirname(os.path.abspath(__file__)) # working directory -imgdir = os.path.join(wd, 'images') imgsjson_fn = os.path.join(wd, 'images.json') with open(imgsjson_fn, 'r') as imgsjson_file: images_info = json.load(imgsjson_file) @@ -74,11 +73,10 @@ with open(os.path.join(wd, 'templates/index.html')) as document_html: with open(os.path.join(wd, 'templates/document.html')) as document_html: document_template = Template(document_html.read()) -with open(os.path.join(wd, 'templates/document_part.html')) as document_html: - document_part_template = Template(document_html.read()) - all_document_parts = '' # to append all content documentslist = [] +doc_imgs = [] + for answer in site.ask(query): # publication_title = '' # print(answer, answer.keys()) @@ -91,37 +89,42 @@ for answer in site.ask(query): print(Colors.GREEN, 'run python3 download_imgs.py to fix the issue', Colors.ENDC) sys.exit() - page = site.pages[[printout_dict['page']]] # request that page from wiki - pagetext = page.text() - pagetext_html = pandoc(pwd=wd, content=pagetext, format_in='mediawiki', format_out='html') - img_local = os.path.join(imgdir, img_info.get('filename')) + img_local = os.path.join('../images', img_info.get('filename')) - # Todo: Create list of all images from document - # TODO: join document_part + document # TODO: look into the template structure of images : columns and rows + doc_imgs.append(img_local) - # RENDER document part - document_part_html = document_part_template.render( - printout_dict=printout_dict, - imgsrc=os.path.join(imgdir, img_info.get('filename')), - text=pagetext_html, - fullurl=fullurl,) - all_document_parts += document_part_html # append resulting html from document part to the previous parts - + # on last File(Part) of Document if printout_dict['Part'] == printout_dict['Partof']: + print(doc_imgs) + + # organize doc_imgs into a matrix (list of lists) + # each sublist containing 4 items [[0,1,2,3][4,5,6,7],[8,9]] + # for template + doc_imgs_matrix = [doc_imgs[i:i+100] for i in range(0, len(doc_imgs), 100)] + + + + # RENDER DOCUMENT # by passing all_document_parts html to document_template content document_html = document_template.render( title=printout_dict.get('Title'), date=printout_dict.get('Date'), + organization=printout_dict.get('Organization'), + creator=printout_dict.get('Creator'), + format=printout_dict.get('Format'), + topic=printout_dict.get('Topic'), + imgsmatrix=doc_imgs_matrix, content=all_document_parts) # render document template + # print(document_html) + htmlpage_fn = "{}.html".format( remove_nonwords(printout_dict.get('Title')[0]) ) with open(os.path.join(static_html, htmlpage_fn), 'w') as htmlfile: htmlfile.write(document_html) - all_document_parts = '' # Reset all_document_parts # add info to documentslist for index creation documentslist.append({'file': htmlpage_fn, @@ -130,6 +133,11 @@ for answer in site.ask(query): 'creator': printout_dict.get('Creator') }) + # reset document_images list + doc_imgs = [] + + + # RENDER index.html from documentslist index_html = index_template.render(index='Index', query=query, diff --git a/static/style.css b/static/style.css index 27fbd59..62aea22 100644 --- a/static/style.css +++ b/static/style.css @@ -1,14 +1,196 @@ -body{font-size: 12pt;} -div.part {border: 1px solid #e5e5e5; - margin-bottom: 10px; - margin-top: 10px; - padding: 20px;} -div#content img {width: 50%;} +body { + font-family: Arial, Helvetica, sans-serif; + background-color: #aaa4a0; +} +/*div.row {display: inline; } +div.column { display: inline; } +div.column img{ width:24%; }*/ -div.metadata span.key {color: red; - font-weight: bold;} +.grid-container { + display: grid; + grid-template-columns: repeat(4, 1fr); + grid-gap: 10px; + position: absolute; + top: 60%; + width: 96%; + height: auto; + left:2%; + /*background-color: blue;*/ + text-align: : center; + z-index: 5; -#orc {color:blue;} \ No newline at end of file +} + +.grid-container > div { + position: relative; + /*background-color: green;*/ + text-align: center; + /*padding-bottom: 1%;*/ + z-index: 5; +} + +img { + position: relative; + max-width: 100%; + max-height: 100%; + + /*align-content: center;*/ + +} + +/*links*/ + +a:link { + text-decoration: none; + color: black; +} + +a:visited { + text-decoration: none; + color: blue; +} + +a:hover { + text-decoration: none; + color: blue; +} + +h1 { + position: fixed; + right:0%; + top:3%; + text-align: right; + background-color: grey; + padding: 3px 30px 3px 10px; + z-index: 10; + color: black; + font-size: 28px; +} + +h2{ + position: fixed; + right: 0%; + top: 15%; + text-align: left; + background-color: grey; + color: black; + padding: 3px 3px 3px 30px; + z-index: 10; + font-size: 20px; +} + +.indexlist { + position: absolute; + top:20%; +} + + +/* Grid buttons */ + +.header { + + position: absolute; + text-align: center; + padding: 32px; + top: 30%; + right: 0%; + z-index: 9; + width: auto; + height: auto; +} + +p { + display: inline-block; + position: relative; +} + +.btn { + border: none; + outline: none; + width: 110px; + text-align: center; + padding: 12px 0px; + background-color: grey; + cursor: pointer; + font-size: 18px; + margin-right: 5px; + z-index: 9; +} + +.btn:hover { + background-color: black; + color: blue; +} + +.btn:active { + background-color: black; + color: white; +} + + +.metadata{ + position: fixed; + top:5%; + left:0%; + background-color: ; + width: 40%; + z-index: 10; + opacity: 0.7; +} + +.collapsible { + background-color: #777; + color: white; + cursor: pointer; + padding: 18px; + width: 10%; + height: auto; + border: none; + text-align: right; + outline: none; + font-size: 15px; + z-index: 10; + margin-top: 10px; +} + +.active, .collapsible:hover { + background-color: #555; +} + +.content { + position: relative; + padding: 0px 18px; + max-height: 0; + overflow: hidden; + width: 90%; + transition: max-height 0.2s ease-out; + background-color: grey; + color: white; + z-index: 10; + text-align: right; +} + +.metadata_links { + display: block; + width: 100%; + position: relative; + right: 0%; + padding: 10px; + text-align: left; + /*background-color: blue;*/ + font-size: 18px; +} + +.metadata_organization { + font-size: 14px; +} + + + + +.fbtn { + font-style: italic; +} diff --git a/templates/document.html b/templates/document.html index 72687fd..9785cb3 100644 --- a/templates/document.html +++ b/templates/document.html @@ -3,14 +3,85 @@ + {{ title[0] }} -

{{ title[0] }}

-

-
- {{ content }} +

{{ title[0] }} ↵

+

+ +
+ +
+ +
{{ creator }}
+ All Creators +
+ +
+
{{ format }}
+ All Formats +
+ +
+
{{ organization }}
+ All Organizations +
+ +
+
{{ topic }}
+ All Topics +
+
+ + + + + + + + + + + +
+

+

+ + + + {% for row in imgsmatrix %} +
+ {% for img in row %} +
+ +
+ {% endfor %} +
+ {% endfor %} + + - diff --git a/templates/document_part.html b/templates/document_part.html deleted file mode 100644 index 18bcebc..0000000 --- a/templates/document_part.html +++ /dev/null @@ -1,31 +0,0 @@ -
- -
- - {{ fullurl }} -
- -
- {{ text | safe }} -
- -
-

Metadata

- {% for key, valuelist in printout_dict.items() %} -
- {% if key == 'Date' %} - {{key}} - {{valuelist.year}} {{valuelist.month}} {{valuelist.day}} - {% elif key == 'page' %} - {{key|upper}} - {{valuelist}} - {% else %} - {% if valuelist|length > 0 %} - {{key|upper}} - {{valuelist | join(", ")}} - {% endif %} - {% endif %} -
- {% endfor %} -
-
diff --git a/templates/index.html b/templates/index.html index 8e45d6e..505e699 100644 --- a/templates/index.html +++ b/templates/index.html @@ -3,6 +3,7 @@ {{title}} +

Resultsss from query:
{{query}}

diff --git a/templates/title.html b/templates/title.html new file mode 100644 index 0000000..9785cb3 --- /dev/null +++ b/templates/title.html @@ -0,0 +1,87 @@ + + + + + + + {{ title[0] }} + + +

{{ title[0] }} ↵

+

+ +
+ +
+ +
{{ creator }}
+ All Creators +
+ +
+
{{ format }}
+ All Formats +
+ +
+
{{ organization }}
+ All Organizations +
+ +
+
{{ topic }}
+ All Topics +
+
+ + + + + + + + + + + +
+

+

+
+ + + + {% for row in imgsmatrix %} +
+ {% for img in row %} +
+ +
+ {% endfor %} +
+ {% endfor %} + + + +