From 82ff7ca8fc574d32e9bcbb714ef846bf57e982e9 Mon Sep 17 00:00:00 2001 From: "E.zn" Date: Wed, 19 Feb 2020 23:43:13 +0100 Subject: [PATCH 1/7] py/sh --- helper-upload_imgs_dir[EDIT].sh | 23 +++++++ upload_imgs_dir[EDIT].py | 116 ++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100755 helper-upload_imgs_dir[EDIT].sh create mode 100644 upload_imgs_dir[EDIT].py diff --git a/helper-upload_imgs_dir[EDIT].sh b/helper-upload_imgs_dir[EDIT].sh new file mode 100755 index 0000000..8c1a4d3 --- /dev/null +++ b/helper-upload_imgs_dir[EDIT].sh @@ -0,0 +1,23 @@ +#!/bin/sh + +python3 upload_imgs_dir.py \ +--title 'Ang Bayan December 1984' \ +--creator 'Central Committee of the Communist Party of the Philippines' \ +--date '1984/12/01' \ +--org 'Communist Party of the Philippines' \ +--format 'Bulletin' \ +--event 'No' \ +--topic 'Communism, Armed Struggle' \ +--language 'English' \ +--dir '/full/path/to/2020_bantayog/Folder name' \ +# --dry + +# Note: +# * Add this values to you upload specific upload. +# * --dry can be enabled to show you what will be uploaded and the metadata, without actully uploading it +# * parameters --event --topic can be added +# * \ allow you to continue the command of a different line +# +# Get help: python3 upload_imgs_dir.py --help + + diff --git a/upload_imgs_dir[EDIT].py b/upload_imgs_dir[EDIT].py new file mode 100644 index 0000000..6655ff0 --- /dev/null +++ b/upload_imgs_dir[EDIT].py @@ -0,0 +1,116 @@ +import os, argparse, sys, re +from mwclient import (Site, + errors) +from jinja2 import Template +from functions import (print_colormsg, + reorder_imgs) + +p = argparse.ArgumentParser(description='Upload files from a directory, with metadata values to the wiki.\n' + 'Note that any VALUES CONTAINING ' + 'SPACES SHOULD BE BETWEEN QUOTATION MARKS', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +# TODO: Add example of command to description +p.add_argument('--host', default='hub.xpub.nl/sandbox', help='wiki host') +p.add_argument('--path', default='/itchwiki/', help='Wiki path. Should end with /') +p.add_argument('--dry', '-d', action='store_true', + help='dry-run: will only print the metadata of each file that ' + 'will be upload, but does NOT upload') +p.add_argument('--dir', required=True, + help='Required. Full path of the image directory, that you wish to upload') + +p.add_argument('--title', required=True, + help='Required. Must not exist yet in the wiki.') +p.add_argument('--date', required=True, + help='Required. Format: yyyy/mm/dd ' + 'For dates without day or month use 01 as default ' + 'ie. 1986: --date "1986/01/01" ' + 'March 1985: --date "1984/05/01"') +p.add_argument('--creator', required=False, action='append', default=[''], + help='Multiple values should be SEPARATED BY COMMA') +p.add_argument('--org', required=False, action='append', default=[''], + help='Organization:Multiple values should be SEPARATED BY ' + 'COMMA') +p.add_argument('--format', required=False, action='append', default=[''], + help='Multiple values should be SEPARATED BY COMMA') +p.add_argument('--event', required=False, action='append', default=[''], + help='Multiple values should be SEPARATED BY COMMA') +p.add_argument('--topic', required=False, action='append', default=[''], + help='Multiple values should be SEPARATED BY COMMA') +p.add_argument('--language', required=False, action='append', default=[''], + help='Multiple values should be SEPARATED BY COMMA') + +# TODO ADD NEW PROPS +args = p.parse_args() + +# login +site = Site(host=args.host, path=args.path) + +wd =os.path.dirname(os.path.abspath(__file__)) # parent working directory +with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd + loginlines = login.read() + user, pwd = loginlines.split('\n') + site.login(username=user, password=pwd) # login to wiki + +# metadata checks +if os.path.isdir(args.dir) is False: + print_colormsg(f'Error: --dir {args.dir} absolute path cannot be found', level='fail') + sys.exit() +elif not re.match(r'\d{4}\/\d{2}\/\d{2}', args.date): + print_colormsg(f'Error: --date {args.date} format should be --date "yyyy/mm/dd"', level='fail') + sys.exit() +elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: + print_colormsg(f'Error: --title "{args.title}" already exists in wiki. Provide a different one', level='fail') + sys.exit() + +# read template file +with open(os.path.join(wd, 'templates/smw_infobox_template.jinja')) as tmplt: + smw_propval_template = Template(tmplt.read()) + +lsimgs = reorder_imgs(dir=args.dir, dry=args.dry) +dirname = os.path.split(args.dir)[-1].replace(' ', '_') +dirname = re.sub(r'[\W]', '', dirname) #remove non letters or digits +# print('lsimgs:', lsimgs, '\n', dirname) + +for n, _file in enumerate(lsimgs): + pagename = f'{dirname}-{_file}' + print_colormsg(pagename, level='ok') + page = site.pages[_file] + + if page.exists: + url = page.imageinfo['descriptionurl'] + print_colormsg( + f'Already exists in {url} Will NOT be uploaded', + level='warning') + else: + img_smw_prop_val = smw_propval_template.render( + title=args.title, + date=args.date, + part=n + 1, + partof=len(lsimgs), + creator=(', ').join(args.creator[1:]), + organization=(', ').join(args.org[1:]), + format=(', ').join(args.format[1:]), + event=(', ').join(args.event[1:]), + topic=(', ').join(args.topic[1:]) + language=(', ').join(args.language[1:]), + ) + + _file_path = os.path.join(args.dir, _file) + if not args.dry: + pageurl = f'https://{args.host}{args.path}index.php/File:{pagename}' + with open(_file_path, 'rb') as _f: + try: + site.upload(file=_file_path, + filename=pagename, + description=img_smw_prop_val, + ignore=True) + print(img_smw_prop_val) + except errors.APIError as e: + print_colormsg(f'Error: {e.info}\n' + f'It will not be uploaded', + level='fail') + + print(f'See image at {pageurl}') + else: + print(img_smw_prop_val) + From 791acf103e5d8223227ca0a21197f5f8f44a0406 Mon Sep 17 00:00:00 2001 From: "E.zn" Date: Thu, 20 Feb 2020 02:09:08 +0100 Subject: [PATCH 2/7] missing comma --- upload_imgs_dir[EDIT].py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upload_imgs_dir[EDIT].py b/upload_imgs_dir[EDIT].py index 6655ff0..d25cc99 100644 --- a/upload_imgs_dir[EDIT].py +++ b/upload_imgs_dir[EDIT].py @@ -91,7 +91,7 @@ for n, _file in enumerate(lsimgs): organization=(', ').join(args.org[1:]), format=(', ').join(args.format[1:]), event=(', ').join(args.event[1:]), - topic=(', ').join(args.topic[1:]) + topic=(', ').join(args.topic[1:]), language=(', ').join(args.language[1:]), ) From 80e346a2c9206c469f4f024466f287e242ab93db Mon Sep 17 00:00:00 2001 From: "E.zn" Date: Thu, 20 Feb 2020 22:24:32 +0100 Subject: [PATCH 3/7] ezn changes save --- helper-upload_imgs_dir[EDIT].sh | 23 ------- upload_imgs_dir[EDIT].py | 116 -------------------------------- 2 files changed, 139 deletions(-) delete mode 100755 helper-upload_imgs_dir[EDIT].sh delete mode 100644 upload_imgs_dir[EDIT].py diff --git a/helper-upload_imgs_dir[EDIT].sh b/helper-upload_imgs_dir[EDIT].sh deleted file mode 100755 index 8c1a4d3..0000000 --- a/helper-upload_imgs_dir[EDIT].sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -python3 upload_imgs_dir.py \ ---title 'Ang Bayan December 1984' \ ---creator 'Central Committee of the Communist Party of the Philippines' \ ---date '1984/12/01' \ ---org 'Communist Party of the Philippines' \ ---format 'Bulletin' \ ---event 'No' \ ---topic 'Communism, Armed Struggle' \ ---language 'English' \ ---dir '/full/path/to/2020_bantayog/Folder name' \ -# --dry - -# Note: -# * Add this values to you upload specific upload. -# * --dry can be enabled to show you what will be uploaded and the metadata, without actully uploading it -# * parameters --event --topic can be added -# * \ allow you to continue the command of a different line -# -# Get help: python3 upload_imgs_dir.py --help - - diff --git a/upload_imgs_dir[EDIT].py b/upload_imgs_dir[EDIT].py deleted file mode 100644 index d25cc99..0000000 --- a/upload_imgs_dir[EDIT].py +++ /dev/null @@ -1,116 +0,0 @@ -import os, argparse, sys, re -from mwclient import (Site, - errors) -from jinja2 import Template -from functions import (print_colormsg, - reorder_imgs) - -p = argparse.ArgumentParser(description='Upload files from a directory, with metadata values to the wiki.\n' - 'Note that any VALUES CONTAINING ' - 'SPACES SHOULD BE BETWEEN QUOTATION MARKS', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) -# TODO: Add example of command to description -p.add_argument('--host', default='hub.xpub.nl/sandbox', help='wiki host') -p.add_argument('--path', default='/itchwiki/', help='Wiki path. Should end with /') -p.add_argument('--dry', '-d', action='store_true', - help='dry-run: will only print the metadata of each file that ' - 'will be upload, but does NOT upload') -p.add_argument('--dir', required=True, - help='Required. Full path of the image directory, that you wish to upload') - -p.add_argument('--title', required=True, - help='Required. Must not exist yet in the wiki.') -p.add_argument('--date', required=True, - help='Required. Format: yyyy/mm/dd ' - 'For dates without day or month use 01 as default ' - 'ie. 1986: --date "1986/01/01" ' - 'March 1985: --date "1984/05/01"') -p.add_argument('--creator', required=False, action='append', default=[''], - help='Multiple values should be SEPARATED BY COMMA') -p.add_argument('--org', required=False, action='append', default=[''], - help='Organization:Multiple values should be SEPARATED BY ' - 'COMMA') -p.add_argument('--format', required=False, action='append', default=[''], - help='Multiple values should be SEPARATED BY COMMA') -p.add_argument('--event', required=False, action='append', default=[''], - help='Multiple values should be SEPARATED BY COMMA') -p.add_argument('--topic', required=False, action='append', default=[''], - help='Multiple values should be SEPARATED BY COMMA') -p.add_argument('--language', required=False, action='append', default=[''], - help='Multiple values should be SEPARATED BY COMMA') - -# TODO ADD NEW PROPS -args = p.parse_args() - -# login -site = Site(host=args.host, path=args.path) - -wd =os.path.dirname(os.path.abspath(__file__)) # parent working directory -with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd - loginlines = login.read() - user, pwd = loginlines.split('\n') - site.login(username=user, password=pwd) # login to wiki - -# metadata checks -if os.path.isdir(args.dir) is False: - print_colormsg(f'Error: --dir {args.dir} absolute path cannot be found', level='fail') - sys.exit() -elif not re.match(r'\d{4}\/\d{2}\/\d{2}', args.date): - print_colormsg(f'Error: --date {args.date} format should be --date "yyyy/mm/dd"', level='fail') - sys.exit() -elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: - print_colormsg(f'Error: --title "{args.title}" already exists in wiki. Provide a different one', level='fail') - sys.exit() - -# read template file -with open(os.path.join(wd, 'templates/smw_infobox_template.jinja')) as tmplt: - smw_propval_template = Template(tmplt.read()) - -lsimgs = reorder_imgs(dir=args.dir, dry=args.dry) -dirname = os.path.split(args.dir)[-1].replace(' ', '_') -dirname = re.sub(r'[\W]', '', dirname) #remove non letters or digits -# print('lsimgs:', lsimgs, '\n', dirname) - -for n, _file in enumerate(lsimgs): - pagename = f'{dirname}-{_file}' - print_colormsg(pagename, level='ok') - page = site.pages[_file] - - if page.exists: - url = page.imageinfo['descriptionurl'] - print_colormsg( - f'Already exists in {url} Will NOT be uploaded', - level='warning') - else: - img_smw_prop_val = smw_propval_template.render( - title=args.title, - date=args.date, - part=n + 1, - partof=len(lsimgs), - creator=(', ').join(args.creator[1:]), - organization=(', ').join(args.org[1:]), - format=(', ').join(args.format[1:]), - event=(', ').join(args.event[1:]), - topic=(', ').join(args.topic[1:]), - language=(', ').join(args.language[1:]), - ) - - _file_path = os.path.join(args.dir, _file) - if not args.dry: - pageurl = f'https://{args.host}{args.path}index.php/File:{pagename}' - with open(_file_path, 'rb') as _f: - try: - site.upload(file=_file_path, - filename=pagename, - description=img_smw_prop_val, - ignore=True) - print(img_smw_prop_val) - except errors.APIError as e: - print_colormsg(f'Error: {e.info}\n' - f'It will not be uploaded', - level='fail') - - print(f'See image at {pageurl}') - else: - print(img_smw_prop_val) - From 3060147e648ba1cb1c0aaf0140ac0e6e2a1b7639 Mon Sep 17 00:00:00 2001 From: "E.zn" Date: Fri, 21 Feb 2020 12:43:20 +0100 Subject: [PATCH 4/7] .sh/.py/.jinja --- helper-upload_imgs_dir.sh | 18 ++++++++++-------- templates/smw_infobox_template.jinja | 3 ++- upload_imgs_dir.py | 5 ++++- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/helper-upload_imgs_dir.sh b/helper-upload_imgs_dir.sh index 138539f..9e585d7 100755 --- a/helper-upload_imgs_dir.sh +++ b/helper-upload_imgs_dir.sh @@ -1,14 +1,16 @@ #!/bin/sh python3 upload_imgs_dir.py \ ---title 'Ang Bayan December 1984' \ ---creator 'Central Committee of the Communist Party of the Philippines' \ ---date '1984/12/01' \ ---org 'Communist Party of the Philippines' \ ---format 'Bulletin' \ ---topic 'Communism, Armed Struggle' \ ---dir '/full/path/to/2020_bantayog/Folder name' \ -# --dry +--title 'No' \ +--creator 'a' \ +--date '2020/01/01' \ +--org 'b' \ +--format 'c' \ +--event 'd' \ +--topic 'e' \ +--language 'English' \ +--dir '/home/ezn/Desktop/XPUB/Special_Issue_XI_[Ezn]/D.lo/2020_bantayog/Kalayaan Vol. 12 No. 1' \ +--dry # Note: # * Add this values to you upload specific upload. diff --git a/templates/smw_infobox_template.jinja b/templates/smw_infobox_template.jinja index ab8d340..5461111 100644 --- a/templates/smw_infobox_template.jinja +++ b/templates/smw_infobox_template.jinja @@ -8,5 +8,6 @@ |Format={{ format }} |Event={{ event }} |Topic={{ topic }} +|Language={{ language }} {{ '}}' }} -[[Template:ImageMetadata]] \ No newline at end of file +[[Template:ImageMetadata]] diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index 3c47d8d..d25cc99 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -36,6 +36,8 @@ p.add_argument('--event', required=False, action='append', default=[''], help='Multiple values should be SEPARATED BY COMMA') p.add_argument('--topic', required=False, action='append', default=[''], help='Multiple values should be SEPARATED BY COMMA') +p.add_argument('--language', required=False, action='append', default=[''], + help='Multiple values should be SEPARATED BY COMMA') # TODO ADD NEW PROPS args = p.parse_args() @@ -89,7 +91,8 @@ for n, _file in enumerate(lsimgs): organization=(', ').join(args.org[1:]), format=(', ').join(args.format[1:]), event=(', ').join(args.event[1:]), - topic=(', ').join(args.topic[1:]) + topic=(', ').join(args.topic[1:]), + language=(', ').join(args.language[1:]), ) _file_path = os.path.join(args.dir, _file) From a54fcac59b38fc01a05622d744ab5140fac9b853 Mon Sep 17 00:00:00 2001 From: "E.zn" Date: Fri, 21 Feb 2020 17:16:15 +0100 Subject: [PATCH 5/7] sh/py --- helper-upload_imgs_dir.sh | 22 +++++++++++++--------- upload_imgs_dir.py | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/helper-upload_imgs_dir.sh b/helper-upload_imgs_dir.sh index 9e585d7..48d601b 100755 --- a/helper-upload_imgs_dir.sh +++ b/helper-upload_imgs_dir.sh @@ -1,16 +1,17 @@ #!/bin/sh python3 upload_imgs_dir.py \ ---title 'No' \ ---creator 'a' \ ---date '2020/01/01' \ ---org 'b' \ ---format 'c' \ ---event 'd' \ ---topic 'e' \ ---language 'English' \ +--title 'NO' \ +--creator 'zzzzzzzzzzzzzzzzzzzzzz' \ +--date '1990/12/01' \ +--org 'aaaaaaaaa' \ +--format 'bbbbbbbbbbb' \ +--event 'dddddddddd' \ +--topic 'ccccc' \ +--language 'English,Tagalog' \ --dir '/home/ezn/Desktop/XPUB/Special_Issue_XI_[Ezn]/D.lo/2020_bantayog/Kalayaan Vol. 12 No. 1' \ ---dry + +#--dry # Note: # * Add this values to you upload specific upload. @@ -21,3 +22,6 @@ python3 upload_imgs_dir.py \ # Get help: python3 upload_imgs_dir.py --help + + + diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index d25cc99..0458f4f 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -92,7 +92,7 @@ for n, _file in enumerate(lsimgs): format=(', ').join(args.format[1:]), event=(', ').join(args.event[1:]), topic=(', ').join(args.topic[1:]), - language=(', ').join(args.language[1:]), + language=(', ').join(args.language[1:]) ) _file_path = os.path.join(args.dir, _file) From 048277c720efdd536aec841f88f6e14f1dac11c7 Mon Sep 17 00:00:00 2001 From: "E.zn" Date: Fri, 21 Feb 2020 17:41:51 +0100 Subject: [PATCH 6/7] sh --- helper-upload_imgs_dir.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/helper-upload_imgs_dir.sh b/helper-upload_imgs_dir.sh index 48d601b..df57bdb 100755 --- a/helper-upload_imgs_dir.sh +++ b/helper-upload_imgs_dir.sh @@ -1,15 +1,15 @@ #!/bin/sh python3 upload_imgs_dir.py \ ---title 'NO' \ ---creator 'zzzzzzzzzzzzzzzzzzzzzz' \ ---date '1990/12/01' \ ---org 'aaaaaaaaa' \ ---format 'bbbbbbbbbbb' \ ---event 'dddddddddd' \ ---topic 'ccccc' \ ---language 'English,Tagalog' \ ---dir '/home/ezn/Desktop/XPUB/Special_Issue_XI_[Ezn]/D.lo/2020_bantayog/Kalayaan Vol. 12 No. 1' \ +--title 'Nassa News: Vol. 21 No. 7' \ +--creator 'The National Secretariat of Social Action Justice and Peace' \ +--date '1984/07/01' \ +--org 'Nationalist Alliance for Justice, Freedom and Democracy (NAJFD)' \ +--format 'Statement' \ +--event 'The ARTEX Strike' \ +--topic 'Workers rights' \ +--language 'English' \ +--dir '/full/path/to/2020_bantayog/Folder name' \ #--dry From e80e0c61164c2374e2c2f92e8202c4d979eba549 Mon Sep 17 00:00:00 2001 From: Castro0o Date: Fri, 21 Feb 2020 17:43:03 +0100 Subject: [PATCH 7/7] helper script with example values and Note --- helper-upload_imgs_dir.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/helper-upload_imgs_dir.sh b/helper-upload_imgs_dir.sh index 48d601b..a2922f8 100755 --- a/helper-upload_imgs_dir.sh +++ b/helper-upload_imgs_dir.sh @@ -1,16 +1,15 @@ #!/bin/sh python3 upload_imgs_dir.py \ ---title 'NO' \ ---creator 'zzzzzzzzzzzzzzzzzzzzzz' \ ---date '1990/12/01' \ ---org 'aaaaaaaaa' \ ---format 'bbbbbbbbbbb' \ ---event 'dddddddddd' \ ---topic 'ccccc' \ +--title 'Ang Bayan December 1984' \ +--creator 'Central Committee of the Communist Party of the Philippines' \ +--date '1984/12/01' \ +--org 'Communist Party of the Philippines' \ +--format 'Bulletin' \ +--topic 'Communism,Armed Struggle' \ +--event 'Event1,Event2' \ --language 'English,Tagalog' \ ---dir '/home/ezn/Desktop/XPUB/Special_Issue_XI_[Ezn]/D.lo/2020_bantayog/Kalayaan Vol. 12 No. 1' \ - +--dir '/full/path/to/2020_bantayog/document directory' \ #--dry # Note: