From cb80a82fe5002947a20f008973fb45152138ecd1 Mon Sep 17 00:00:00 2001 From: Castro0o Date: Mon, 10 Feb 2020 12:03:27 +0100 Subject: [PATCH 01/14] moved hyperlink to below img --- templates/document_part.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/templates/document_part.html b/templates/document_part.html index 5524bae..c435c5a 100644 --- a/templates/document_part.html +++ b/templates/document_part.html @@ -1,9 +1,8 @@
- - - + + {{ fullurl }}
-- 2.30.2 From dcec809e997289e2ea39dcd6f623a97c0c0feec3 Mon Sep 17 00:00:00 2001 From: Castro0o Date: Fri, 14 Feb 2020 17:04:15 +0100 Subject: [PATCH 02/14] 1st img upload script --- .gitignore | 2 ++ sandbox/upload_imgs_dir.py | 58 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 sandbox/upload_imgs_dir.py diff --git a/.gitignore b/.gitignore index 929ba43..67798e6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ lib64 pyvenv.cfg share/ __pycache__/ +*.jpg +*.jpeg diff --git a/sandbox/upload_imgs_dir.py b/sandbox/upload_imgs_dir.py new file mode 100644 index 0000000..f062b25 --- /dev/null +++ b/sandbox/upload_imgs_dir.py @@ -0,0 +1,58 @@ +import os, argparse +from mwclient import Site + + +p = argparse.ArgumentParser(description="Upload files from a folder", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +p.add_argument("--host", metavar='', default="hub.xpub.nl/sandbox", help='wiki host') +p.add_argument("--path", metavar='', default="/itchwiki/", help="Wiki path. Should end with /") +p.add_argument("--scheme", default="https", help="http or https") + +args = p.parse_args() + +site = Site(host=args.host, path=args.path, scheme=args.scheme) + +wd = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # parent working directory + +with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd + loginlines = login.read() + user, pwd = loginlines.split('\n') + site.login(username=user, password=pwd) # login to wiki + +site = Site(host=args.host, path=args.path) + +# login +with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd + loginlines = login.read() + user, pwd = loginlines.split('\n') + site.login(username=user, password=pwd) # login to wiki + +# todo: add arguments: title date creator imgsdir (full path) +# todo: check title existance, chech date format +# todo: loop though images in folder (.jpg, .jpeg, .png) ensuring correct order 1 before 11 + + +smw_prop_val = '''{{ImageMetadata +|Title={title} +|Date={date} +|Part={part} +|Partof={partof} +|Creator={creator} +}} +[[Template:ImageMetadata]] +''' + +img_smw_prop_val = smw_prop_val.format(title='cat from hell', date='2020/02/13', part='1', partof='1', creator='test') +print(img_smw_prop_val) + +filename = 'cat2.jpg' +page = site.images[filename] +img_url = f'https://{args.host}{args.path}/index.php/File:{filename}' + +if page.exists is True: + print(f'{filename} is already upload it, you can see it at {img_url}' ) + print('The upload process wont proceed. Please upload all images in folder by hand') +else: + with open(filename, 'rb') as _file: + site.upload(file=_file, filename=filename, description='img_smw_prop_val', ignore=True) + -- 2.30.2 From 15f9322eb117ab68bba4168da993e5167185ed7c Mon Sep 17 00:00:00 2001 From: Castro0o Date: Sat, 15 Feb 2020 18:16:26 +0100 Subject: [PATCH 03/14] arguments --- sandbox/upload_imgs_dir.py | 44 ++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/sandbox/upload_imgs_dir.py b/sandbox/upload_imgs_dir.py index f062b25..d43e46f 100644 --- a/sandbox/upload_imgs_dir.py +++ b/sandbox/upload_imgs_dir.py @@ -1,36 +1,62 @@ -import os, argparse +import os, argparse, sys, re from mwclient import Site -p = argparse.ArgumentParser(description="Upload files from a folder", +p = argparse.ArgumentParser(description='Upload files from a directory, with metadata values to the wiki.\n' + 'Note that any value containing spaces should be between quotation marks', formatter_class=argparse.ArgumentDefaultsHelpFormatter) -p.add_argument("--host", metavar='', default="hub.xpub.nl/sandbox", help='wiki host') -p.add_argument("--path", metavar='', default="/itchwiki/", help="Wiki path. Should end with /") -p.add_argument("--scheme", default="https", help="http or https") +# TODO: Add example of command to description +p.add_argument('--host', default='hub.xpub.nl/sandbox', help='wiki host') +p.add_argument('--path', default='/itchwiki/', help='Wiki path. Should end with /') +p.add_argument('--scheme', default='https', help='http or https') +p.add_argument('--dry', '-d', action='store_true', + help='dry-run: will only print the metadata of each file that will be upload, but does NOT upload') +p.add_argument('--title', required=True, help='Metadata **Title** value of publication. Must not exist yet in the wiki.') +p.add_argument('--date', required=True, help='Metadata **Date** value of publication. Format yyyy/mm/dd ' + 'For dates without day or date use 01 ie. 1986 --date "1986/01/01" March 1985: --date "1984/05/01"') +p.add_argument('--creator', required=True, action='append', help='Metadata **Creator** value(s) of publication.' + 'Multiple values should be SEPARATED BY COMMA') +p.add_argument('--dir', required=True, help='Full path of the image directory, that you wish to upload') args = p.parse_args() + site = Site(host=args.host, path=args.path, scheme=args.scheme) wd = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # parent working directory - with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd loginlines = login.read() user, pwd = loginlines.split('\n') site.login(username=user, password=pwd) # login to wiki site = Site(host=args.host, path=args.path) - # login with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd loginlines = login.read() user, pwd = loginlines.split('\n') site.login(username=user, password=pwd) # login to wiki -# todo: add arguments: title date creator imgsdir (full path) -# todo: check title existance, chech date format # todo: loop though images in folder (.jpg, .jpeg, .png) ensuring correct order 1 before 11 +if os.path.isdir(args.dir) is False: + print(f'Error: --dir {args.dir} absolute path cannot be found') + sys.exit() +elif not re.match(r'\d{4}\/\d{2}\/\d{2}', args.date): + print(f'Error: --date {args.date} format should be --date "yyyy/mm/dd"') + sys.exit() +elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: + print(f'Error: --title "{args.title}" already exists in wiki. Please provide a different one') + sys.exit() + + +for _file in os.listdir(args.dir): + print(_file) + + +if args.dry == True: + print(args) + sys.exit() + smw_prop_val = '''{{ImageMetadata |Title={title} -- 2.30.2 From 94f45441b615f23a330a1aebc085ca2260c36f64 Mon Sep 17 00:00:00 2001 From: Castro0o Date: Sat, 15 Feb 2020 18:45:14 +0100 Subject: [PATCH 04/14] warnings --- functions.py | 9 ++++++++ .../upload_imgs_dir.py => upload_imgs_dir.py | 21 +++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) rename sandbox/upload_imgs_dir.py => upload_imgs_dir.py (79%) diff --git a/functions.py b/functions.py index 2c9b054..da7b14c 100644 --- a/functions.py +++ b/functions.py @@ -94,6 +94,15 @@ def clean_dir(dirfullpath): if os.path.isfile(f): os.remove(f) +def print_colormsg(msg, level): + if level == 'fail': + print(Colors.FAIL) + elif level == 'warning': + print(Colors.WARNING) + elif level == 'ok': + print(Colors.BLUE) + print(msg) + print(Colors.ENDC) class Colors: diff --git a/sandbox/upload_imgs_dir.py b/upload_imgs_dir.py similarity index 79% rename from sandbox/upload_imgs_dir.py rename to upload_imgs_dir.py index d43e46f..0ec4fe2 100644 --- a/sandbox/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -1,6 +1,6 @@ import os, argparse, sys, re from mwclient import Site - +from functions import print_colormsg p = argparse.ArgumentParser(description='Upload files from a directory, with metadata values to the wiki.\n' 'Note that any value containing spaces should be between quotation marks', @@ -23,7 +23,7 @@ args = p.parse_args() site = Site(host=args.host, path=args.path, scheme=args.scheme) -wd = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # parent working directory +wd =os.path.dirname(os.path.abspath(__file__)) # parent working directory with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd loginlines = login.read() user, pwd = loginlines.split('\n') @@ -36,22 +36,25 @@ with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd user, pwd = loginlines.split('\n') site.login(username=user, password=pwd) # login to wiki -# todo: loop though images in folder (.jpg, .jpeg, .png) ensuring correct order 1 before 11 if os.path.isdir(args.dir) is False: - print(f'Error: --dir {args.dir} absolute path cannot be found') + print_colormsg(f'Error: --dir {args.dir} absolute path cannot be found', level='fail') sys.exit() elif not re.match(r'\d{4}\/\d{2}\/\d{2}', args.date): - print(f'Error: --date {args.date} format should be --date "yyyy/mm/dd"') + print_colormsg(f'Error: --date {args.date} format should be --date "yyyy/mm/dd"', level='fail') sys.exit() elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: - print(f'Error: --title "{args.title}" already exists in wiki. Please provide a different one') + print_colormsg(f'Error: --title "{args.title}" already exists in wiki. Provide a different one', level='fail') sys.exit() - +# TODO os.listdir(args.dir) ORDER! +# TODO PartOf from os.listdir(args.dir) that fit on to condition +# TODO Part base on os.lisdir order! for _file in os.listdir(args.dir): - print(_file) - + if (os.path.splitext(_file)[-1]).lower() not in ['.jpg', '.jpeg', '.png']: + print_colormsg(f'File: {_file} is not a .jpg .jpeg or .png hence will Not be upload', level='warning') + else: + print_colormsg(_file, level='ok') if args.dry == True: print(args) -- 2.30.2 From 1c8ca7d045f0e9ae3a189f800bef6d22b6d83050 Mon Sep 17 00:00:00 2001 From: Castro0o Date: Tue, 18 Feb 2020 14:01:50 +0100 Subject: [PATCH 05/14] wip: list images --- upload_imgs_dir.py | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index 0ec4fe2..085384b 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -20,7 +20,7 @@ p.add_argument('--creator', required=True, action='append', help='Metadata **Cre p.add_argument('--dir', required=True, help='Full path of the image directory, that you wish to upload') args = p.parse_args() - +# login site = Site(host=args.host, path=args.path, scheme=args.scheme) wd =os.path.dirname(os.path.abspath(__file__)) # parent working directory @@ -29,14 +29,7 @@ with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd user, pwd = loginlines.split('\n') site.login(username=user, password=pwd) # login to wiki -site = Site(host=args.host, path=args.path) -# login -with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd - loginlines = login.read() - user, pwd = loginlines.split('\n') - site.login(username=user, password=pwd) # login to wiki - - +# metadata checks if os.path.isdir(args.dir) is False: print_colormsg(f'Error: --dir {args.dir} absolute path cannot be found', level='fail') sys.exit() @@ -47,20 +40,16 @@ elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: print_colormsg(f'Error: --title "{args.title}" already exists in wiki. Provide a different one', level='fail') sys.exit() + # TODO os.listdir(args.dir) ORDER! +lsimgs = [_file for _file in os.listdir(args.dir) if (os.path.splitext(_file)[-1]).lower() in + ['.jpg', '.jpeg', '.png']] +lsimgs.sort() +print('lsimgs:', lsimgs) # TODO PartOf from os.listdir(args.dir) that fit on to condition # TODO Part base on os.lisdir order! -for _file in os.listdir(args.dir): - if (os.path.splitext(_file)[-1]).lower() not in ['.jpg', '.jpeg', '.png']: - print_colormsg(f'File: {_file} is not a .jpg .jpeg or .png hence will Not be upload', level='warning') - else: - print_colormsg(_file, level='ok') - -if args.dry == True: - print(args) - sys.exit() - +# todo creator value comma-seperated smw_prop_val = '''{{ImageMetadata |Title={title} |Date={date} @@ -71,6 +60,25 @@ smw_prop_val = '''{{ImageMetadata [[Template:ImageMetadata]] ''' +if args.dry == True: + print(args) + sys.exit() + +for n _file in enumerate(lsimgs): + print_colormsg(_file, level='ok') + page = site.pages[_file] + if page.exists: + url = page.imageinfo['descriptionurl'] + print_colormsg( + f'Already exists in {url} Will be uploaded as new version', + level='warning') + smw_prop_val.format(title=, + ) + + + + + img_smw_prop_val = smw_prop_val.format(title='cat from hell', date='2020/02/13', part='1', partof='1', creator='test') print(img_smw_prop_val) -- 2.30.2 From 59ba70bebb0e840dcfb28ab8be5f9660c5a5743f Mon Sep 17 00:00:00 2001 From: Castro0o Date: Tue, 18 Feb 2020 15:56:00 +0100 Subject: [PATCH 06/14] images reorder --- functions.py | 38 +++++++++++++++++++++++++++++++++++++- upload_imgs_dir.py | 26 ++++++++++++++++---------- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/functions.py b/functions.py index da7b14c..09b7305 100644 --- a/functions.py +++ b/functions.py @@ -113,4 +113,40 @@ class Colors: FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' - UNDERLINE = '\033[4m' \ No newline at end of file + UNDERLINE = '\033[4m' + +def listimgs(dir): + lsimgs = [_file for _file in os.listdir(dir) if + (os.path.splitext(_file)[-1]).lower() in + ['.jpg', '.jpeg', '.png']] + lsimgs.sort() + return lsimgs + + +def reorder_imgs(dir, dry): + lsimgs = listimgs(dir) + for img in lsimgs: + img_name, img_ext = os.path.splitext(img) + + # does file follow \d{1,}\.img_ext + numb_exp = re.compile( + r'(?P.*?)(?P\d+)(?P%s)'% re.escape(img_ext)) + match = re.search(numb_exp, img) + if not match: + print(f'Image {img} Filename is not suitable for bulk upload.' + f'Filename pattern does not match 1.jpg 01.jpg' + f'You have to DO IT MANUALLY') + else: + # only change name of single digit numbers + if len(match.groupdict()['num']) == 1: + name = match.groupdict()['name'] + num = match.groupdict()['num'].zfill(2) # pad with 0 + ext = match.groupdict()['ext'] + new_img = name + num + ext + if dry == False: + os.rename(os.path.join(dir, img), + os.path.join(dir, new_img)) + print(f'Renaming: {img} >>>>> {new_img}') + + return listimgs(dir) + diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index 085384b..58e2154 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -1,9 +1,11 @@ import os, argparse, sys, re from mwclient import Site -from functions import print_colormsg +from functions import (print_colormsg, + reorder_imgs) p = argparse.ArgumentParser(description='Upload files from a directory, with metadata values to the wiki.\n' - 'Note that any value containing spaces should be between quotation marks', + 'Note that any VALUES CONTAINING ' + 'SPACES SHOULD BE BETWEEN QUOTATION MARKS', formatter_class=argparse.ArgumentDefaultsHelpFormatter) # TODO: Add example of command to description p.add_argument('--host', default='hub.xpub.nl/sandbox', help='wiki host') @@ -17,7 +19,8 @@ p.add_argument('--date', required=True, help='Metadata **Date** value of publica 'For dates without day or date use 01 ie. 1986 --date "1986/01/01" March 1985: --date "1984/05/01"') p.add_argument('--creator', required=True, action='append', help='Metadata **Creator** value(s) of publication.' 'Multiple values should be SEPARATED BY COMMA') -p.add_argument('--dir', required=True, help='Full path of the image directory, that you wish to upload') +p.add_argument('--dir', required=True, help='Full path of the image directory, ' + 'that you wish to upload') args = p.parse_args() # login @@ -42,10 +45,13 @@ elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: # TODO os.listdir(args.dir) ORDER! -lsimgs = [_file for _file in os.listdir(args.dir) if (os.path.splitext(_file)[-1]).lower() in - ['.jpg', '.jpeg', '.png']] -lsimgs.sort() + +lsimgs = reorder_imgs(dir=args.dir, dry=args.dry) + print('lsimgs:', lsimgs) + +sys.exit() ## STOP HERE + # TODO PartOf from os.listdir(args.dir) that fit on to condition # TODO Part base on os.lisdir order! @@ -64,7 +70,7 @@ if args.dry == True: print(args) sys.exit() -for n _file in enumerate(lsimgs): +for n, _file in enumerate(lsimgs): print_colormsg(_file, level='ok') page = site.pages[_file] if page.exists: @@ -72,8 +78,6 @@ for n _file in enumerate(lsimgs): print_colormsg( f'Already exists in {url} Will be uploaded as new version', level='warning') - smw_prop_val.format(title=, - ) @@ -91,5 +95,7 @@ if page.exists is True: print('The upload process wont proceed. Please upload all images in folder by hand') else: with open(filename, 'rb') as _file: - site.upload(file=_file, filename=filename, description='img_smw_prop_val', ignore=True) + dirname = (os.path.split(dir)[-1]) + site.upload(file=_file, filename=f'{dirname}_{filename}', + description='img_smw_prop_val', ignore=True) -- 2.30.2 From 82a6404b40dc215c66ca1f6f75a61d4a918ce6ac Mon Sep 17 00:00:00 2001 From: Castro0o Date: Tue, 18 Feb 2020 21:47:27 +0100 Subject: [PATCH 07/14] padding folder files with zeros for correct order --- functions.py | 22 +++++++++++++--------- upload_imgs_dir.py | 1 - 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/functions.py b/functions.py index 09b7305..b7b9035 100644 --- a/functions.py +++ b/functions.py @@ -1,4 +1,4 @@ -import os, json, re, shlex +import os, json, re, shlex, sys import subprocess from datetime import datetime @@ -124,29 +124,33 @@ def listimgs(dir): def reorder_imgs(dir, dry): + # does zero pad file numbers + lsimgs = listimgs(dir) for img in lsimgs: img_name, img_ext = os.path.splitext(img) - + print(img) # does file follow \d{1,}\.img_ext numb_exp = re.compile( r'(?P.*?)(?P\d+)(?P%s)'% re.escape(img_ext)) match = re.search(numb_exp, img) if not match: print(f'Image {img} Filename is not suitable for bulk upload.' - f'Filename pattern does not match 1.jpg 01.jpg' + f'Filename pattern dn\'t match 1.jpg 01.jpg something01.jpg' f'You have to DO IT MANUALLY') + sys.exit() else: # only change name of single digit numbers if len(match.groupdict()['num']) == 1: name = match.groupdict()['name'] - num = match.groupdict()['num'].zfill(2) # pad with 0 + num = match.groupdict()['num'].zfill(3) # pad with 0s ext = match.groupdict()['ext'] new_img = name + num + ext + src_img = os.path.join(dir, img) + dst_img = os.path.join(dir, new_img) + print(f'Renaming: {img} >>>>> {new_img}') if dry == False: - os.rename(os.path.join(dir, img), - os.path.join(dir, new_img)) - print(f'Renaming: {img} >>>>> {new_img}') - - return listimgs(dir) + os.replace(src_img, dst_img) + lsimgs = listimgs(dir) # update list w/ renamed imgs + return listimgs(dir) diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index 58e2154..062e896 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -44,7 +44,6 @@ elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: sys.exit() -# TODO os.listdir(args.dir) ORDER! lsimgs = reorder_imgs(dir=args.dir, dry=args.dry) -- 2.30.2 From 2b0e6654e0c60ec1193f7c0f15f005ea89ef62d8 Mon Sep 17 00:00:00 2001 From: Castro0o Date: Tue, 18 Feb 2020 21:49:17 +0100 Subject: [PATCH 08/14] small corrections to def reorder_imgs --- functions.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/functions.py b/functions.py index b7b9035..f938d24 100644 --- a/functions.py +++ b/functions.py @@ -125,11 +125,10 @@ def listimgs(dir): def reorder_imgs(dir, dry): # does zero pad file numbers - + # and returns correct order of files lsimgs = listimgs(dir) for img in lsimgs: img_name, img_ext = os.path.splitext(img) - print(img) # does file follow \d{1,}\.img_ext numb_exp = re.compile( r'(?P.*?)(?P\d+)(?P%s)'% re.escape(img_ext)) @@ -151,6 +150,5 @@ def reorder_imgs(dir, dry): print(f'Renaming: {img} >>>>> {new_img}') if dry == False: os.replace(src_img, dst_img) - lsimgs = listimgs(dir) # update list w/ renamed imgs - return listimgs(dir) + return listimgs(dir) # update list w/ renamed imgs -- 2.30.2 From c996cb98b6889910f6850042cb0d66f5d98b9b3c Mon Sep 17 00:00:00 2001 From: Castro0o Date: Tue, 18 Feb 2020 22:54:00 +0100 Subject: [PATCH 09/14] smw_prop_val_ template rendered --- functions.py | 16 ++++++++ upload_imgs_dir.py | 97 +++++++++++++++++++++++++++------------------- 2 files changed, 73 insertions(+), 40 deletions(-) diff --git a/functions.py b/functions.py index f938d24..fcf23dd 100644 --- a/functions.py +++ b/functions.py @@ -115,6 +115,8 @@ class Colors: BOLD = '\033[1m' UNDERLINE = '\033[4m' +# image upload function + def listimgs(dir): lsimgs = [_file for _file in os.listdir(dir) if (os.path.splitext(_file)[-1]).lower() in @@ -152,3 +154,17 @@ def reorder_imgs(dir, dry): os.replace(src_img, dst_img) return listimgs(dir) # update list w/ renamed imgs +smw_propval_template = ''' +{{ImageMetadata +|Title={title} +|Date={date} +|Part={part} +|Partof={partof} +|Creator={creator} +|Organization={organization} +|Format={format} +|Event={event} +|Topic={topic} +}} +[[Template:ImageMetadata]] +''' diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index 062e896..e767508 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -1,7 +1,8 @@ import os, argparse, sys, re from mwclient import Site from functions import (print_colormsg, - reorder_imgs) + reorder_imgs, + smw_propval_template) p = argparse.ArgumentParser(description='Upload files from a directory, with metadata values to the wiki.\n' 'Note that any VALUES CONTAINING ' @@ -10,21 +11,41 @@ p = argparse.ArgumentParser(description='Upload files from a directory, with met # TODO: Add example of command to description p.add_argument('--host', default='hub.xpub.nl/sandbox', help='wiki host') p.add_argument('--path', default='/itchwiki/', help='Wiki path. Should end with /') -p.add_argument('--scheme', default='https', help='http or https') p.add_argument('--dry', '-d', action='store_true', - help='dry-run: will only print the metadata of each file that will be upload, but does NOT upload') - -p.add_argument('--title', required=True, help='Metadata **Title** value of publication. Must not exist yet in the wiki.') -p.add_argument('--date', required=True, help='Metadata **Date** value of publication. Format yyyy/mm/dd ' - 'For dates without day or date use 01 ie. 1986 --date "1986/01/01" March 1985: --date "1984/05/01"') -p.add_argument('--creator', required=True, action='append', help='Metadata **Creator** value(s) of publication.' - 'Multiple values should be SEPARATED BY COMMA') -p.add_argument('--dir', required=True, help='Full path of the image directory, ' - 'that you wish to upload') + help='dry-run: will only print the metadata of each file that ' + 'will be upload, but does NOT upload') +p.add_argument('--dir', required=True, + help='Full path of the image directory, that you wish to upload') + +p.add_argument('--title', required=True, + help='Metadata **Title** value of publication. Must not exist yet' + ' in the wiki.') +p.add_argument('--date', required=True, + help='Metadata **Date** value of publication. Format yyyy/mm/dd ' + 'For dates without day or mont use 01 as default ' + 'ie. 1986: --date "1986/01/01" ' + 'March 1985: --date "1984/05/01"') +p.add_argument('--creator', required=False, action='append', default=[''], + help='Metadata **Creator** value(s) of publication. Multiple ' + 'values should be SEPARATED BY COMMA') +p.add_argument('--organization', required=False, action='append', default=[''], + help='Metadata **Organization** value(s) of publication. ' + 'Multiple values should be SEPARATED BY COMMA') +p.add_argument('--format', required=False, action='append', default=[''], + help='Metadata **Format** value(s) of publication. ' + 'Multiple values should be SEPARATED BY COMMA') +p.add_argument('--event', required=False, action='append', default=[''], + help='Metadata **Event** value(s) of publication. ' + 'Multiple values should be SEPARATED BY COMMA') +p.add_argument('--topic', required=False, action='append', default=[''], + help='Metadata **Topic** value(s) of publication. ' + 'Multiple values should be SEPARATED BY COMMA') + +# TODO ADD NEW PROPS args = p.parse_args() # login -site = Site(host=args.host, path=args.path, scheme=args.scheme) +site = Site(host=args.host, path=args.path) wd =os.path.dirname(os.path.abspath(__file__)) # parent working directory with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd @@ -46,45 +67,41 @@ elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: lsimgs = reorder_imgs(dir=args.dir, dry=args.dry) - -print('lsimgs:', lsimgs) - -sys.exit() ## STOP HERE - -# TODO PartOf from os.listdir(args.dir) that fit on to condition -# TODO Part base on os.lisdir order! - -# todo creator value comma-seperated -smw_prop_val = '''{{ImageMetadata -|Title={title} -|Date={date} -|Part={part} -|Partof={partof} -|Creator={creator} -}} -[[Template:ImageMetadata]] -''' - -if args.dry == True: - print(args) - sys.exit() +dirname = os.path.split(args.dir)[-1].replace(' ', '_') +dirname = re.sub(r'[\W]', '', dirname) #remove non letters or digits +# print('lsimgs:', lsimgs, '\n', dirname) for n, _file in enumerate(lsimgs): - print_colormsg(_file, level='ok') + pagename = f'{dirname}-{_file}' + print_colormsg(pagename, level='ok') page = site.pages[_file] + if page.exists: url = page.imageinfo['descriptionurl'] print_colormsg( - f'Already exists in {url} Will be uploaded as new version', + f'Already exists in {url} Will NOT be uploaded', level='warning') + else: + img_smw_prop_val = smw_propval_template.format( + title=args.title, + date=args.date, + part=n + 1, + partof=len(lsimgs), + creator=(',').join(args.creator[1:]), + organization=(',').join(args.organization[1:]), + format=(',').join(args.format[1:]), + event=(',').join(args.event[1:]), + topic=(',').join(args.topic[1:]) + ) + + print(img_smw_prop_val) +if args.dry == True: + print(args) + sys.exit() - -img_smw_prop_val = smw_prop_val.format(title='cat from hell', date='2020/02/13', part='1', partof='1', creator='test') -print(img_smw_prop_val) - filename = 'cat2.jpg' page = site.images[filename] img_url = f'https://{args.host}{args.path}/index.php/File:{filename}' -- 2.30.2 From 29f569346e8f15cc5547b1aaf11aa260b8dc841b Mon Sep 17 00:00:00 2001 From: Castro0o Date: Tue, 18 Feb 2020 23:26:53 +0100 Subject: [PATCH 10/14] upload working. But template formating nees % notation --- functions.py | 4 ++-- upload_imgs_dir.py | 40 +++++++++++++++------------------------- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/functions.py b/functions.py index fcf23dd..59f0dd5 100644 --- a/functions.py +++ b/functions.py @@ -155,7 +155,7 @@ def reorder_imgs(dir, dry): return listimgs(dir) # update list w/ renamed imgs smw_propval_template = ''' -{{ImageMetadata +\{\{ImageMetadata |Title={title} |Date={date} |Part={part} @@ -165,6 +165,6 @@ smw_propval_template = ''' |Format={format} |Event={event} |Topic={topic} -}} +\}\} [[Template:ImageMetadata]] ''' diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index e767508..2f0da3c 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -64,8 +64,6 @@ elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: print_colormsg(f'Error: --title "{args.title}" already exists in wiki. Provide a different one', level='fail') sys.exit() - - lsimgs = reorder_imgs(dir=args.dir, dry=args.dry) dirname = os.path.split(args.dir)[-1].replace(' ', '_') dirname = re.sub(r'[\W]', '', dirname) #remove non letters or digits @@ -87,31 +85,23 @@ for n, _file in enumerate(lsimgs): date=args.date, part=n + 1, partof=len(lsimgs), - creator=(',').join(args.creator[1:]), - organization=(',').join(args.organization[1:]), - format=(',').join(args.format[1:]), - event=(',').join(args.event[1:]), - topic=(',').join(args.topic[1:]) + creator=(', ').join(args.creator[1:]), + organization=(', ').join(args.organization[1:]), + format=(', ').join(args.format[1:]), + event=(', ').join(args.event[1:]), + topic=(', ').join(args.topic[1:]) ) - print(img_smw_prop_val) - -if args.dry == True: - print(args) - sys.exit() - + _file_path = os.path.join(args.dir, _file) + print(_file_path) + if not args.dry: + with open(_file_path, 'rb') as _f: + site.upload(file=_file_path, + filename=pagename, + description=img_smw_prop_val, + ignore=True) + print(img_smw_prop_val) + print(f'https://{args.host}{args.path}index.php/File:{pagename}') -filename = 'cat2.jpg' -page = site.images[filename] -img_url = f'https://{args.host}{args.path}/index.php/File:{filename}' - -if page.exists is True: - print(f'{filename} is already upload it, you can see it at {img_url}' ) - print('The upload process wont proceed. Please upload all images in folder by hand') -else: - with open(filename, 'rb') as _file: - dirname = (os.path.split(dir)[-1]) - site.upload(file=_file, filename=f'{dirname}_{filename}', - description='img_smw_prop_val', ignore=True) -- 2.30.2 From c4d546b6b1f86a2e3c0c18435ec8b6097abcf1a7 Mon Sep 17 00:00:00 2001 From: Castro0o Date: Tue, 18 Feb 2020 23:27:28 +0100 Subject: [PATCH 11/14] rm sandbox/wiki_images.py --- sandbox/wiki_images.py | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 sandbox/wiki_images.py diff --git a/sandbox/wiki_images.py b/sandbox/wiki_images.py deleted file mode 100644 index 630784b..0000000 --- a/sandbox/wiki_images.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -from mwclient import Site -from pprint import pprint - -site = Site(host='hub.xpub.nl/sandbox', path='/itchwiki/') -wd = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # parent working directory - -with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd - loginlines = login.read() - user, pwd = loginlines.split('\n') - site.login(username=user, password=pwd) # login to wiki - -print(site) - -for n, img in enumerate(site.allimages()): - if n < 5: - print(img) - print('Image attributes:') - pprint(img.__dict__) # info contained in each img object - print('Image object methods:', dir(img)) - # important img info to dictionary - print(img.name, img.page_title, img.imageinfo['timestamp'], img.imageinfo['url'], - img.imageinfo['descriptionshorturl']) - print('\n') -- 2.30.2 From 1e109546b65ca789e3e9d3ee800d626eb7a8d8fa Mon Sep 17 00:00:00 2001 From: Castro0o Date: Wed, 19 Feb 2020 08:59:01 +0100 Subject: [PATCH 12/14] uploading image with smw info box --- functions.py | 15 --------------- templates/smw_infobox_template.jinja | 12 ++++++++++++ upload_imgs_dir.py | 10 +++++++--- 3 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 templates/smw_infobox_template.jinja diff --git a/functions.py b/functions.py index 59f0dd5..35707d5 100644 --- a/functions.py +++ b/functions.py @@ -153,18 +153,3 @@ def reorder_imgs(dir, dry): if dry == False: os.replace(src_img, dst_img) return listimgs(dir) # update list w/ renamed imgs - -smw_propval_template = ''' -\{\{ImageMetadata -|Title={title} -|Date={date} -|Part={part} -|Partof={partof} -|Creator={creator} -|Organization={organization} -|Format={format} -|Event={event} -|Topic={topic} -\}\} -[[Template:ImageMetadata]] -''' diff --git a/templates/smw_infobox_template.jinja b/templates/smw_infobox_template.jinja new file mode 100644 index 0000000..ab8d340 --- /dev/null +++ b/templates/smw_infobox_template.jinja @@ -0,0 +1,12 @@ +{{ '{{' }}ImageMetadata +|Title={{ title }} +|Date={{ date }} +|Part={{ part }} +|Partof={{ partof }} +|Creator={{ creator }} +|Organization={{ organization }} +|Format={{ format }} +|Event={{ event }} +|Topic={{ topic }} +{{ '}}' }} +[[Template:ImageMetadata]] \ No newline at end of file diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index 2f0da3c..6c4d443 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -1,8 +1,8 @@ import os, argparse, sys, re from mwclient import Site +from jinja2 import Template from functions import (print_colormsg, - reorder_imgs, - smw_propval_template) + reorder_imgs) p = argparse.ArgumentParser(description='Upload files from a directory, with metadata values to the wiki.\n' 'Note that any VALUES CONTAINING ' @@ -64,6 +64,10 @@ elif len(list(site.ask(f'[[Title::{args.title}]]'))) > 0: print_colormsg(f'Error: --title "{args.title}" already exists in wiki. Provide a different one', level='fail') sys.exit() +# read template file +with open(os.path.join(wd, 'templates/smw_infobox_template.jinja')) as tmplt: + smw_propval_template = Template(tmplt.read()) + lsimgs = reorder_imgs(dir=args.dir, dry=args.dry) dirname = os.path.split(args.dir)[-1].replace(' ', '_') dirname = re.sub(r'[\W]', '', dirname) #remove non letters or digits @@ -80,7 +84,7 @@ for n, _file in enumerate(lsimgs): f'Already exists in {url} Will NOT be uploaded', level='warning') else: - img_smw_prop_val = smw_propval_template.format( + img_smw_prop_val = smw_propval_template.render( title=args.title, date=args.date, part=n + 1, -- 2.30.2 From 9652ec2a2ff9a68af265f0f906534f4dc3f7e364 Mon Sep 17 00:00:00 2001 From: Castro0o Date: Wed, 19 Feb 2020 09:19:30 +0100 Subject: [PATCH 13/14] print functions --- functions.py | 9 ++++----- upload_imgs_dir.py | 23 ++++++++++++++--------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/functions.py b/functions.py index 35707d5..4c6b438 100644 --- a/functions.py +++ b/functions.py @@ -96,13 +96,12 @@ def clean_dir(dirfullpath): def print_colormsg(msg, level): if level == 'fail': - print(Colors.FAIL) + color_cmd = Colors.FAIL elif level == 'warning': - print(Colors.WARNING) + color_cmd = Colors.WARNING elif level == 'ok': - print(Colors.BLUE) - print(msg) - print(Colors.ENDC) + color_cmd = Colors.BLUE + print(color_cmd, msg, Colors.ENDC) class Colors: diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index 6c4d443..1eb6b4e 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -1,5 +1,6 @@ import os, argparse, sys, re -from mwclient import Site +from mwclient import (Site, + errors) from jinja2 import Template from functions import (print_colormsg, reorder_imgs) @@ -97,15 +98,19 @@ for n, _file in enumerate(lsimgs): ) _file_path = os.path.join(args.dir, _file) - print(_file_path) if not args.dry: + pageurl = f'https://{args.host}{args.path}index.php/File:{pagename}' with open(_file_path, 'rb') as _f: - site.upload(file=_file_path, - filename=pagename, - description=img_smw_prop_val, - ignore=True) - - print(img_smw_prop_val) - print(f'https://{args.host}{args.path}index.php/File:{pagename}') + try: + site.upload(file=_file_path, + filename=pagename, + description=img_smw_prop_val, + ignore=True) + print(img_smw_prop_val) + except errors.APIError as e: + print_colormsg(f'Error: {e.info}\n' + f'It will not be uploaded', + level='fail') + print(f'See image at {pageurl}') -- 2.30.2 From 3f1645dbc488027d07da5f437d7bd98c48215f6f Mon Sep 17 00:00:00 2001 From: Castro0o Date: Wed, 19 Feb 2020 10:02:47 +0100 Subject: [PATCH 14/14] documentation & helper script --- README.md | 6 ++++++ helper-upload_imgs_dir.sh | 21 +++++++++++++++++++++ upload_imgs_dir.py | 31 ++++++++++++++----------------- 3 files changed, 41 insertions(+), 17 deletions(-) create mode 100755 helper-upload_imgs_dir.sh diff --git a/README.md b/README.md index f07bb5f..56fe182 100644 --- a/README.md +++ b/README.md @@ -78,3 +78,9 @@ Each of the saved documents: * resulting in `static_html/index.html` +# Bulk image upload upload_imgs_dir.py + +Get Help: `python upload_imgs_dir.py --help` + +**Edit and run via** `.helper-upload_imgs_dir.sh` + diff --git a/helper-upload_imgs_dir.sh b/helper-upload_imgs_dir.sh new file mode 100755 index 0000000..138539f --- /dev/null +++ b/helper-upload_imgs_dir.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +python3 upload_imgs_dir.py \ +--title 'Ang Bayan December 1984' \ +--creator 'Central Committee of the Communist Party of the Philippines' \ +--date '1984/12/01' \ +--org 'Communist Party of the Philippines' \ +--format 'Bulletin' \ +--topic 'Communism, Armed Struggle' \ +--dir '/full/path/to/2020_bantayog/Folder name' \ +# --dry + +# Note: +# * Add this values to you upload specific upload. +# * --dry can be enabled to show you what will be uploaded and the metadata, without actully uploading it +# * parameters --event --topic can be added +# * \ allow you to continue the command of a different line +# +# Get help: python3 upload_imgs_dir.py --help + + diff --git a/upload_imgs_dir.py b/upload_imgs_dir.py index 1eb6b4e..3c47d8d 100644 --- a/upload_imgs_dir.py +++ b/upload_imgs_dir.py @@ -16,31 +16,26 @@ p.add_argument('--dry', '-d', action='store_true', help='dry-run: will only print the metadata of each file that ' 'will be upload, but does NOT upload') p.add_argument('--dir', required=True, - help='Full path of the image directory, that you wish to upload') + help='Required. Full path of the image directory, that you wish to upload') p.add_argument('--title', required=True, - help='Metadata **Title** value of publication. Must not exist yet' - ' in the wiki.') + help='Required. Must not exist yet in the wiki.') p.add_argument('--date', required=True, - help='Metadata **Date** value of publication. Format yyyy/mm/dd ' - 'For dates without day or mont use 01 as default ' + help='Required. Format: yyyy/mm/dd ' + 'For dates without day or month use 01 as default ' 'ie. 1986: --date "1986/01/01" ' 'March 1985: --date "1984/05/01"') p.add_argument('--creator', required=False, action='append', default=[''], - help='Metadata **Creator** value(s) of publication. Multiple ' - 'values should be SEPARATED BY COMMA') -p.add_argument('--organization', required=False, action='append', default=[''], - help='Metadata **Organization** value(s) of publication. ' - 'Multiple values should be SEPARATED BY COMMA') + help='Multiple values should be SEPARATED BY COMMA') +p.add_argument('--org', required=False, action='append', default=[''], + help='Organization:Multiple values should be SEPARATED BY ' + 'COMMA') p.add_argument('--format', required=False, action='append', default=[''], - help='Metadata **Format** value(s) of publication. ' - 'Multiple values should be SEPARATED BY COMMA') + help='Multiple values should be SEPARATED BY COMMA') p.add_argument('--event', required=False, action='append', default=[''], - help='Metadata **Event** value(s) of publication. ' - 'Multiple values should be SEPARATED BY COMMA') + help='Multiple values should be SEPARATED BY COMMA') p.add_argument('--topic', required=False, action='append', default=[''], - help='Metadata **Topic** value(s) of publication. ' - 'Multiple values should be SEPARATED BY COMMA') + help='Multiple values should be SEPARATED BY COMMA') # TODO ADD NEW PROPS args = p.parse_args() @@ -91,7 +86,7 @@ for n, _file in enumerate(lsimgs): part=n + 1, partof=len(lsimgs), creator=(', ').join(args.creator[1:]), - organization=(', ').join(args.organization[1:]), + organization=(', ').join(args.org[1:]), format=(', ').join(args.format[1:]), event=(', ').join(args.event[1:]), topic=(', ').join(args.topic[1:]) @@ -113,4 +108,6 @@ for n, _file in enumerate(lsimgs): level='fail') print(f'See image at {pageurl}') + else: + print(img_smw_prop_val) -- 2.30.2