download_imgs.py script completed

andre
Castro0o 5 years ago
parent 363254fa33
commit 183dfcf283

@ -1,14 +1,20 @@
import os import os, json
from mwclient import Site from mwclient import Site
from pprint import pprint from pprint import pprint
from functions import update_json
site = Site(host='hub.xpub.nl/sandbox', path='/itchwiki/') site = Site(host='hub.xpub.nl/sandbox', path='/itchwiki/')
wd = os.path.abspath('.') # working directory wd = os.path.abspath('.') # working directory
imgdir = os.path.join(wd, 'images') imgdir = os.path.join(wd, 'images')
os.makedirs(imgdir, exist_ok=True) # create images/ dir os.makedirs(imgdir, exist_ok=True) # create images/ dir
imgsjson_fn = os.path.join(wd, 'images.json')
with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd with open(os.path.join(wd, 'login.txt'), 'r') as login: # read login user & pwd
loginlines = login.read() loginlines = login.read()
user, pwd = loginlines.split('\n') user, pwd = loginlines.split('\n')
@ -31,6 +37,15 @@ for n , img in enumerate(site.allimages()):
'urldesc': img.imageinfo['descriptionshorturl'], 'urldesc': img.imageinfo['descriptionshorturl'],
} }
pprint(img_dict) pprint(img_dict)
# location of image storage
img_fn = os.path.join(imgdir, img_dict['filename']) img_fn = os.path.join(imgdir, img_dict['filename'])
# function updates images.json and returns whether the img needs to be downloaded or not
download = update_json(imgsjson_fn, img_dict, img_fn)
# image download
if download is True:
print('DOWNLOADING:', img_fn)
with open(img_fn, 'wb') as img_file: with open(img_fn, 'wb') as img_file:
img.download(destination=img_file) img.download(destination=img_file)
print('\n')

@ -0,0 +1,33 @@
import os, json
def update_json(imgsjson_fn, img_dict, img_fn):
# write img_dict to json file
if os.path.isfile(imgsjson_fn) is True: # if json exists
with open(imgsjson_fn, 'r') as imgsjson_file: # read its content
imgsjson_dict = json.load(imgsjson_file)
# print(imgsjson_dict)
else: # other wise
imgsjson_dict = {} # imgsjson_dict will be an empty dictionary
# is file already in dict
if img_dict['name'] in imgsjson_dict.keys():
# check if
# file is locally stored
img_issaved = os.path.isfile(img_fn)
# timestamp in json is same as in img object
img_samets = imgsjson_dict[img_dict['name']]['timestamp'] == img_dict['timestamp']
if all([img_issaved, img_samets]) is False: # if one or more is False
# ask it to download again
download = True
imgsjson_dict[img_dict['name']] = img_dict # add img_dict to imgsjson_dict under the key of the img.name
else:
download = False
else:
download = True
imgsjson_dict[img_dict['name']] = img_dict # add img_dict to imgsjson_dict under the key of the img.name
with open(imgsjson_fn, 'w') as imgsjson_file:
json.dump(imgsjson_dict, imgsjson_file, indent=4)
return download
Loading…
Cancel
Save