download_imgs.py script completed

download_imgs.py

@@ -1,14 +1,20 @@
-import os
+import os, json
 
 from mwclient import Site
 from pprint import pprint
 
+from functions import update_json
+
 site = Site(host='hub.xpub.nl/sandbox', path='/itchwiki/')
 wd = os.path.abspath('.') # working directory
 
 imgdir = os.path.join(wd, 'images')
 os.makedirs(imgdir, exist_ok=True) # create images/ dir
 
+imgsjson_fn = os.path.join(wd, 'images.json')
+
+
+
 with open(os.path.join(wd, 'login.txt'), 'r') as login:  # read login user & pwd
     loginlines = login.read()
     user, pwd = loginlines.split('\n')
@@ -31,6 +37,15 @@ for n , img in enumerate(site.allimages()):
             'urldesc': img.imageinfo['descriptionshorturl'],
         }
         pprint(img_dict)
+
+        # location of image storage
         img_fn = os.path.join(imgdir, img_dict['filename'])
-        with open(img_fn, 'wb') as img_file:
-            img.download(destination=img_file)
+        # function updates images.json and returns whether the img needs to be downloaded or not
+        download = update_json(imgsjson_fn, img_dict, img_fn)
+
+        # image download
+        if download is True:
+            print('DOWNLOADING:', img_fn)
+            with open(img_fn, 'wb') as img_file:
+                img.download(destination=img_file)
+        print('\n')

functions.py

@@ -0,0 +1,33 @@
+import os, json
+
+
+def update_json(imgsjson_fn, img_dict, img_fn):
+    # write img_dict  to json file
+    if os.path.isfile(imgsjson_fn) is True:  # if json exists
+        with open(imgsjson_fn, 'r') as imgsjson_file:  # read its content
+            imgsjson_dict = json.load(imgsjson_file)
+            # print(imgsjson_dict)
+    else:  # other wise
+        imgsjson_dict = {}  # imgsjson_dict will be an empty dictionary
+
+    # is file already in dict
+    if img_dict['name'] in imgsjson_dict.keys():
+        # check if
+        # file is locally stored
+        img_issaved = os.path.isfile(img_fn)
+        # timestamp in json is same as in img object
+        img_samets = imgsjson_dict[img_dict['name']]['timestamp'] == img_dict['timestamp']
+        if all([img_issaved, img_samets]) is False:  # if one or more is False
+            # ask it to download again
+            download = True
+            imgsjson_dict[img_dict['name']] = img_dict  # add img_dict to imgsjson_dict under the key of the img.name
+        else:
+            download = False
+    else:
+        download = True
+        imgsjson_dict[img_dict['name']] = img_dict  # add img_dict to imgsjson_dict under the key of the img.name
+
+    with open(imgsjson_fn, 'w') as imgsjson_file:
+        json.dump(imgsjson_dict, imgsjson_file, indent=4)
+
+    return download