harvesting_the_net/_pics.ipynb

In [7]:

import selenium 
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import requests
import os
import io
import json
import random
import json
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

DRIVER_PATH = '../../geckodriver'
DRIVER_PATH2 = '../chromedriver'

In [8]:

def harvesting(query:str, max_links_to_fetch:int, wd:webdriver, sleep_between_interactions:int=1):
    dictionary = open('axios-example/pics.json',)
    l = json.load(dictionary)
    
    cycles = 3

    def scroll_to_end(wd):
        wd.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(sleep_between_interactions)    
    
    # build the google query
    search_url = "https://www.google.com/search?safe=off&site=&tbm=isch&source=hp&q={q}&oq={q}&gs_l=img"

    # load the page
    wd.get(search_url.format(q=query))

    image_urls = set()
    image_count = 0
    results_start = 0
    
    sx ="sx"
    dx="dx"

    
    while image_count < max_links_to_fetch:
        for loop in range(cycles):
            scroll_to_end(wd)
            time.sleep(.1)

        # get all image thumbnail results
        thumbnail_results = wd.find_elements(By.CLASS_NAME,"Q4LuWd")
        number_results = len(thumbnail_results)
        print(f"Found: {number_results} search results. Extracting links from {results_start}:{number_results}")
            
        # some useful variables
        nPic = 0
        holdPic=0
        groundPic = 0
        currentMeta = ''

        
        for img in thumbnail_results[results_start:number_results]:
            # click the thumbnail to get the actual image
            try:
                img.click()
                time.sleep(.1)
            except Exception:
                continue

            # extract image url
            actual_images = wd.find_elements(By.CLASS_NAME,'n3VNCb')
            actual_image = actual_images[-1]
            if actual_image.get_attribute('src') and 'http' in actual_image.get_attribute('src'):
                image_urls.add(actual_image.get_attribute('src'))
                linkPic = actual_image.get_attribute('src')
                #
               # print(linkPic)
                
                l[nPic] = linkPic
                time.sleep(.5)

               # print(l)  

                
                with open("axios-example/pics.json", "w") as outfile: 
                    json.dump(l, outfile) 
                
                nPic = nPic+1

                holdPic = holdPic+1
                time.sleep(.5)

            image_count = len(image_urls)

            if len(image_urls) >= max_links_to_fetch:
                print(f"Found: {len(image_urls)} image links, done!")
                break
        else:
            print("Found:", len(image_urls), "image links, looking for more ...")
            time.sleep(1)
            return
            load_more_button = wd.find_element(By.CLASS_NAME,"Mye4qd")
            if load_more_button:
                wd.execute_script("document.querySelector('.mye4qd').click();")

        # move the result startpoint further down
        results_start = len(thumbnail_results)

    return image_urls

In [9]:

if __name__ == '__main__':
    wd = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
    query = "VAAST COLSON"
    wd.get('https://google.com')
    time.sleep(1)
    wd.find_element(By.ID,'W0wltc').click()

    time.sleep(.5)
    
    search_box = wd.find_element(By.CLASS_NAME,'gLFyf')
    search_box.send_keys(query)
    
    links = harvesting(query,1000,wd)
    
    wd.quit()

Found: 267 search results. Extracting links from 0:267
Found: 200 image links, done!

In [90]:

 #if (nPic%2):
                #    wd2.execute_script(f"document.querySelector('.scene3D').innerHTML += '<div><img class={dx} style={pos} id=pic{nPic} src={linkPic}></div>'")
               # else:
                 #   wd2.execute_script(f"document.querySelector('.scene3D').innerHTML += '<div><img class={sx} style={pos} id=pic{nPic} src={linkPic}></div>'")
                #if (holdPic == 50):
                #    for rPic in range(groundPic,groundPic+10):
                #        try:
                #            wd2.execute_script(f"remove('pic{rPic}')")
                #        except:
                #            print('eheh')
                #    groundPic = groundPic+10
                #    holdPic = 0

In [68]:

import json 
dictionary = open('/meta.json',)
l = json.load(dictionary)
for x in range(10):
    print(l[str(x)])

View Vaast Colson's artworks on artnet. Learn about the artist and find an in-depth biography, exhibitions, original artworks, the latest news, ...
Biography Vaast Colson (Kapellen, Belgium, 1977) Lives and works in Antwerp, Belgium. Monopolis — Antwerp. 10 September — 6 November 2005. Exhibition.
Vaast Colson (°1977, Belgium) Vaast Colson's artworks are essentially ludic models of social spaces. His projects place the spectators into a certain social ...
Stay up to date with Vaast Colson (Belgian, 1977), APT artist. Discover works for sale, auction results, market data, news and exhibitions on MutualArt.
Browse the latest artworks, exhibitions, shows by Vaast Colson. On Artland you can find art for sale, browse 3D gallery exhibitions and much more.
Vaast Colson MAES & MATTHYS GALLERY Antwerp. 29. Nov 22. Feb 2009. Un-Scene WIELS Contemporary Art Centre, Brussels. 19. Oct 04. Jan 2009.
Vaast Colson and Kati Heck “Show me yours & I'll show you mine” at Museum of Contemporary Art Antwerp. 15.05.2016. Reading Time 2′. Share.
MOSTRE ED EVENTI CHE RIGUARDANO Vaast Colson. TUTTIIN CORSOCONCLUSIFUTURI. Sanguine. Luc Tuymans on Baroque. Dal 17 ottobre 2018 al 25 febbraio 2019
Vaast Colson (Kapellen, 1977) is een Belgisch kunstschilder. Hij studeerde schilderkunst aan de Koninklijke Academie voor Schone Kunsten van Antwerpen.
Vaast Colson about his drawing activities, on the occasion of The Big Draw in Antwerp. www.thebigdraw.be Sine Van Menxel's solo exhibition, Fencing a...

In [ ]: