harvesting_the_net/text/text.py

import selenium 
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import requests
import os
import io
import multiprocessing
import json
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

def meta(queries,wd3):
    url = 'https://www.google.it/search?q=vaast+colson&sxsrf=ALiCzsYcMyMvNo67pjpN-zRgfmTXw1L6Zw%3A1670252242664&source=hp&ei=0gaOY7zUJfbjxc8Pu_2QmAE&iflsig=AJiK0e8AAAAAY44U4oHBLhsJUz-SOp-oMy5eO2PMIDLU&gs_ssp=eJzj4tVP1zc0TDKvMDJMqjQzYPTiKUtMLC5RSM7PKc7PAwB8eAkY&oq=vaast+colsan&gs_lcp=Cgdnd3Mtd2l6EAEYADIHCC4QgAQQDTIGCAAQHhANMgYIABAeEA0yBggAEB4QDTIGCAAQHhANMggIABAIEB4QDToECCMQJzoLCC4QgAQQxwEQ0QM6CAguENQCEIAEOggILhCABBDUAjoFCAAQgAQ6BQguEIAEOgsILhCABBDHARCvAToOCC4QgAQQxwEQ0QMQ1AI6CAguEIAEEMsBOggIABCABBDLAToLCC4Q1AIQgAQQywE6DgguEK8BEMcBEIAEEMsBOgoIABCABBAKEMsBOgcIABCABBAKOgYIABAWEB46CAgAEBYQHhAKOgUIABCGA1AAWMkOYK0XaAFwAHgBgAHhAogB-RKSAQcxLjYuNS4xmAEAoAEB&sclient=gws-wiz'
    wd3.get(url)
    wd3.find_element(By.ID,'W0wltc').click()
    
    dictionary = open('./text.json',)
    l = json.load(dictionary)

    i=2
    n = 0
    
    while True:
        try:                      
            wd3.find_element(By.CSS_SELECTOR, "a[aria-label='Page {}']".format(i)).click()
            searchResults = wd3.find_elements(By.CLASS_NAME, 'VwiC3b')


            for result in searchResults:
                #l = ''
                l[n] = result.text
                time.sleep(.5)

                print(l)  

                n=n+1
                with open("./text.json", "w") as outfile: 
                    json.dump(l, outfile) 

                time.sleep(.5)

            print("Navigating to Next Page " + str(i))
            i=i+1
            time.sleep(1)
        except:
            print("Last page reached")
            break

wd = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
queries = ["VAAST COLSON"]
meta(queries,wd)