from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.firefox.options import Options def search(query): open('geckodriver.log', 'w').close() #Clear Log File options = Options() options.headless = True driver = webdriver.Firefox(options=options, executable_path = '/usr/local/bin/geckodriver') driver.get(f'https://duckduckgo.com/{query}&iar=news&ia=news') html = driver.page_source soup = BeautifulSoup(html, 'html.parser') results = [] if soup.find_all(class_="no-results t-m") == []: print(f'{len(soup.find_all(class_="result"))} results' ) for i in range(len(soup.find_all(class_="result"))): result = {} result["title"] = soup.find_all(class_="result__a")[i].get_text() try: result['domain'] = soup.find_all(class_="result__url")[i]['href'] result['description'] = soup.find_all(class_="result__snippet")[i].get_text() except IndexError: result['domain'] = '' result['description'] = '' results.append(result) return results else: return None