diff --git a/scrapper.py b/scrapper.py index 265aa50..853dfdb 100644 --- a/scrapper.py +++ b/scrapper.py @@ -13,7 +13,7 @@ import time import re -ACTIVE_ENV = 'prod' +ACTIVE_ENV = 'dev' config = { @@ -138,10 +138,10 @@ def get_amazon_ranks(url, marketplace, ratingPrefix, keyword, page, count): else: print("No cookie file found, visiting fresh") driver.get(url) - sleep(2) # Give time to solve CAPTCHA manually (if needed) + sleep(1) # Give time to solve CAPTCHA manually (if needed) save_cookies(driver, COOKIE_FILE) - sleep(3) # Wait for JS to load + sleep(2) # Wait for JS to load items = driver.find_elements(By.XPATH, '//div[contains(@class,"s-result-item") and @data-asin]') for idx, item in enumerate(items, start=1): try: @@ -190,7 +190,7 @@ try : count = get_amazon_ranks(url, marketplace, ratingPrefix, keyword, i, count) if count == -1: break - sleep(2) + sleep(1) finally: print("[INFO] Closing WebDriver...") driver.quit()