saif.haq
/
keyword_ranking_crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reduce sleep

main
saif 2025-08-18 15:07:47 +05:00
parent 9d40fae97c
commit 20a2b766c1
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapper.py
View File

@ -13,7 +13,7 @@ import time
import re import re
ACTIVE_ENV = 'prod' ACTIVE_ENV = 'dev'
config = { config = {
@ -138,10 +138,10 @@ def get_amazon_ranks(url, marketplace, ratingPrefix, keyword, page, count):
else: else:
print("No cookie file found, visiting fresh") print("No cookie file found, visiting fresh")
driver.get(url) driver.get(url)
sleep(2) # Give time to solve CAPTCHA manually (if needed) sleep(1) # Give time to solve CAPTCHA manually (if needed)
save_cookies(driver, COOKIE_FILE) save_cookies(driver, COOKIE_FILE)
sleep(3) # Wait for JS to load sleep(2) # Wait for JS to load
items = driver.find_elements(By.XPATH, '//div[contains(@class,"s-result-item") and @data-asin]') items = driver.find_elements(By.XPATH, '//div[contains(@class,"s-result-item") and @data-asin]')
for idx, item in enumerate(items, start=1): for idx, item in enumerate(items, start=1):
try: try:
@ -190,7 +190,7 @@ try :
count = get_amazon_ranks(url, marketplace, ratingPrefix, keyword, i, count) count = get_amazon_ranks(url, marketplace, ratingPrefix, keyword, i, count)
if count == -1: if count == -1:
break break
sleep(2) sleep(1)
finally: finally:
print("[INFO] Closing WebDriver...") print("[INFO] Closing WebDriver...")
driver.quit() driver.quit()