saif.haq
/
keyword_ranking_crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

screenshots added

main
saif 2025-08-18 15:17:05 +05:00
parent 20a2b766c1
commit a282bfa3e2
1 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
scrapper.py
View File

@ -13,7 +13,7 @@ import time
import re
ACTIVE_ENV = 'dev'
ACTIVE_ENV = 'prod'
config = {
@ -22,14 +22,16 @@ config = {
"marketplace_path": "/home/ec2-user/keyword_ranking_crawler/marketplaces.json",
"keyword_path": "/home/ec2-user/keyword_ranking_crawler/keywords.json",
"cookies_path": "/home/ec2-user/keyword_ranking_crawler/cookies.json",
"data_path": "/mnt/AmazonReports/Amazon/keyword_ranking"
"data_path": "/mnt/AmazonReports/Amazon/keyword_ranking",
"screenshots_path": "/home/ec2-user/keyword_ranking_crawler/screenshots"
},
"dev": {
"base_path" : "C:/Users/saif.haq/Desktop/Scrapper",
"marketplace_path": "marketplaces.json",
"keyword_path": "keywords.json",
"cookies_path": "cookies.json",
"data_path": "data"
"data_path": "data",
"screenshots_path": "screenshots"
}
}
@ -170,7 +172,8 @@ def get_amazon_ranks(url, marketplace, ratingPrefix, keyword, page, count):
count += 1
except:
continue
screenshot_path = config[ACTIVE_ENV]['screenshots_path'] + f"/{marketplace}-{keyword}-{page}--{int(time.time() * 1000)}.png"
driver.save_screenshot(screenshot_path)
file_path = f"{BASE_PATH}/{int(time.time() * 1000)}-{marketplace}-{keyword}.json"
save_ranking(ranks, file_path )
if( len(ranks) == 0 ):