saif.haq
/
keyword_ranking_crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

screenshots added

main
saif 2025-08-18 15:17:05 +05:00
parent 20a2b766c1
commit a282bfa3e2
1 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
scrapper.py
View File

@ -13,7 +13,7 @@ import time
import re import re
ACTIVE_ENV = 'dev' ACTIVE_ENV = 'prod'
config = { config = {
@ -22,14 +22,16 @@ config = {
"marketplace_path": "/home/ec2-user/keyword_ranking_crawler/marketplaces.json", "marketplace_path": "/home/ec2-user/keyword_ranking_crawler/marketplaces.json",
"keyword_path": "/home/ec2-user/keyword_ranking_crawler/keywords.json", "keyword_path": "/home/ec2-user/keyword_ranking_crawler/keywords.json",
"cookies_path": "/home/ec2-user/keyword_ranking_crawler/cookies.json", "cookies_path": "/home/ec2-user/keyword_ranking_crawler/cookies.json",
"data_path": "/mnt/AmazonReports/Amazon/keyword_ranking" "data_path": "/mnt/AmazonReports/Amazon/keyword_ranking",
"screenshots_path": "/home/ec2-user/keyword_ranking_crawler/screenshots"
}, },
"dev": { "dev": {
"base_path" : "C:/Users/saif.haq/Desktop/Scrapper", "base_path" : "C:/Users/saif.haq/Desktop/Scrapper",
"marketplace_path": "marketplaces.json", "marketplace_path": "marketplaces.json",
"keyword_path": "keywords.json", "keyword_path": "keywords.json",
"cookies_path": "cookies.json", "cookies_path": "cookies.json",
"data_path": "data" "data_path": "data",
"screenshots_path": "screenshots"
} }
} }
@ -170,7 +172,8 @@ def get_amazon_ranks(url, marketplace, ratingPrefix, keyword, page, count):
count += 1 count += 1
except: except:
continue continue
screenshot_path = config[ACTIVE_ENV]['screenshots_path'] + f"/{marketplace}-{keyword}-{page}--{int(time.time() * 1000)}.png"
driver.save_screenshot(screenshot_path)
file_path = f"{BASE_PATH}/{int(time.time() * 1000)}-{marketplace}-{keyword}.json" file_path = f"{BASE_PATH}/{int(time.time() * 1000)}-{marketplace}-{keyword}.json"
save_ranking(ranks, file_path ) save_ranking(ranks, file_path )
if( len(ranks) == 0 ): if( len(ranks) == 0 ):