diff --git a/scrapper.py b/scrapper.py index 853dfdb..49a76fb 100644 --- a/scrapper.py +++ b/scrapper.py @@ -13,7 +13,7 @@ import time import re -ACTIVE_ENV = 'dev' +ACTIVE_ENV = 'prod' config = { @@ -22,14 +22,16 @@ config = { "marketplace_path": "/home/ec2-user/keyword_ranking_crawler/marketplaces.json", "keyword_path": "/home/ec2-user/keyword_ranking_crawler/keywords.json", "cookies_path": "/home/ec2-user/keyword_ranking_crawler/cookies.json", - "data_path": "/mnt/AmazonReports/Amazon/keyword_ranking" + "data_path": "/mnt/AmazonReports/Amazon/keyword_ranking", + "screenshots_path": "/home/ec2-user/keyword_ranking_crawler/screenshots" }, "dev": { "base_path" : "C:/Users/saif.haq/Desktop/Scrapper", "marketplace_path": "marketplaces.json", "keyword_path": "keywords.json", "cookies_path": "cookies.json", - "data_path": "data" + "data_path": "data", + "screenshots_path": "screenshots" } } @@ -170,7 +172,8 @@ def get_amazon_ranks(url, marketplace, ratingPrefix, keyword, page, count): count += 1 except: continue - + screenshot_path = config[ACTIVE_ENV]['screenshots_path'] + f"/{marketplace}-{keyword}-{page}--{int(time.time() * 1000)}.png" + driver.save_screenshot(screenshot_path) file_path = f"{BASE_PATH}/{int(time.time() * 1000)}-{marketplace}-{keyword}.json" save_ranking(ranks, file_path ) if( len(ranks) == 0 ):