saif.haq
/
keyword_ranking_crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add delete logic

main
saif 2025-08-12 14:03:41 +05:00
parent 11e013b075
commit 03adb32bf0
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapper.py
View File

@ -1,5 +1,7 @@
import os
import pickle
import glob
import shutil
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
@ -11,6 +13,14 @@ import json
import time
import re
chrome_profile_path = "/home/ec2-user/keyword_ranking_crawler/chrome_path_copy"
# Remove Chrome's lock files
for lock_file in glob.glob(os.path.join(chrome_profile_path, "Singleton*")):
try:
os.remove(lock_file)
except:
pass
with open("marketplaces.json", "r", encoding="utf-8") as f:
data = json.load(f)
@ -38,7 +48,7 @@ def get_driver():
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36")
options.add_argument("--start-maximized")
options.add_argument("user-data-dir=/home/ec2-user/keyword_ranking_crawler/chrome_path_copy")
options.add_argument(f"user-data-dir={chrome_profile_path}")
options.add_argument("profile-directory=Default")
driver = webdriver.Chrome( service=Service(ChromeDriverManager().install()),options=options)
# Remove navigator.webdriver