saif.haq
/
keyword_ranking_crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add delete logic

main
saif 2025-08-12 14:03:41 +05:00
parent 11e013b075
commit 03adb32bf0
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapper.py
View File

@ -1,5 +1,7 @@
import os import os
import pickle import pickle
import glob
import shutil
from selenium import webdriver from selenium import webdriver
from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
@ -11,6 +13,14 @@ import json
import time import time
import re import re
chrome_profile_path = "/home/ec2-user/keyword_ranking_crawler/chrome_path_copy"
# Remove Chrome's lock files
for lock_file in glob.glob(os.path.join(chrome_profile_path, "Singleton*")):
try:
os.remove(lock_file)
except:
pass
with open("marketplaces.json", "r", encoding="utf-8") as f: with open("marketplaces.json", "r", encoding="utf-8") as f:
data = json.load(f) data = json.load(f)
@ -38,7 +48,7 @@ def get_driver():
options.add_experimental_option('useAutomationExtension', False) options.add_experimental_option('useAutomationExtension', False)
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36") options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36")
options.add_argument("--start-maximized") options.add_argument("--start-maximized")
options.add_argument("user-data-dir=/home/ec2-user/keyword_ranking_crawler/chrome_path_copy") options.add_argument(f"user-data-dir={chrome_profile_path}")
options.add_argument("profile-directory=Default") options.add_argument("profile-directory=Default")
driver = webdriver.Chrome( service=Service(ChromeDriverManager().install()),options=options) driver = webdriver.Chrome( service=Service(ChromeDriverManager().install()),options=options)
# Remove navigator.webdriver # Remove navigator.webdriver