From c56d66661999f8397e59799a8b27b0095e647391 Mon Sep 17 00:00:00 2001 From: saif Date: Tue, 12 Aug 2025 13:12:14 +0500 Subject: [PATCH] add profile --- keywords.json | 1004 +------------------------------------------------ scrapper.py | 44 ++- 2 files changed, 35 insertions(+), 1013 deletions(-) diff --git a/keywords.json b/keywords.json index d7a0aab..472eace 100644 --- a/keywords.json +++ b/keywords.json @@ -1,1002 +1,6 @@ [ - "20 x 20 outdoor pillow insert", - "bedding", - "bath towels", - "kitchen towels", - "kitchenb towels", - "mattress protector queen waterproof", - "(_targeting_auto_)", - "12 inch cast iron", - "12x20 pillow 6", - "16x16 pillow 6", - "18x18 pillow 6", - "22x44 towels", - "24x24 pillow insert", - "26x26 pillow inserts", - "2xl bathrobe", - "3 gallon grow", - "3 gallon pots", - "4ft mattress protector", - "5 gallon grow bags", - "5 gallon pots", - "6 pack pillows", - "76x190 mattress protector", - "adjustable pant hangers", - "adult bibs", - "air tight containers", - "air tight storage containers", - "airtight containers", - "airtight fridge containers", - "airtight storage containers", - "alese 90x200", - "alfombrilla baño blanca", - "apron", - "apron white", - "apronmen", - "aqua apron", - "army green cropped puffer vest women", - "army green shorts", - "army green shorts men", - "asciugamani bianchi", - "asciugamani blu", - "asciugamani da hotel bianchi", - "asciugamani doccia", - "asciugamani grigi", - "asciugamani grigio", - "asciugamani grigio freddo", - "asciugamani mani", - "asciugamani neri", - "asciugamani turchese", - "asciugamani turchesi", - "asciugamano viola", - "badetuch pflaume", - "badjas dames", - "bag dispenser", - "bag holder", - "bag organizer", - "bamboo cutting board", - "bamboo drawer organizer", - "bamboo mattress protector", - "bartschere", - "bath sheet set", - "bath sheet towels", - "bath sheets", - "bath towel", - "bath towel bulk", - "bath towel bundle", - "bath towel pack", - "bath towels 24 x 48", - "bath towels 27 x 54", - "bathtowels 35 x 70", - "bathrobe black", - "bathroom floor mats", - "beach towels", - "bed pads washable waterproof", - "bed sheet set", - "bed storage", - "bed sheets", - "bed sheets twin xl", - "bed skirt", - "bedbug pillow protectors", - "bedding comforter sets", - "beige bath towel", - "beige cropped puffer vest", - "beige dish towels", - "beige face cloths", - "beige face towels", - "beige flannels", - "beige king comforter", - "beige king duvet", - "beige kitchen towels", - "beige pillow cases", - "beige queen comforter", - "beige tea towels", - "beige towel", - "beige towels", - "beige turtleneck women", - "beige twin duvet", - "beige wash cloth", - "beige wash cloths", - "beige washcloths", - "bento box", - "bento lunch box", - "bettdecke 135x200", - "bettwäsche", - "black apron", - "black bar mops", - "black bath sheet", - "black bathrobe", - "black bathrobe men", - "black bathrobe women", - "black bed skirt queen", - "black cropped puffer vest", - "black cropped puffer vest women", - "black cropped vest", - "black door hinges", - "black face cloth", - "black face towels", - "black flannels", - "black golf polo men", - "black grey comforter king", - "black hangers", - "black king comforter", - "black lab coat", - "black lab coat women", - "black pillow cases set of 2", - "black polo", - "black puffer vest women", - "black robe", - "black robe men", - "black running shoes men", - "black scrub pants", - "black scrub tops for women", - "black server apron", - "black shorts", - "black towel set", - "black towels", - "black turtleneck women", - "black underscrub long sleeve women", - "black wash cloth", - "black wash cloths", - "black washcloths", - "black white shoes", - "black and white apron", - "blackfriday", - "blackgreytwincomforter set", - "blacklabcoat", - "blackscrubpants forwomen", - "blackshortsmen", - "blacktowels", - "blackturtleneck", - "blacktwincomforter", - "blackwhiterunningshoes", - "blackwhiterunningshoesmen", - "blackwhiteshoesmen", - "blanc gant de toilette", - "blanc gant toilette", - "blanche serviette pour le visage", - "blanket king size bed", - "blue apron", - "blue bath towels", - "blue beach towel", - "blue face cloth", - "blue fleece blanket", - "blue golf polo", - "blue golf polo men", - "blue hand towels", - "blue king comforter", - "blue kitchen towels", - "blue lab coat", - "blue mattress pad", - "blue mattress protector", - "blue mattress topper", - "blue queen comforter", - "blue robe", - "blue server apron", - "blue shorts men", - "blue tea towels", - "blue towels", - "blue turtleneck men", - "blue turtleneck women", - "blue turtleneck men", - "blue underscrub long sleeve", - "blue waist apron", - "blue wash cloths", - "blue washcloths", - "blueshorts", - "body pillow", - "boite conservation verre", - "bread box", - "breastfeeding nightgown", - "brown apron", - "brown bar mops", - "brown bath sheet", - "brown blanket king", - "brown cropped puffer vest", - "brown cropped puffer vest women", - "brown flannel shirt women", - "brown kitchen towels", - "brown puffer vest women", - "brown towel set", - "brown towels", - "brown turkish beach towel", - "brown turtle neck", - "brown turtleneck men", - "brown turtleneck women", - "brown wash cloth", - "brown washcloths", - "browntowels", - "brownturtleneckmen", - "bulk bed pillows", - "bunk bed mattress protector", - "burgundy apron", - "burgundy bath towel", - "burgundy bathrobe men", - "burgundy comforter twin", - "burgundy flannel shirt women", - "burgundy hand towels", - "burgundy king comforter", - "burgundy mattress cover", - "burgundy mattress cover twin", - "burgundy mattress pad", - "burgundy mattress pad twin", - "burgundy mattress protector", - "burgundy mattress protector twin", - "burgundy mattress topper", - "burgundy mattress topper twin", - "burgundy pillow cases", - "burgundy robe", - "burgundy robe men", - "burgundy robes for women", - "burgundy salon towels", - "burgundy server apron", - "burgundy shorts men", - "burgundy towel", - "burgundy towel set", - "burgundy towels", - "burgundy turtleneck", - "burgundy waist apron", - "burgundy wash cloths", - "burgundy wash cloths pack of 12", - "burgundy washcloths", - "burgundy washcloths pack of 12", - "burgundy towel set", - "burgundytowels", - "burgundyturtleneck", - "butterfly comforter", - "bügel", - "cabana towels", - "california king mattress cover", - "california king mattress protector", - "can organizer", - "can storage pantry", - "canorganizerforpantry", - "cast iron pan", - "cast iron skillet", - "cast iron skillets", - "castironskillets", - "ceil blue scrub top", - "ceil blue scrub top", - "cereal container", - "cereal containers", - "cheap men shoes", - "christmas planters", - "christmas hand towels for bathroom", - "cintres velours", - "ciruela toalla de baño", - "clear bins", - "clear organization", - "clear organizer", - "clear pantry", - "clear shoe box", - "clear shoe organizer", - "clear space organizer", - "clearance sale bathrobe", - "clearance sale robe", - "clearspace organizer", - "closet organization", - "closet organizer", - "closet organizers", - "closet organizers storage", - "coffee turtleneck", - "college dorm room essentials", - "cologne organizer", - "colored lab coats", - "comforter duvet cover set", - "comforter king grey", - "comforter queen grey", - "comforter set", - "comforter sets", - "comforter twin grey black", - "conservation verre", - "cool grey towels", - "cool grey turkish towel", - "copricuscini letto", - "coprimaterasso 160", - "coprimaterasso impermeabile 180x200", - "coprimaterasso matrimoniale", - "coprimaterasso singolo", - "coprimaterasso matrimoniale 180x200", - "coral bath towel", - "coral bathrobe women", - "coral robe", - "coral towel set", - "cot bed mattress protector", - "cot mattress protector", - "cot mattress protector 60", - "cotton bath mat", - "cotton bathrobe", - "cotton cloth 6", - "cotton kitchen towels", - "cotton robe women", - "cotton tea towels", - "cotton white kitchen towel", - "cotton white kitchen towels 6", - "couch cover", - "couette 135", - "cream king comforter", - "cream twin comforter", - "cream wash cloths", - "cream washcloths", - "crib mattress protector", - "crib mattress protector 2 pack", - "crib mattress protector 2 set", - "cropped puffer vest women", - "cropped vest women", - "cupcake stand", - "cutlery organizer", - "cybermonday", - "dark brown washcloths", - "dark green shorts", - "dark grey bathrobe", - "dark grey bathrobe women", - "dark grey robe", - "dark grey robe women", - "dark grey shorts", - "dark grey throw blanket", - "débarbouillette", - "debarbouillette bebe", - "débarbouillette pour bébé", - "débarbouillette rose", - "débarbouillettes bleu", - "delivery gown for hospital maternity", - "dish cloth green", - "dish towel 6 pack", - "dish towel green", - "dish towel set", - "dish towel sets for kitchen", - "dish towel 6 pack", - "dish towels", - "doctor coat", - "dog car seat cover for back seat", - "dog seat cover", - "dorm room", - "double bed mattress protector bamboo", - "drap housse 90x190 bleu", - "drap de bain gris clair", - "drawer organizer", - "dressinggown", - "dust mite mattress cover", - "dusty olive wash cloth", - "dusty olive washcloth", - "duvet cover set", - "edredon 140x200", - "edredon cama 80", - "egg cartons", - "egg container", - "egg holder", - "egg organiser", - "egg storage", - "egg tray", - "electric blue towels", - "electric blue washcloths", - "electric blue hand towel", - "essuie mains gris", - "essuie mains bleu marine", - "felt hangers", - "fingernail clipplers", - "fitted sheet king", - "fitted sheet queen", - "flat bed sheet", - "flat bed sheet bulk", - "flat bed sheet twin", - "flat sheet", - "flat sheet full white", - "flat sheets bulk king", - "fleece throw blanket", - "flour sack 6", - "flour sack beige", - "flour sack dish towels", - "flour sack kitchen towels", - "flour sack towels beige", - "flower pot", - "foldable shoe box", - "foldable shoe storage", - "food container", - "food containers", - "french press", - "french press stainless steel", - "fridge containers", - "fridge organisers", - "fridge organization", - "fridge organizer", - "fridge organizers", - "fridge organizing", - "frying pan", - "frying pan 24cm", - "frying pan non stick", - "frying pans nonstick", - "full flat sheet", - "full flat sheet only", - "full mattress protector", - "full mattress protector 2 pack", - "full mattress protector 2 set", - "full size mattress protector cooling", - "full top sheet", - "full mattress protector", - "full size mattress bag", - "full size bed sheets", - "full size mattress protector bamboo", - "fullxl mattress protector 2 pack", - "fullxl mattress protector 2 set", - "funda colchon 135x190", - "funda colchon 150x200", - "funko pop display", - "funko pop stand", - "gant de toilette", - "gant de toilette bebe", - "gant de toilette bleu", - "gant de toilette marine", - "gant toilette enfant", - "gifts boyfriend", - "grande toalla azul", - "gray bar towel", - "gray bathroom mat", - "gray flat sheet", - "gray hand towels", - "gray queen comforter", - "gray wash cloths", - "gray washcloths", - "gray flat sheet", - "graytwinsheets", - "green apron", - "green bathrobe", - "green bathrobe women", - "green beach towel", - "green cloth napkins", - "green comforter set twin", - "green comforter twin", - "green cropped puffer vest", - "green cropped puffer vest women", - "green dish towel", - "green flour sack", - "green king comforter", - "green kitchen towel", - "green puffer vest", - "green puffer vest women", - "green robe", - "green robe women", - "green robes for women", - "green scrub top", - "green shorts", - "green tea towels", - "green turtleneck", - "greenshortsmen", - "grey adult bib", - "grey apron", - "grey bar towel", - "grey bath mat", - "grey bath sheet", - "grey bath towels", - "grey bathrobe", - "grey beach towel", - "grey beach towels", - "grey blackout curtains", - "grey cloth napkins", - "grey comforter twin", - "grey dish towel", - "grey dish towels bulk", - "grey face cloth", - "grey flannel shirt women", - "grey flour sack towels bulk", - "grey hand towels", - "grey hangers", - "grey king comforter", - "grey kitchen towel", - "grey mattress cover", - "grey mattress pad", - "grey mattress protector", - "grey pillow cases set of 2", - "grey pillow cases set of 4", - "grey queen comforter", - "grey robe", - "grey robes for women", - "grey running shoes men", - "grey scrub top", - "grey shoes men", - "grey shorts", - "grey shorts men", - "grey table cloth", - "grey tea towel bulk", - "grey towel", - "grey towel set", - "grey towels", - "grey towels sets bathroom", - "grey towels bathroom", - "grey turtle neck", - "grey turtleneck men", - "grey turtleneck women", - "grey twin comforter", - "grey underscrub long sleeve women", - "grey velvet hangers", - "grey wash cloths", - "grey washcloths", - "greybeachtowel", - "greyrobewomen", - "greyshorts", - "greyturtleneckmen", - "gris débarbouillette", - "gris essuie mains", - "gris gant de toilette", - "gris serviette", - "gris serviette invité", - "gris toallas 30x30", - "gris serviette", - "grow 5 gallon", - "guest towels", - "gusseisenpfanne", - "hair scissors", - "hair shears", - "halloween patient gown", - "hand towel 16 x 28", - "hand towels 16 x 28", - "hand towels blue", - "handtowelsforbathroom", - "hanger 20 pack", - "hanger 30", - "hangers 100 pack", - "hangers 20 pack", - "hangers 200 pack", - "hangers in bulk", - "hangers utopia", - "heavy-duty hangers for pants", - "hojo organizer", - "home decor", - "hoojo organizer", - "hospital gown", - "hotel robe", - "housse matelas 90 200", - "housse matelas 90x200", - "humane mouse trap", - "hunter green scrub top", - "hunter green wash cloths", - "hunter green wash cloths pack of 12", - "hunter green washcloths", - "hunter green washcloths pack of 12", - "idesign organizer", - "impermeable 140", - "incontinence bed pads", - "indoor plant pots", - "ivory apron", - "ivory towel set", - "ivory wash cloths", - "ivory washcloths", - "kennel cover", - "khaki cropped puffer vest women", - "khaki puffer vest", - "khaki puffer vest women", - "khaki turtleneck", - "khakiturtleneckmen", - "kichly cafetiere", - "kichly french press", - "kids hangers", - "king beige comforter", - "king beige duvet", - "king blue comforter", - "king burgundy comforter", - "king burgundy mattress protector", - "king burgundy mattress topper", - "king comforter set", - "king cream comforter", - "king fitted sheet", - "king flat sheets", - "king flat sheets only 6 pack", - "king grey comforter", - "king grey comforter set", - "king grey mattress cover", - "king grey mattress pad", - "king grey mattress topper", - "king mattress bag", - "king mattress cover", - "king mattress protector 2 pack", - "king mattress protector 2 set", - "king navy comforter", - "king pillow cases set of 2", - "king pillow protector", - "king pink comforter", - "king plum comforter", - "king purple comforter", - "king red comforter", - "king red mattress protector", - "king red mattress topper", - "king size duvet", - "king size mattress protector", - "king size waterproof mattress protector", - "king waterproof mattress protector", - "king mattress protector", - "king size sheets set", - "king size sheets set", - "kitchen organization", - "kitchen organizer", - "kitchen towel 6 pack", - "kitchen towels", - "kitchen towels green", - "kleiderbügel grau", - "kleiderbügel kunststoff", - "kleiderbügel weiß", - "knit tunic blouse", - "lab coat men", - "lab coat women", - "labcoat", - "large turtleneck", - "largeshortsmen", - "largeturtleneckmen", - "lavender bath sheet", - "lavender bath towel", - "lavender towel", - "lavender wash cloths", - "lavender washcloths", - "lavender washcloths pack of 12", - "light gray king comforter", - "light grey comforter", - "light grey flannel shirt women", - "light grey king comforter", - "light grey king duvet", - "light grey towels", - "light grey turkish beach towel", - "light grey queen comforter", - "lightgrey comforter", - "lightgrey king comforter", - "lightweightrobe", - "lilacwashcloths", - "lime green vest", - "limegreenshorts", - "lingette bambou bebe", - "long sleeve tunic", - "lunch box", - "luxury bath towels", - "luxury hand towels", - "manicure kit", - "manicure set", - "manicure set for women", - "maroon shorts men", - "maroonturtleneck", - "matratzen bezug 140x200", - "matratzen bezug 90x200", - "matratzenbezug 140", - "matratzenbezug 140x200", - "matratzenbezug 160x200", - "matratzenbezug 90x200", - "matratzenschoner 140 200 wasserdicht", - "matratzenschoner 140x200", - "matratzenschoner 140x200cm", - "matratzenschoner 160x200", - "matratzenschoner 60 120", - "matratzenschoner 60x120", - "matratzenschoner 70 140 baby", - "matratzenschoner 90 200 wasserdicht", - "matratzenschoner 90 x 200", - "matratzenschoner 90 x 200 wasserdicht", - "matratzenschoner 90x200 wasserdicht", - "matratzenschoner baby", - "matratzenschoner baby 60", - "matratzenschoner baby wasserdicht", - "matratzenschoner70x140 baby", - "mattress 2", - "mattress 2 pack", - "mattress bag for moving", - "mattress cover", - "mattress cover 2", - "mattress cover 2 pack", - "mattress cover double", - "mattress cover full", - "mattress cover full beige", - "mattress cover king", - "mattress cover king burgundy", - "mattress cover king red", - "mattress cover king size", - "mattress cover king size bed", - "mattress cover king size bed", - "mattress cover queen size bed", - "mattress cover single", - "mattress cover twin", - "mattress cover twin black", - "mattress encasement", - "mattress encasement single", - "mattress encasement twin", - "mattress encasement twinxl", - "mattress encasements bulk", - "mattress pad", - "mattress pad 2 pack", - "mattress pad full beige", - "mattress pad king black", - "mattress pad king blue", - "mattress pad king burgundy", - "mattress pad king navy", - "mattress pad king red", - "mattress pad navy", - "mattress pad queen blue", - "mattress pad queen navy", - "mattress pad twin black", - "mattress pad twin gray", - "mattress pad twin grey", - "mattress protector", - "mattress protector bulk", - "mattress protector double", - "mattress protector double bed", - "mattress protector gray", - "mattress protector grey", - "mattress protector king", - "mattress protector king navy", - "mattress protector navy", - "mattress protector single", - "mattress protector super king", - "mattress protector super queen", - "mattress protector twin", - "mattress topper blue queen", - "mattress topper full beige", - "mattress topper king black", - "mattress topper king burgundy", - "mattress topper king navy", - "mattress topper king red", - "mattress topper navy queen", - "mattress topper twin black", - "mattress topper twin grey", - "mattress pad 2 pack", - "mattressprotectorqueen", - "medium turtleneck", - "mediumshortsmen", - "mediumturtleneckmen", - "men grooming", - "menrobe", - "menshoes11", - "menworkshirt", - "mens bathrobe navy", - "mens grooming kit", - "mens medium bathrobe", - "mens medium robe", - "mens navy robe", - "mens robe black", - "mens robe dark grey", - "mens robe grey", - "mens running shoes", - "mens shoes", - "mens shoes 13", - "mens shoes 9", - "mens shorts", - "mens small bathrobe", - "mens small robe", - "mensrunningshoes12", - "mensshoes10", - "mensshoes12", - "mensshoes13", - "mensshoes9", - "metal cafetiere", - "mint green vest", - "mop towel", - "moptowel", - "multi-layer pant hangers", - "mustard turtleneck", - "möbelerhöhung", - "nail clipper", - "nail clippers", - "nail kit", - "natural flour sack bulk", - "natural tea towels bulk", - "navy apron", - "navy bath robe", - "navy bath sheet", - "navy bathrobe", - "navy beach towels", - "navy blue bath mat", - "navy blue golf polo men", - "navy blue scrub pants for women", - "navy comforter set", - "navy face cloth", - "navy golf polo", - "navy hand towels", - "navy king comforter", - "navy lab coat", - "navy mattress pad", - "navy mattress protector", - "navy mattress topper", - "navy pillow cases", - "navy robe", - "navy running shoes men", - "navy scrub top", - "navy scrub tops", - "navy scrub tops for women", - "navy shoes men", - "navy shorts men", - "navy towel", - "navy towels", - "navy turkish towel", - "navy turkish beach towel", - "navy turtle neck", - "navy turtleneck", - "navy turtleneck men", - "navy underscrub long sleeve women", - "navy wash cloths", - "navy washcloths", - "navy blue scrub tops", - "navy blue scrub tops for women", - "navy blue turtleneck men", - "navy blue turtleneck for women", - "navyblueturtleneck", - "navyshorts", - "navyturtleneck", - "neri asciugamani", - "noir chiffserviettes", - "non-slip pant hangers", - "old money clothes men", - "olive cropped puffer vest", - "olive cropped puffer vest women", - "olive shorts", - "olive shorts men", - "open front bins", - "orange apron", - "orange bathrobe", - "orange flour sack towels", - "orange kitchen towel", - "orange kitchen towels", - "orange robe", - "orange tuch serviette", - "orange underscrub long sleeve women", - "orangeturtleneckmen", - "organization", - "organizer", - "organizing", - "pant hanger", - "pantry bins", - "pantry organization", - "pantry organizer", - "pantry organizers", - "pantry organizers storage", - "paquete de toallas", - "patient gown", - "perfume holder", - "perfume stand", - "petite serviette bleu marine", - "pillow case", - "pillow cases queen size set of 2", - "pillow cases standard size", - "pillow protector", - "pillow protector toddler", - "pillow protectors standard", - "pillow protectors waterproof", - "pillow sets", - "pillows bulk", - "pillows bulk queen", - "pillows set of 6", - "pine treecomfortertwin", - "pink adult bib", - "pink apron", - "pink bath towel", - "pink bathrobe", - "pink hangers", - "pink hooded robe", - "pink organizer", - "pink robe", - "pink robe hood", - "pink robes for women", - "pink server apron", - "pink towel set", - "pink towels", - "pink turkish towel", - "pink turtleneck women", - "pink underscrub long sleeve women", - "pink waist apron", - "pink wash cloths", - "pink washcloths", - "pink bath towel", - "pinkunderscrublongsleeve", - "plant pots", - "planter pot", - "plaque remover", - "plastic containers", - "plastic hangers", - "plastic pantry organizer", - "pleated bed skirt", - "plum bath sheet", - "plum bath sheets", - "plum bath towel", - "plum bathrobe", - "plum beach towels", - "plum king comforter", - "plum tappetini da bagno", - "plum towels", - "plum wash cloths", - "plum washcloths", - "plus size pj set", - "pool towels", - "postpartum gown", - "postpartum robe", - "prime day", - "prime deals", - "printed blue comforter", - "printed navy comforter", - "protector colchon 160x200", - "protector colchon 200", - "protector colchon 90", - "protector colchon 90x190", - "protector colchón 90x200", - "protector colchon 160x200", - "protector colchon 90x200", - "protege matelas 80x200", - "protege matelas 90x200", - "protege matelas 90x190", - "protege oreiller 60x60", - "protege oreiller 65x65", - "prune tapis de bain", - "purple apron", - "purple aprons bulk", - "purple bath sheets", - "purple comforter queen", - "purple face cloth", - "purple king comforter", - "purple king comforter set", - "purple pillow case", - "purple queen comforter set", - "purple robe", - "purple robes for women", - "purple towels", - "purple washcloths", - "purplebeachtowel", - "purplerobe", - "queen bed skirt", - "queen bed skirt size", - "queen beige comforter", - "queen burgundy comforter", - "queen comforter set", - "queen flat sheet", - "queen grey bed skirt", - "queen grey comforter", - "queen mattress bag", - "queen mattress pad 2 pack", - "queen mattress protector", - "queen mattress protector 2 pack", - "queen mattress protector 2 set", - "queen mattress protector waterproof", - "queen pillow protector gray", - "queen purple comforter", - "queen red comforter", - "queen sheet set", - "queen sheets", - "queen size bed skirt", - "queen sleeper sofa mattress cover", - "queen top sheet", - "queen bed sheets", - "queen bed sheets grey", - "queen mattress protector", - "queen mattress protector waterproof", - "queen sheet set", - "queen sheets", - "queen size fuzzy blanket", - "queensheetset", - "raised garden bed", - "rasied garden bed fabric", - "red apron", - "red bath sheet", - "red bath towels", - "red beach towel", - "red beach towels", - "red cloth napkins", - "red comforter twin", - "red flannel shirt women", - "red flat sheet only", - "red king comforter", - "red mattress cover", - "red mattress cover twin", - "red mattress pad", - "red mattress pad twin", - "red mattress protector", - "red mattress protector twin", - "red mattress topper", - "red mattress topper twin", - "red robe", - "red running shoes men", - "red server apron", - "red shoes men", - "red shorts", - "red shorts men", - "red towel set", - "red towel set", - "red towels", - "red turtleneck women", - "red underscrub long sleeve women", - "red wash cloths", - "red wash cloths pack of 12", - "red washcloths", - "red washcloths pack of 12", - "red pillow covers 18 x 18" + "bedsheet", + "pillows", + "fitted sheets", + "comforters" ] \ No newline at end of file diff --git a/scrapper.py b/scrapper.py index 82c62d1..9693363 100644 --- a/scrapper.py +++ b/scrapper.py @@ -3,6 +3,9 @@ import pickle from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By +from selenium.webdriver.chrome.service import Service +from webdriver_manager.chrome import ChromeDriverManager + from time import sleep import json import time @@ -21,17 +24,31 @@ with open("cookies.json", "r", encoding="utf-8") as f: # Or if it's a Python dict already: marketplaces = data["marketplaces"] -BASE_PATH= '/mnt/AmazonReports/Amazon/keyword_ranking' -#BASE_PATH= 'data' +#BASE_PATH= '/mnt/AmazonReports/Amazon/keyword_ranking' +BASE_PATH= 'data' MAX_PAGE = 10 def get_driver(): options = Options() - options.add_argument("--headless") - options.add_argument("--disable-blink-features=AutomationControlled") - driver = webdriver.Chrome(options=options) + #options.add_argument("--headless") + options.add_argument("--disable-blink-features=AutomationControlled") # Removes automation flag + options.add_experimental_option("excludeSwitches", ["enable-automation"]) + options.add_experimental_option('useAutomationExtension', False) + options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36") + options.add_argument("--start-maximized") + options.add_argument("user-data-dir=/home/ec2-user/keyword_ranking_crawler/chrome_path") + options.add_argument("profile-directory=Default") + driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options) + # Remove navigator.webdriver + driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", { + "source": """ + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined + }) + """ +}) return driver def save_cookies(driver, path): @@ -156,14 +173,15 @@ driver = get_driver() for keyword in keywords: for marketplace, details in marketplaces.items(): - url = details['url'] - ratingPrefix = details['ratingPrefix'] - count =1 - for i in range(1, MAX_PAGE): - count = get_amazon_ranks(url, marketplace, ratingPrefix, keyword, i, count) - if count == -1: - break - sleep(3) + if marketplace == 'AMAZON_USA': + url = details['url'] + ratingPrefix = details['ratingPrefix'] + count =1 + for i in range(1, MAX_PAGE): + count = get_amazon_ranks(url, marketplace, ratingPrefix, keyword, i, count) + if count == -1: + break + sleep(3) driver.quit()