saif.haq
/
temu-labels-crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

increase page count

main
saif 2025-01-29 14:27:17 +05:00
parent a697aec5bb
commit d1b07c5eae
2 changed files with 33 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
download-shipping-labels.js
View File

@ -24,13 +24,13 @@ const utils = require("./utils");
const state = JSON.parse(fs.readFileSync(__dirname + "/state.json")); const state = JSON.parse(fs.readFileSync(__dirname + "/state.json"));
const email = utils.decryptString( const email = utils.decryptString(
process.env["temu-email"], process.env["temu-personal-email"],
cryptoConfig.algo, cryptoConfig.algo,
cryptoConfig.key, cryptoConfig.key,
cryptoConfig.iv cryptoConfig.iv
); );
const password = utils.decryptString( const password = utils.decryptString(
process.env["temu-password"], process.env["temu-personal-password"],
cryptoConfig.algo, cryptoConfig.algo,
cryptoConfig.key, cryptoConfig.key,
cryptoConfig.iv cryptoConfig.iv
@ -56,7 +56,7 @@ const utils = require("./utils");
config[environment]["chrome_profile_path"] config[environment]["chrome_profile_path"]
); );
const browser = await puppeteer.launch( const browser = await puppeteer.launch(
utils.getBrowserConfig(chromeProfilePath, environment) utils.getBrowserConfig(chromeProfilePath, environment),
); );
const page = await browser.newPage(); const page = await browser.newPage();
await loadPageCookies(page); await loadPageCookies(page);
@ -116,10 +116,26 @@ const utils = require("./utils");
/* /*
* data * data
*/ */
const pagination = 10; const pagination = 100;
let total_items = 0; let total_items = 0;
let currentPage = 1; let currentPage = 1;
let maxPage = 200; let maxPage = 10;
await new Promise((resolve) => setTimeout(resolve, 2_000));
// set the pagination to 100 / page
const pageinationSeletor =
"#orders-tab-list > div.y0DVv7GO > div > div._38NAUUfN > div._15QWqbZs > ul > li.PGT_sizeChanger_123 > div > div > div > div > div";
await page.waitForSelector(pageinationSeletor);
await page.click(pageinationSeletor);
console.log('Clicking on pagination Select')
await new Promise((resolve) => setTimeout(resolve, 3_000));
// select 100 /page option
const pagesPerPageSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_mediumDropdown_123.PT_dropdown_123.PT_portalTopLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > ul > li:nth-child(5)";
await page.waitForSelector(pagesPerPageSelector);
await page.click(pagesPerPageSelector);
await new Promise((resolve) => setTimeout(resolve, 1_000));
/** /**
* Capture response * Capture response
@ -265,7 +281,8 @@ const utils = require("./utils");
const closeBtnSelector = "div._2hT9O4-z.Y9rEd1VR"; const closeBtnSelector = "div._2hT9O4-z.Y9rEd1VR";
await page.waitForSelector(closeBtnSelector); await page.waitForSelector(closeBtnSelector);
await page.click(closeBtnSelector); await page.click(closeBtnSelector);
// wait to download
await new Promise((resolve) => setTimeout(resolve, 2_000));
console.log(bIndex); console.log(bIndex);
const orderNumber = await page.evaluate( const orderNumber = await page.evaluate(
(el) => el.textContent.trim(), (el) => el.textContent.trim(),
@ -310,7 +327,7 @@ const utils = require("./utils");
}); });
if (liText === null) { if (liText === null) {
total_items = 10; total_items = 100;
} else { } else {
total_items = parseInt(liText.split(" ")[1]); total_items = parseInt(liText.split(" ")[1]);
console.log(`Total Items count : ${total_items}`); console.log(`Total Items count : ${total_items}`);
@ -322,7 +339,9 @@ const utils = require("./utils");
while (true) { while (true) {
try { try {
// crawl next pages // crawl next pages
const currentStatePage = JSON.parse(fs.readFileSync(__dirname + "/state.json")).last_page; const currentStatePage = JSON.parse(
fs.readFileSync(__dirname + "/state.json")
).last_page;
if (currentStatePage > currentPage) { if (currentStatePage > currentPage) {
console.log(`Moving from ${currentPage} to ${currentStatePage}`); console.log(`Moving from ${currentPage} to ${currentStatePage}`);
const hasNextBtn = await page.evaluate(() => { const hasNextBtn = await page.evaluate(() => {
@ -364,6 +383,8 @@ const utils = require("./utils");
console.log(`Crawling for page ${currentPage}`); console.log(`Crawling for page ${currentPage}`);
await new Promise((resolve) => setTimeout(resolve, 2_000));
await utils.tryTemuLogin(page, email, password, loginPage); await utils.tryTemuLogin(page, email, password, loginPage);
await new Promise((resolve) => setTimeout(resolve, 4_000)); await new Promise((resolve) => setTimeout(resolve, 4_000));

4
utils.js
View File

@ -47,6 +47,7 @@ exports.decryptString = decryptString;
const getBrowserConfig = function (profilePath, env) { const getBrowserConfig = function (profilePath, env) {
let browserConfig = {}; let browserConfig = {};
browserConfig["ignoreDefaultArgs"] = ["--disable-extensions"]; browserConfig["ignoreDefaultArgs"] = ["--disable-extensions"];
browserConfig["timeout"] = 60000;
browserConfig["headless"] = true; browserConfig["headless"] = true;
// prod specific // prod specific
if (env === "prod") { if (env === "prod") {
@ -63,8 +64,7 @@ const getBrowserConfig = function (profilePath, env) {
// dev specific // dev specific
if (env === "dev") { if (env === "dev") {
browserConfig["headless"] = false; browserConfig["headless"] = false;
browserConfig["args"] = ['--incognito'] browserConfig["args"] = [`--user-data-dir=${profilePath}`];
// browserConfig["args"] = [`--user-data-dir=${profilePath}`];
} }
return browserConfig; return browserConfig;
}; };