saif.haq
/
temu-labels-crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

increase page count

main
saif 2025-01-29 14:27:17 +05:00
parent a697aec5bb
commit d1b07c5eae
2 changed files with 33 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
download-shipping-labels.js
View File

@ -24,13 +24,13 @@ const utils = require("./utils");
const state = JSON.parse(fs.readFileSync(__dirname + "/state.json"));
const email = utils.decryptString(
process.env["temu-email"],
process.env["temu-personal-email"],
cryptoConfig.algo,
cryptoConfig.key,
cryptoConfig.iv
);
const password = utils.decryptString(
process.env["temu-password"],
process.env["temu-personal-password"],
cryptoConfig.algo,
cryptoConfig.key,
cryptoConfig.iv
@ -56,7 +56,7 @@ const utils = require("./utils");
config[environment]["chrome_profile_path"]
);
const browser = await puppeteer.launch(
utils.getBrowserConfig(chromeProfilePath, environment)
utils.getBrowserConfig(chromeProfilePath, environment),
);
const page = await browser.newPage();
await loadPageCookies(page);
@ -116,10 +116,26 @@ const utils = require("./utils");
/*
* data
*/
const pagination = 10;
const pagination = 100;
let total_items = 0;
let currentPage = 1;
let maxPage = 200;
let maxPage = 10;
await new Promise((resolve) => setTimeout(resolve, 2_000));
// set the pagination to 100 / page
const pageinationSeletor =
"#orders-tab-list > div.y0DVv7GO > div > div._38NAUUfN > div._15QWqbZs > ul > li.PGT_sizeChanger_123 > div > div > div > div > div";
await page.waitForSelector(pageinationSeletor);
await page.click(pageinationSeletor);
console.log('Clicking on pagination Select')
await new Promise((resolve) => setTimeout(resolve, 3_000));
// select 100 /page option
const pagesPerPageSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_mediumDropdown_123.PT_dropdown_123.PT_portalTopLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > ul > li:nth-child(5)";
await page.waitForSelector(pagesPerPageSelector);
await page.click(pagesPerPageSelector);
await new Promise((resolve) => setTimeout(resolve, 1_000));
/**
* Capture response
@ -265,7 +281,8 @@ const utils = require("./utils");
const closeBtnSelector = "div._2hT9O4-z.Y9rEd1VR";
await page.waitForSelector(closeBtnSelector);
await page.click(closeBtnSelector);
// wait to download
await new Promise((resolve) => setTimeout(resolve, 2_000));
console.log(bIndex);
const orderNumber = await page.evaluate(
(el) => el.textContent.trim(),
@ -310,7 +327,7 @@ const utils = require("./utils");
});
if (liText === null) {
total_items = 10;
total_items = 100;
} else {
total_items = parseInt(liText.split(" ")[1]);
console.log(`Total Items count : ${total_items}`);
@ -322,7 +339,9 @@ const utils = require("./utils");
while (true) {
try {
// crawl next pages
const currentStatePage = JSON.parse(fs.readFileSync(__dirname + "/state.json")).last_page;
const currentStatePage = JSON.parse(
fs.readFileSync(__dirname + "/state.json")
).last_page;
if (currentStatePage > currentPage) {
console.log(`Moving from ${currentPage} to ${currentStatePage}`);
const hasNextBtn = await page.evaluate(() => {
@ -345,7 +364,7 @@ const utils = require("./utils");
// goto next page
if (hasNextBtn) {
await page.evaluate( async() => {
await page.evaluate(async () => {
const liElement = document.querySelector("li.PGT_next_123");
await new Promise((r) => setTimeout(r, 3000));
if (liElement) {
@ -353,7 +372,7 @@ const utils = require("./utils");
}
});
}
currentPage++;
// wait
continue;
@ -364,6 +383,8 @@ const utils = require("./utils");
console.log(`Crawling for page ${currentPage}`);
await new Promise((resolve) => setTimeout(resolve, 2_000));
await utils.tryTemuLogin(page, email, password, loginPage);
await new Promise((resolve) => setTimeout(resolve, 4_000));

4
utils.js
View File

@ -47,6 +47,7 @@ exports.decryptString = decryptString;
const getBrowserConfig = function (profilePath, env) {
let browserConfig = {};
browserConfig["ignoreDefaultArgs"] = ["--disable-extensions"];
browserConfig["timeout"] = 60000;
browserConfig["headless"] = true;
// prod specific
if (env === "prod") {
@ -63,8 +64,7 @@ const getBrowserConfig = function (profilePath, env) {
// dev specific
if (env === "dev") {
browserConfig["headless"] = false;
browserConfig["args"] = ['--incognito']
// browserConfig["args"] = [`--user-data-dir=${profilePath}`];
browserConfig["args"] = [`--user-data-dir=${profilePath}`];
}
return browserConfig;
};