saif.haq
/
temu-labels-crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix shipping rates script

main
saif 2025-03-07 13:01:11 +05:00
parent 19db6c9232
commit 02b56767c0
2 changed files with 280 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

256
buy-shipping.js
View File

@ -7,10 +7,13 @@ const path = require("path");
const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
const utils = require("./utils");
const emailUtils = require("./email-utils");
(async function () {
console.log(
`===========< STARTED BUY SHIPPING ${utils.getPakistanStandardTime(luxon.DateTime.now())} >=========`
`===========< STARTED ${utils.getPakistanStandardTime(
luxon.DateTime.now()
)} >=========`
);
const syncDate = luxon.DateTime.now().toFormat("yyyy-MM-dd");
@ -23,13 +26,13 @@ const utils = require("./utils");
let rates = [];
const email = utils.decryptString(
process.env["temu-email"],
process.env["temu-personal-email"],
cryptoConfig.algo,
cryptoConfig.key,
cryptoConfig.iv
);
const password = utils.decryptString(
process.env["temu-password"],
process.env["temu-personal-password"],
cryptoConfig.algo,
cryptoConfig.key,
cryptoConfig.iv
@ -87,7 +90,7 @@ const utils = require("./utils");
*/
const loginPage = config[environment]["temuLoginPage"];
await page.goto(loginPage, {
waitUntil: ["domcontentloaded"],
waitUntil: ["networkidle2"],
});
await utils.tryTemuLogin(page, email, password, loginPage);
@ -97,43 +100,9 @@ const utils = require("./utils");
const UnshippedOrdersRequestPage =
config[environment]["temuUnshippedOrdersPage"];
await page.goto(UnshippedOrdersRequestPage, {
waitUntil: ["domcontentloaded"],
waitUntil: ["networkidle2"],
});
/**
* Capture response
*/
const checkShippingRates = async (page, timer) => {
return new Promise((resolve, reject) => {
// Timeout mechanism to resolve with an empty list after 5 seconds
const timeout = setTimeout(() => {
page.off("response", handleResponse); // Remove listener on timeout
resolve([]); // Resolve with an empty list
}, timer);
const handleResponse = async (res) => {
try {
const req = res.request();
if (req.url().includes("/query_shipping_provider_optional")) {
const resJson = await res.json();
// Remove listener and clear timeout once response is captured
clearTimeout(timeout);
page.off("response", handleResponse);
resolve(resJson.result.online_channel_vo_list || []);
}
} catch (ex) {
// Remove listener and clear timeout on error
clearTimeout(timeout);
page.off("response", handleResponse);
reject(ex);
}
};
page.on("response", handleResponse);
});
};
const getOrdersFromPage = async (page) => {
let orderNumbers = [];
try {
@ -169,7 +138,7 @@ const utils = require("./utils");
let total_items = 0;
let currentPage = 1;
await new Promise((resolve) => setTimeout(resolve, 5_000));
await new Promise((resolve) => setTimeout(resolve, 2_000));
// set the pagination to 100 / page
const pageinationSeletor =
"#orders-tab-list > div.y0DVv7GO > div > div._38NAUUfN > div._15QWqbZs > ul > li.PGT_sizeChanger_123 > div > div > div > div > div";
@ -211,7 +180,7 @@ const utils = require("./utils");
console.log(`Crawling for page ${currentPage}`);
await utils.tryTemuLogin(page, email, password, loginPage);
await new Promise((resolve) => setTimeout(resolve, 8_000));
await new Promise((resolve) => setTimeout(resolve, 4000));
// load cookies
await loadPageCookies(page);
@ -256,6 +225,7 @@ const utils = require("./utils");
await new Promise((r) => setTimeout(r, 5000));
} catch (e) {
console.log(e);
break;
}
}
} catch (e) {
@ -263,6 +233,169 @@ const utils = require("./utils");
}
}
/*
* Select shipping date
*/
const selectShippingDate1 = async (page) => {
try {
const modalSelector = ".MDL_innerWrapper_123"; // Class selector for the modal
await page.waitForSelector(modalSelector);
// Optionally, wait for the modal content to be loaded and scrollable
const modalContentSelector = ".MDL_body_123"; // Adjust if modal content has a specific class
await page.waitForSelector(modalContentSelector);
// Scroll to the bottom of the modal content
await page.evaluate((modalContentSelector) => {
const modalContent = document.querySelector(modalContentSelector);
if (modalContent) {
modalContent.scrollTo(0, modalContent.scrollHeight);
}
}, modalContentSelector);
const shippingDateSelector =
"#packageList\\[0\\]\\.trackingInfoList\\[0\\]\\.shipDate > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123";
const shippingDateInput = await page.$(shippingDateSelector);
if (!shippingDateInput) {
console.log("Select not found.");
return;
}
await page.click(shippingDateSelector);
await new Promise((resolve) => setTimeout(resolve, 2_000));
const shippingDateOptionSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_largeDropdown_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div:nth-child(1) > ul > li.cIL_item_123.cIL_large_123.cIL_highlight_123.ST_itemRendererLabel_123";
const shippingDateOption = await page.$(shippingDateOptionSelector);
if (!shippingDateOption) {
console.log("Option not found");
return;
}
await page.click(shippingDateOptionSelector);
console.log("Selected a shipping date");
} catch (error) {
console.error("Error selecting shipping date:", error);
}
};
/*
* Select shipping date
*/
const selectShippingDate = async (page) => {
try {
console.log("Selecting Dates");
// Selector for the shipping date dropdown
const shippingDateSelector =
'div[id="packageList[0].trackingInfoList[0].shipDate"] input:first-of-type';
const shippingDateInput = await page.$(shippingDateSelector);
if (shippingDateInput) {
console.log("Select found!");
await page.click(shippingDateSelector);
const shippingDateOptionSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_largeDropdown_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div:nth-child(1) > ul > li.cIL_item_123.cIL_large_123.cIL_highlight_123.ST_itemRendererLabel_123";
const shippingDateOptionSelectorInput = await page.$(
shippingDateOptionSelector
);
await page.waitForTimeout(5 * 1000);
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
if (shippingDateOptionSelectorInput) {
await shippingDateOptionSelectorInput.click();
console.log("Selected a shipping date");
} else {
console.log("Option not found");
}
} else {
console.log("Select not found.");
}
} catch (error) {
console.error("Error selecting shipping date:", error);
}
};
/**
* Capture response
*/
const checkShippingRates = async (page, timer) => {
return new Promise((resolve, reject) => {
// Timeout mechanism to resolve with an empty list after 5 seconds
const timeout = setTimeout(() => {
page.off("response", handleResponse); // Remove listener on timeout
resolve([]); // Resolve with an empty list
}, timer);
const handleResponse = async (res) => {
try {
const req = res.request();
if (req.url().includes("/query_shipping_provider_optional")) {
const resJson = await res.json();
// Remove listener and clear timeout once response is captured
clearTimeout(timeout);
page.off("response", handleResponse);
resolve(resJson.result.online_channel_vo_list || []);
}
} catch (ex) {
// Remove listener and clear timeout on error
clearTimeout(timeout);
page.off("response", handleResponse);
reject(ex);
}
};
page.on("response", handleResponse);
});
};
/*
* check for shipping rate in response
*/
/*
* map response to rates
*/
const mapResponseToRates = (shippingRates, order) => {
for (const shippingRate of shippingRates) {
try {
let rate = {};
rate["orderId"] = order;
rate["channel_id"] = shippingRate["channel_id"];
rate["ship_product_name"] = shippingRate["ship_product_name"];
rate["ship_company_id"] = shippingRate["ship_company_id"];
rate["shipping_company_name"] = shippingRate["shipping_company_name"];
rate["faraway_type"] = shippingRate["faraway_type"];
rate["service_code"] = shippingRate["service_code"];
rate["ship_logistics_type"] = shippingRate["ship_logistics_type"];
rate["require_reservation"] = shippingRate["require_reservation"];
//
rate["amount"] =
shippingRate["online_estimated_vo"]["charge_amount_si"] / 100_000;
rate["currency_type"] =
shippingRate["online_estimated_vo"]["currency_type"];
rate["charge_amount_with_currency_str"] =
shippingRate["online_estimated_vo"][
"currecharge_amount_with_currency_strncy_type"
];
rate["estimated_delivery_date"] =
shippingRate["online_estimated_vo"]["estimated_delivery_date"];
rate["estimated_text"] =
shippingRate["online_estimated_vo"]["estimated_text"];
rate["is_selected"] = false;
rates.push(rate);
} catch (e) {
console.log(e);
}
}
};
/*
* write rates to file
*/
const writeToFile = async (data, path) => {
fs.writeFileSync(path, JSON.stringify(data, null, 2));
console.log(`Saved JSON to ${path}`);
};
/*
* check if dimension already already otherwise create new one
@ -491,9 +624,9 @@ const utils = require("./utils");
// get orders
if (orders_list.length > 0) {
// goto every order page
for (const [index, order] of orders_list.entries()) {
try {
try {
// goto every order page
for (const [index, order] of orders_list.entries()) {
console.log(
`Syncing Order ${order} ( ${index + 1} / ${orders_list.length} )`
);
@ -504,9 +637,9 @@ const utils = require("./utils");
const orderPage = utils.getTemuOrderPage(order);
await page.goto(orderPage, {
waitUntil: ["domcontentloaded"],
waitUntil: ["networkidle2"],
});
await new Promise((resolve) => setTimeout(resolve, 1000));
await new Promise((resolve) => setTimeout(resolve, 2000));
// check for buy shipping button
const buyShippingSelector = "div._3yOxLjm0._2pgGmJ7w.IoqjAtdZ";
@ -516,10 +649,14 @@ const utils = require("./utils");
continue;
}
const responsePromise = checkShippingRates(page, 10_000);
// button exist
await buyShippingBtn.click();
console.log("Clicking on Buy Shipping Button");
await new Promise((resolve) => setTimeout(resolve, 5 * 1_000));
await selectShippingDate1(page);
const responsePromise = checkShippingRates(page, 10_000);
let orderShippingRates = await responsePromise;
@ -527,24 +664,43 @@ const utils = require("./utils");
if (utils.isEmpty(orderShippingRates)) {
console.log(`Shipping Rates not found`);
continue;
// populate orders details to populate shipping rates
const promise = checkShippingRates(page, 10_000);
await populateDataInFields(page, order);
await new Promise((resolve) => setTimeout(resolve, 2 * 1000));
orderShippingRates = await promise;
}
// fields are already populate / save the shipping rates
mapResponseToRates(orderShippingRates, order);
// write the JSON data to a file
const outputFilePath = path.join(
config[environment].temu_orders_shipping_rates,
"/unprocessed",
`${order}.json`
);
if (rates.length > 1) {
// save to rates to file
await writeToFile(rates, outputFilePath);
}
// wait 10 seconds
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
// reinitialize rates
rates = [];
} catch (e) {
console.log(e);
}
} catch (e) {
console.log(e);
emailUtils.notify(`Sync Temu Orders Shipping Rates`, e.message);
}
}
console.log(`==========< ENDED BUY SHIPPING ${utils.getPakistanStandardTime(luxon.DateTime.now())} >==========`);
console.log(
`==========< ENDED ${utils.getPakistanStandardTime(
luxon.DateTime.now()
)} >==========`
);
await page.close();
await browser.close();
})();

89
sync-shipping-rates.js
View File

@ -236,25 +236,82 @@ const emailUtils = require("./email-utils");
/*
* Select shipping date
*/
const selectShippingDate = async (page) => {
const selectShippingDate1 = async (page) => {
try {
console.log("Selecting Dates")
// select shipping date
const modalSelector = ".MDL_innerWrapper_123"; // Class selector for the modal
await page.waitForSelector(modalSelector);
// Optionally, wait for the modal content to be loaded and scrollable
const modalContentSelector = ".MDL_body_123"; // Adjust if modal content has a specific class
await page.waitForSelector(modalContentSelector);
// Scroll to the bottom of the modal content
await page.evaluate((modalContentSelector) => {
const modalContent = document.querySelector(modalContentSelector);
if (modalContent) {
modalContent.scrollTo(0, modalContent.scrollHeight);
}
}, modalContentSelector);
const shippingDateSelector =
'div[id="packageList[0].trackingInfoList[0].shipDate"] input:first-of-type';
await page.waitForSelector(shippingDateSelector);
"#packageList\\[0\\]\\.trackingInfoList\\[0\\]\\.shipDate > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123";
const shippingDateInput = await page.$(shippingDateSelector);
if (!shippingDateInput) {
console.log("Select not found.");
return;
}
await page.click(shippingDateSelector);
// wait for 5 seconds
await new Promise((resolve) => setTimeout(resolve, 2 * 1000));
// select shipping date
await new Promise((resolve) => setTimeout(resolve, 2_000));
const shippingDateOptionSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_largeDropdown_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div:nth-child(1) > ul > li.cIL_item_123.cIL_large_123.cIL_highlight_123.ST_itemRendererLabel_123";
await page.waitForSelector(shippingDateOptionSelector);
const shippingDateOption = await page.$(shippingDateOptionSelector);
if (!shippingDateOption) {
console.log("Option not found");
return;
}
await page.click(shippingDateOptionSelector);
} catch (e) {
console.log(e);
console.log("Selected a shipping date");
} catch (error) {
console.error("Error selecting shipping date:", error);
}
};
/*
* Select shipping date
*/
const selectShippingDate = async (page) => {
try {
console.log("Selecting Dates");
// Selector for the shipping date dropdown
const shippingDateSelector =
'div[id="packageList[0].trackingInfoList[0].shipDate"] input:first-of-type';
const shippingDateInput = await page.$(shippingDateSelector);
if (shippingDateInput) {
console.log("Select found!");
await page.click(shippingDateSelector);
const shippingDateOptionSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_largeDropdown_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div:nth-child(1) > ul > li.cIL_item_123.cIL_large_123.cIL_highlight_123.ST_itemRendererLabel_123";
const shippingDateOptionSelectorInput = await page.$(
shippingDateOptionSelector
);
await page.waitForTimeout(5 * 1000);
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
if (shippingDateOptionSelectorInput) {
await shippingDateOptionSelectorInput.click();
console.log("Selected a shipping date");
} else {
console.log("Option not found");
}
} else {
console.log("Select not found.");
}
} catch (error) {
console.error("Error selecting shipping date:", error);
}
};
@ -592,11 +649,13 @@ const emailUtils = require("./email-utils");
continue;
}
// button exist
await buyShippingBtn.click();
console.log("Clicking on Buy Shipping Button");
await selectShippingDate(page);
await new Promise((resolve) => setTimeout(resolve, 5 * 1_000));
await selectShippingDate1(page);
const responsePromise = checkShippingRates(page, 10_000);
let orderShippingRates = await responsePromise;