temu-labels-crawler/sync-shipping-rates.js

707 lines
24 KiB
JavaScript

const puppeteer = require("puppeteer");
const axios = require("axios");
const luxon = require("luxon");
const { exit } = require("process");
const fs = require("fs");
const path = require("path");
const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
const utils = require("./utils");
const emailUtils = require("./email-utils");
(async function () {
console.log(
`===========< STARTED ${utils.getPakistanStandardTime(
luxon.DateTime.now()
)} >=========`
);
const syncDate = luxon.DateTime.now().toFormat("yyyy-MM-dd");
/**
* loading config data
*/
const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
const environment = process.env["ENVIRONMENT"];
const cryptoConfig = utils.getCryptoConfig();
let rates = [];
const email = utils.decryptString(
process.env["temu-personal-email"],
cryptoConfig.algo,
cryptoConfig.key,
cryptoConfig.iv
);
const password = utils.decryptString(
process.env["temu-personal-password"],
cryptoConfig.algo,
cryptoConfig.key,
cryptoConfig.iv
);
/*
* load cookies
*/
const loadPageCookies = async function (page) {
const cookiesFileName = `cookies.json`;
if (fs.existsSync(__dirname + `/cookies/${cookiesFileName}`)) {
const cookiesStr = fs.readFileSync(
__dirname + `/cookies/${cookiesFileName}`
);
const cookies = JSON.parse(cookiesStr);
await page.setCookie(...cookies);
}
};
// launch browser and open page
const chromeProfilePath = path.resolve(
__dirname,
config[environment]["chrome_profile_path"]
);
const browser = await puppeteer.launch(
utils.getBrowserConfig(chromeProfilePath, environment)
);
const page = await browser.newPage();
// await loadPageCookies(page);
await page.setViewport({
width: 1600,
height: 900,
});
// Inject CSS to show the cursor
await page.evaluate(() => {
const style = document.createElement("style");
style.innerHTML = "* { cursor: auto !important; }";
document.head.appendChild(style);
});
// save cookies on page load
const cookiesFileName = `cookies.json`;
page.on("load", async function () {
// save cookies
const cookies = await page.cookies();
fs.writeFileSync(
__dirname + `/cookies/${cookiesFileName}`,
JSON.stringify(cookies, null, 2)
);
});
/*
* goto login page
*/
const loginPage = config[environment]["temuLoginPage"];
await page.goto(loginPage, {
waitUntil: ["networkidle2"],
});
await utils.tryTemuLogin(page, email, password, loginPage);
await new Promise((resolve) => setTimeout(resolve, 7000));
// goto orders request page
const UnshippedOrdersRequestPage =
config[environment]["temuUnshippedOrdersPage"];
await page.goto(UnshippedOrdersRequestPage, {
waitUntil: ["networkidle2"],
});
const getOrdersFromPage = async (page) => {
let orderNumbers = [];
try {
const orderPOSelector = "div._3GLf87F3";
const orderPoList = await page.$$(orderPOSelector);
console.log(`Total Orders On Page ${orderPoList.length}`);
if (orderPoList === undefined || orderPoList.length === 0) {
console.log("No Unshipped Orders Found !! ");
return;
}
for (const element of orderPoList) {
try {
const orderNumber = await page.evaluate(
(el) => el.textContent.trim(),
element
);
orderNumbers.push(orderNumber);
} catch (e) {
console.log(e);
}
}
return orderNumbers;
} catch (e) {
console.log(`Error in Crawling Orders ${e}`);
}
return orderNumbers;
};
// orders array
let orders_list = [];
const pagination = 100;
let total_items = 0;
let currentPage = 1;
await new Promise((resolve) => setTimeout(resolve, 2_000));
// set the pagination to 100 / page
const pageinationSeletor =
"#orders-tab-list > div.y0DVv7GO > div > div._38NAUUfN > div._15QWqbZs > ul > li.PGT_sizeChanger_123 > div > div > div > div > div";
await page.waitForSelector(pageinationSeletor);
await page.click(pageinationSeletor);
console.log("Clicking on pagination Select");
await new Promise((resolve) => setTimeout(resolve, 3_000));
// select 100 /page option
const pagesPerPageSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_mediumDropdown_123.PT_dropdown_123.PT_portalTopLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > ul > li:nth-child(5)";
await page.waitForSelector(pagesPerPageSelector);
await page.click(pagesPerPageSelector);
await new Promise((resolve) => setTimeout(resolve, 1_000));
if (orders_list.length === 0) {
try {
// get total items
await page
.waitForSelector("li.PGT_totalText_123", { timeout: 5000 })
.catch(() => {});
const liText = await page.evaluate(() => {
const liElement = document.querySelector("li.PGT_totalText_123");
return liElement ? liElement.textContent : null;
});
if (liText === null) {
total_items = 10;
} else {
total_items = parseInt(liText.split(" ")[1]);
console.log(`Total Items count : ${total_items}`);
}
let total_pages = Math.ceil(total_items / pagination);
console.log(`Total Pages count : ${total_pages}`);
// crawl next pages
while (true) {
try {
console.log(`Crawling for page ${currentPage}`);
await utils.tryTemuLogin(page, email, password, loginPage);
await new Promise((resolve) => setTimeout(resolve, 4000));
// load cookies
await loadPageCookies(page);
// get orders
let orders = await getOrdersFromPage(page);
orders_list.push(...orders);
// increment page
++currentPage;
// Evaluate the presence of both classes in the <li> element
const hasNextBtn = await page.evaluate(() => {
const liElement = document.querySelector(
"li.PGT_next_123.PGT_disabled_123"
);
return liElement == null;
});
// break if doesn't have next button
if (!hasNextBtn) {
console.log("No next button");
break;
}
if (currentPage > total_pages) {
console.log("Last Page Reached");
break;
}
// goto next page
if (hasNextBtn) {
await page.evaluate(() => {
const liElement = document.querySelector("li.PGT_next_123");
if (liElement) {
liElement.click();
}
});
}
// wait
await new Promise((r) => setTimeout(r, 5000));
} catch (e) {
console.log(e);
break;
}
}
} catch (e) {
console.log(e);
}
}
/*
* Select shipping date
*/
const selectShippingDate1 = async (page) => {
try {
const modalSelector = ".MDL_innerWrapper_123"; // Class selector for the modal
await page.waitForSelector(modalSelector);
// Optionally, wait for the modal content to be loaded and scrollable
const modalContentSelector = ".MDL_body_123"; // Adjust if modal content has a specific class
await page.waitForSelector(modalContentSelector);
// Scroll to the bottom of the modal content
await page.evaluate((modalContentSelector) => {
const modalContent = document.querySelector(modalContentSelector);
if (modalContent) {
modalContent.scrollTo(0, modalContent.scrollHeight);
}
}, modalContentSelector);
const shippingDateSelector =
"#packageList\\[0\\]\\.trackingInfoList\\[0\\]\\.shipDate > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123";
const shippingDateInput = await page.$(shippingDateSelector);
if (!shippingDateInput) {
console.log("Select not found.");
return;
}
await page.click(shippingDateSelector);
await new Promise((resolve) => setTimeout(resolve, 2_000));
const shippingDateOptionSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_largeDropdown_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div:nth-child(1) > ul > li.cIL_item_123.cIL_large_123.cIL_highlight_123.ST_itemRendererLabel_123";
const shippingDateOption = await page.$(shippingDateOptionSelector);
if (!shippingDateOption) {
console.log("Option not found");
return;
}
await page.click(shippingDateOptionSelector);
console.log("Selected a shipping date");
} catch (error) {
console.error("Error selecting shipping date:", error);
}
};
/*
* Select shipping date
*/
const selectShippingDate = async (page) => {
try {
console.log("Selecting Dates");
// Selector for the shipping date dropdown
const shippingDateSelector =
'div[id="packageList[0].trackingInfoList[0].shipDate"] input:first-of-type';
const shippingDateInput = await page.$(shippingDateSelector);
if (shippingDateInput) {
console.log("Select found!");
await page.click(shippingDateSelector);
const shippingDateOptionSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_largeDropdown_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div:nth-child(1) > ul > li.cIL_item_123.cIL_large_123.cIL_highlight_123.ST_itemRendererLabel_123";
const shippingDateOptionSelectorInput = await page.$(
shippingDateOptionSelector
);
await page.waitForTimeout(5 * 1000);
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
if (shippingDateOptionSelectorInput) {
await shippingDateOptionSelectorInput.click();
console.log("Selected a shipping date");
} else {
console.log("Option not found");
}
} else {
console.log("Select not found.");
}
} catch (error) {
console.error("Error selecting shipping date:", error);
}
};
/**
* Capture response
*/
const checkShippingRates = async (page, timer) => {
return new Promise((resolve, reject) => {
// Timeout mechanism to resolve with an empty list after 5 seconds
const timeout = setTimeout(() => {
page.off("response", handleResponse); // Remove listener on timeout
resolve([]); // Resolve with an empty list
}, timer);
const handleResponse = async (res) => {
try {
const req = res.request();
if (req.url().includes("/query_shipping_provider_optional")) {
const resJson = await res.json();
// Remove listener and clear timeout once response is captured
clearTimeout(timeout);
page.off("response", handleResponse);
resolve(resJson.result.online_channel_vo_list || []);
}
} catch (ex) {
// Remove listener and clear timeout on error
clearTimeout(timeout);
page.off("response", handleResponse);
reject(ex);
}
};
page.on("response", handleResponse);
});
};
/*
* check for shipping rate in response
*/
/*
* map response to rates
*/
const mapResponseToRates = (shippingRates, order) => {
for (const shippingRate of shippingRates) {
try {
let rate = {};
rate["orderId"] = order;
rate["channel_id"] = shippingRate["channel_id"];
rate["ship_product_name"] = shippingRate["ship_product_name"];
rate["ship_company_id"] = shippingRate["ship_company_id"];
rate["shipping_company_name"] = shippingRate["shipping_company_name"];
rate["faraway_type"] = shippingRate["faraway_type"];
rate["service_code"] = shippingRate["service_code"];
rate["ship_logistics_type"] = shippingRate["ship_logistics_type"];
rate["require_reservation"] = shippingRate["require_reservation"];
//
rate["amount"] =
shippingRate["online_estimated_vo"]["charge_amount_si"] / 100_000;
rate["currency_type"] =
shippingRate["online_estimated_vo"]["currency_type"];
rate["charge_amount_with_currency_str"] =
shippingRate["online_estimated_vo"][
"currecharge_amount_with_currency_strncy_type"
];
rate["estimated_delivery_date"] =
shippingRate["online_estimated_vo"]["estimated_delivery_date"];
rate["estimated_text"] =
shippingRate["online_estimated_vo"]["estimated_text"];
rate["is_selected"] = false;
rates.push(rate);
} catch (e) {
console.log(e);
}
}
};
/*
* write rates to file
*/
const writeToFile = async (data, path) => {
fs.writeFileSync(path, JSON.stringify(data, null, 2));
console.log(`Saved JSON to ${path}`);
};
/*
* check if dimension already already otherwise create new one
*/
const calculateDimensions = (dailyPickPackMap, skuToQuantityMap) => {
let length = 0.0;
let width = 0.0;
let height = 0.0;
for (const [sku, entity] of skuToQuantityMap) {
try {
const pickPack = dailyPickPackMap.get(sku);
length = pickPack.length;
width = pickPack.width;
height = height + pickPack.height * entity.quantity;
} catch (e) {
console.log(e);
}
}
return {
length: Math.ceil(length),
width: Math.ceil(width),
height: Math.ceil(height),
};
};
// length width height in inches
/**
* calculate weight
*/
const calculateSkuWeight = (dailyPickPackMap, skuToQuantityMap) => {
try {
let weight = 0.0;
for (const [sku, entity] of skuToQuantityMap) {
const pickPack = dailyPickPackMap.get(sku);
weight = weight + pickPack.weight * entity.quantity;
}
return Math.round(weight);
} catch (e) {
console.log(e);
return 0;
}
};
const getDimensionString = (length, width, height) => {
return `Custom ${length}in x ${width}in x ${height}in`;
};
/*
* populate fields in form
*/
const populateDataInFields = async (page, order) => {
try {
// Selector for all items in the order
const itemsSelector = "div.n3xi4S2p";
// Get all item elements
const itemElements = await page.$$(itemsSelector);
if (!itemElements || itemElements.length === 0) {
console.log(`No Items Found in Order ${order}`);
return;
}
// get orders with 1 item with quantity 1
console.log(`Item Quantity : ${itemElements.length}`);
if (itemElements.length > 1) {
console.log(`${order} : has more then 1 item`);
return;
}
let itemSkuQuantityList = [];
// loop over order items and get sku and quantities
for (const itemElement of itemElements) {
let itemSkuQuantity = {};
const sku = await itemElement
.$eval("span._3E6fOFxc:nth-of-type(2)", (skuEl) =>
skuEl.textContent.trim()
)
.catch(() => null); // Catch errors if the element doesn't exist
const quantity = await page
.$eval("span._3Fs-U187", (element) => {
// Extract the text content, split by "Qty:", and trim the result
const text = element.parentElement.textContent || "";
return text.replace("Qty:", "").trim();
})
.catch(() => 1);
// itemSkuQuantity["sku"] = sku.split("-")[0] ?? "";
itemSkuQuantity["sku"] = sku.substring(0, sku.lastIndexOf("-")) ?? "";
itemSkuQuantity["quantity"] = parseInt(quantity);
// check quantity should be 1 as well
if (parseInt(quantity) > 1) {
console.log("Item has more than quantity");
return;
}
itemSkuQuantityList.push(itemSkuQuantity);
console.log(`Order: ${order}, SKU: ${sku}, Quantity: ${quantity}`);
}
// date for daily pick pack
let date = utils.getFirstDayToCurrentMonth();
if (
itemSkuQuantityList !== undefined &&
itemSkuQuantityList.length !== 0
) {
// sku str
let skuStr = itemSkuQuantityList
.filter((item) => item.sku.trim() !== "")
.map((item) => item.sku)
.join(",");
// cosmos url
const dailyPickPackUrl = utils.getSkuDailyPickPack(skuStr, date);
console.log(dailyPickPackUrl);
// request cosmos
const axiosConfig = {
method: "get",
url: dailyPickPackUrl,
headers: {
"Content-Type": "application/json",
},
};
// get pick packs
const response = await axios(axiosConfig);
const pickPacks = response.data;
// sku map
const skuMapToPickPack = new Map(
pickPacks.map((item) => [item.sku, item])
);
// sku quality map
const skuMapToQuatityMap = new Map(
itemSkuQuantityList.map((item) => [item.sku, item])
);
// weight selector
const weightInputSelector =
'div[id="packageList[0].trackingInfoList[0].weight"] input:first-of-type';
await page.waitForSelector(weightInputSelector);
// calculate weight of order items
const totalCalWeight = calculateSkuWeight(
skuMapToPickPack,
skuMapToQuatityMap
);
// Type inside the input field
await page.type(weightInputSelector, String(totalCalWeight));
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
// dimension selector
const dimensionInputSelector =
'div[id="packageList[0].trackingInfoList[0].sizeInfo"] input:first-of-type';
await page.waitForSelector(dimensionInputSelector);
// click on add dimension
await page.click(dimensionInputSelector);
// wait for 2 seconds
await new Promise((resolve) => setTimeout(resolve, 2 * 1000));
let { length, width, height } = calculateDimensions(
skuMapToPickPack,
skuMapToQuatityMap
);
console.log(`length : ${length}`);
console.log(`width : ${width}`);
console.log(`height : ${height}`);
// check for dimension already exists
const listItemsSelector =
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_mediumDropdown_123.ST_customItem_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div._2A9Ayt3Y > ul > li";
const listItems = await page.$$(listItemsSelector);
console.log(`Dimensions count ${listItems.length}`);
// Loop through and extract text content or perform actions
for (const listItem of listItems) {
const text = await page.evaluate(
(el) => el.textContent.trim(),
listItem
);
if (text === getDimensionString(length, width, height)) {
console.log(`Clicked on list item with text: ${text}`);
await listItem.click();
return;
}
}
// click on Add dimensions
await page.click("div._3fps8VlR > div._1IbQdfUN");
// length input
const lengthSelector =
"#length > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input";
await page.waitForSelector(lengthSelector);
await page.type(lengthSelector, String(length));
// width input
const widthSelector =
"#width > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input";
await page.waitForSelector(widthSelector);
await page.type(widthSelector, String(width));
// height input
const heightSelector =
"#height > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input";
await page.waitForSelector(heightSelector);
await page.type(heightSelector, String(height));
// wait for 5 seconds
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
await page.mouse.click(100, 100, { button: "left" });
// click on save btn
const saveBtnSelector =
"body > div:nth-child(14) > div > div > div > div.MDL_bottom_123 > div.MDL_footer_123 > div > div._3yOxLjm0._2pgGmJ7w._1eT_m6dA > span._2ISpB3A2";
await page.waitForSelector(saveBtnSelector);
await page.click(saveBtnSelector);
// wait for 3 seconds
await new Promise((resolve) => setTimeout(resolve, 3 * 1000));
}
} catch (e) {
console.log(e);
}
};
// get orders
if (orders_list.length > 0) {
try {
// goto every order page
for (const [index, order] of orders_list.entries()) {
console.log(
`Syncing Order ${order} ( ${index + 1} / ${orders_list.length} )`
);
// try login
await utils.tryTemuLogin(page, email, password, loginPage);
await new Promise((resolve) => setTimeout(resolve, 2000));
const orderPage = utils.getTemuOrderPage(order);
await page.goto(orderPage, {
waitUntil: ["networkidle2"],
});
await new Promise((resolve) => setTimeout(resolve, 2000));
// check for buy shipping button
const buyShippingSelector = "div._3yOxLjm0._2pgGmJ7w.IoqjAtdZ";
const buyShippingBtn = await page.$(buyShippingSelector);
if (!buyShippingBtn) {
console.log("No Buy Shipping Button found");
continue;
}
// button exist
await buyShippingBtn.click();
console.log("Clicking on Buy Shipping Button");
await new Promise((resolve) => setTimeout(resolve, 5 * 1_000));
await selectShippingDate1(page);
const responsePromise = checkShippingRates(page, 10_000);
let orderShippingRates = await responsePromise;
await new Promise((resolve) => setTimeout(resolve, 2 * 1_000));
if (utils.isEmpty(orderShippingRates)) {
console.log(`Shipping Rates not found`);
continue;
// populate orders details to populate shipping rates
const promise = checkShippingRates(page, 10_000);
await populateDataInFields(page, order);
await new Promise((resolve) => setTimeout(resolve, 2 * 1000));
orderShippingRates = await promise;
}
// fields are already populate / save the shipping rates
mapResponseToRates(orderShippingRates, order);
// write the JSON data to a file
const outputFilePath = path.join(
config[environment].temu_orders_shipping_rates,
"/unprocessed",
`${order}.json`
);
if (rates.length > 1) {
// save to rates to file
await writeToFile(rates, outputFilePath);
}
// wait 10 seconds
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
// reinitialize rates
rates = [];
}
} catch (e) {
console.log(e);
emailUtils.notify(`Sync Temu Orders Shipping Rates`, e.message);
}
}
console.log(
`==========< ENDED ${utils.getPakistanStandardTime(
luxon.DateTime.now()
)} >==========`
);
await page.close();
await browser.close();
})();