588 lines
19 KiB
JavaScript
588 lines
19 KiB
JavaScript
const puppeteer = require("puppeteer");
|
|
const axios = require("axios");
|
|
const luxon = require("luxon");
|
|
const { exit } = require("process");
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
|
|
|
|
const utils = require("./utils");
|
|
|
|
(async function () {
|
|
console.log(`===========< STARTED ${luxon.DateTime.now()} >=========`);
|
|
|
|
const syncDate = luxon.DateTime.now().toFormat("yyyy-MM-dd");
|
|
/**
|
|
* loading config data
|
|
*/
|
|
const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
|
|
const environment = process.env["ENVIRONMENT"];
|
|
const cryptoConfig = utils.getCryptoConfig();
|
|
let rates = [];
|
|
|
|
const email = utils.decryptString(
|
|
process.env["temu-email"],
|
|
cryptoConfig.algo,
|
|
cryptoConfig.key,
|
|
cryptoConfig.iv
|
|
);
|
|
const password = utils.decryptString(
|
|
process.env["temu-password"],
|
|
cryptoConfig.algo,
|
|
cryptoConfig.key,
|
|
cryptoConfig.iv
|
|
);
|
|
|
|
/*
|
|
* load cookies
|
|
*/
|
|
const loadPageCookies = async function (page) {
|
|
const cookiesFileName = `cookies.json`;
|
|
if (fs.existsSync(__dirname + `/cookies/${cookiesFileName}`)) {
|
|
const cookiesStr = fs.readFileSync(
|
|
__dirname + `/cookies/${cookiesFileName}`
|
|
);
|
|
const cookies = JSON.parse(cookiesStr);
|
|
await page.setCookie(...cookies);
|
|
}
|
|
};
|
|
|
|
// launch browser and open page
|
|
const chromeProfilePath = path.resolve(
|
|
__dirname,
|
|
config[environment]["chrome_profile_path"]
|
|
);
|
|
const browser = await puppeteer.launch(
|
|
utils.getBrowserConfig(chromeProfilePath, environment)
|
|
);
|
|
const page = await browser.newPage();
|
|
await loadPageCookies(page);
|
|
await page.setViewport({
|
|
width: 1600,
|
|
height: 900,
|
|
});
|
|
|
|
// Inject CSS to show the cursor
|
|
await page.evaluate(() => {
|
|
const style = document.createElement("style");
|
|
style.innerHTML = "* { cursor: auto !important; }";
|
|
document.head.appendChild(style);
|
|
});
|
|
|
|
// save cookies on page load
|
|
const cookiesFileName = `cookies.json`;
|
|
page.on("load", async function () {
|
|
// save cookies
|
|
const cookies = await page.cookies();
|
|
fs.writeFileSync(
|
|
__dirname + `/cookies/${cookiesFileName}`,
|
|
JSON.stringify(cookies, null, 2)
|
|
);
|
|
});
|
|
|
|
/*
|
|
* goto login page
|
|
*/
|
|
const loginPage = config[environment]["temuLoginPage"];
|
|
await page.goto(loginPage, {
|
|
waitUntil: ["domcontentloaded"],
|
|
});
|
|
|
|
await utils.tryTemuLogin(page, email, password, loginPage);
|
|
await new Promise((resolve) => setTimeout(resolve, 7000));
|
|
|
|
// goto orders request page
|
|
const UnshippedOrdersRequestPage =
|
|
config[environment]["temuUnshippedOrdersPage"];
|
|
await page.goto(UnshippedOrdersRequestPage, {
|
|
waitUntil: ["domcontentloaded"],
|
|
});
|
|
|
|
const getOrdersFromPage = async (page) => {
|
|
let orderNumbers = [];
|
|
try {
|
|
const orderPOSelector = "div._3GLf87F3";
|
|
const orderPoList = await page.$$(orderPOSelector);
|
|
console.log(`Total Orders On Page ${orderPoList.length}`);
|
|
if (orderPoList === undefined || orderPoList.length === 0) {
|
|
console.log("No Unshipped Orders Found !! ");
|
|
return;
|
|
}
|
|
|
|
for (const element of orderPoList) {
|
|
try {
|
|
const orderNumber = await page.evaluate(
|
|
(el) => el.textContent.trim(),
|
|
element
|
|
);
|
|
orderNumbers.push(orderNumber);
|
|
} catch (e) {
|
|
console.log(e);
|
|
}
|
|
}
|
|
return orderNumbers;
|
|
} catch (e) {
|
|
console.log(`Error in Crawling Orders ${e}`);
|
|
}
|
|
return orderNumbers;
|
|
};
|
|
|
|
// orders array
|
|
let orders_list = [];
|
|
const pagination = 10;
|
|
let total_items = 0;
|
|
let currentPage = 1;
|
|
|
|
if (orders_list.length === 0) {
|
|
try {
|
|
// get total items
|
|
await page
|
|
.waitForSelector("li.PGT_totalText_123", { timeout: 5000 })
|
|
.catch(() => {});
|
|
const liText = await page.evaluate(() => {
|
|
const liElement = document.querySelector("li.PGT_totalText_123");
|
|
return liElement ? liElement.textContent : null;
|
|
});
|
|
|
|
if (liText === null) {
|
|
total_items = 10;
|
|
} else {
|
|
total_items = parseInt(liText.split(" ")[1]);
|
|
console.log(`Total Items count : ${total_items}`);
|
|
}
|
|
|
|
let total_pages = Math.ceil(total_items / pagination);
|
|
console.log(`Total Pages count : ${total_pages}`);
|
|
// crawl next pages
|
|
while (true) {
|
|
try {
|
|
console.log(`Crawling for page ${currentPage}`);
|
|
|
|
await utils.tryTemuLogin(page, email, password, loginPage);
|
|
await new Promise((resolve) => setTimeout(resolve, 4000));
|
|
|
|
// load cookies
|
|
await loadPageCookies(page);
|
|
|
|
// get orders
|
|
let orders = await getOrdersFromPage(page);
|
|
orders_list.push(...orders);
|
|
|
|
// increment page
|
|
++currentPage;
|
|
|
|
// Evaluate the presence of both classes in the <li> element
|
|
const hasNextBtn = await page.evaluate(() => {
|
|
const liElement = document.querySelector(
|
|
"li.PGT_next_123.PGT_disabled_123"
|
|
);
|
|
return liElement == null;
|
|
});
|
|
|
|
// break if doesn't have next button
|
|
if (!hasNextBtn) {
|
|
console.log("No next button");
|
|
break;
|
|
}
|
|
|
|
if (currentPage > total_pages) {
|
|
console.log("Last Page Reached");
|
|
break;
|
|
}
|
|
|
|
// goto next page
|
|
if (hasNextBtn) {
|
|
await page.evaluate(() => {
|
|
const liElement = document.querySelector("li.PGT_next_123");
|
|
if (liElement) {
|
|
liElement.click();
|
|
}
|
|
});
|
|
}
|
|
|
|
// wait
|
|
await new Promise((r) => setTimeout(r, 5000));
|
|
} catch (e) {
|
|
console.log(e);
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.log(e);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Capture response
|
|
*/
|
|
const checkShippingRates = async (page, timer) => {
|
|
return new Promise((resolve, reject) => {
|
|
// Timeout mechanism to resolve with an empty list after 5 seconds
|
|
const timeout = setTimeout(() => {
|
|
page.off("response", handleResponse); // Remove listener on timeout
|
|
resolve([]); // Resolve with an empty list
|
|
}, timer);
|
|
|
|
const handleResponse = async (res) => {
|
|
try {
|
|
const req = res.request();
|
|
if (req.url().includes("/query_shipping_provider_optional")) {
|
|
const resJson = await res.json();
|
|
// Remove listener and clear timeout once response is captured
|
|
clearTimeout(timeout);
|
|
page.off("response", handleResponse);
|
|
resolve(resJson.result.online_channel_vo_list || []);
|
|
}
|
|
} catch (ex) {
|
|
// Remove listener and clear timeout on error
|
|
clearTimeout(timeout);
|
|
page.off("response", handleResponse);
|
|
reject(ex);
|
|
}
|
|
};
|
|
|
|
page.on("response", handleResponse);
|
|
});
|
|
};
|
|
|
|
/*
|
|
* check for shipping rate in response
|
|
*/
|
|
|
|
/*
|
|
* map response to rates
|
|
*/
|
|
const mapResponseToRates = (shippingRates, order) => {
|
|
for (const shippingRate of shippingRates) {
|
|
try {
|
|
let rate = {};
|
|
rate["orderId"] = order;
|
|
rate["channel_id"] = shippingRate["channel_id"];
|
|
rate["ship_product_name"] = shippingRate["ship_product_name"];
|
|
rate["ship_company_id"] = shippingRate["ship_company_id"];
|
|
rate["shipping_company_name"] = shippingRate["shipping_company_name"];
|
|
rate["faraway_type"] = shippingRate["faraway_type"];
|
|
rate["service_code"] = shippingRate["service_code"];
|
|
rate["ship_logistics_type"] = shippingRate["ship_logistics_type"];
|
|
rate["require_reservation"] = shippingRate["require_reservation"];
|
|
//
|
|
rate["amount"] =
|
|
shippingRate["online_estimated_vo"]["charge_amount_si"] / 100_000;
|
|
rate["currency_type"] =
|
|
shippingRate["online_estimated_vo"]["currency_type"];
|
|
rate["charge_amount_with_currency_str"] =
|
|
shippingRate["online_estimated_vo"][
|
|
"currecharge_amount_with_currency_strncy_type"
|
|
];
|
|
rate["estimated_delivery_date"] =
|
|
shippingRate["online_estimated_vo"]["estimated_delivery_date"];
|
|
rate["estimated_text"] =
|
|
shippingRate["online_estimated_vo"]["estimated_text"];
|
|
rate["is_selected"] = false;
|
|
rates.push(rate);
|
|
} catch (e) {
|
|
console.log(e);
|
|
}
|
|
}
|
|
};
|
|
|
|
/*
|
|
* write rates to file
|
|
*/
|
|
const writeToFile = async (data, path) => {
|
|
fs.writeFileSync(path, JSON.stringify(data, null, 2));
|
|
console.log(`Saved JSON to ${path}`);
|
|
};
|
|
|
|
/*
|
|
* check if dimension already already otherwise create new one
|
|
*/
|
|
const calculateDimensions = (dailyPickPackMap, skuToQuantityMap) => {
|
|
let length = 0.0;
|
|
let width = 0.0;
|
|
let height = 0.0;
|
|
for (const [sku, entity] of skuToQuantityMap) {
|
|
try {
|
|
const pickPack = dailyPickPackMap.get(sku);
|
|
length = pickPack.length;
|
|
width = pickPack.width;
|
|
height = height + pickPack.height * entity.quantity;
|
|
} catch (e) {
|
|
console.log(e);
|
|
}
|
|
}
|
|
return {
|
|
length: Math.ceil(length),
|
|
width: Math.ceil(width),
|
|
height: Math.ceil(height),
|
|
};
|
|
};
|
|
// length width height in inches
|
|
|
|
/**
|
|
* calculate weight
|
|
*/
|
|
const calculateSkuWeight = (dailyPickPackMap, skuToQuantityMap) => {
|
|
try {
|
|
let weight = 0.0;
|
|
for (const [sku, entity] of skuToQuantityMap) {
|
|
const pickPack = dailyPickPackMap.get(sku);
|
|
weight = weight + pickPack.weight * entity.quantity;
|
|
}
|
|
return Math.round(weight);
|
|
} catch (e) {
|
|
console.log(e);
|
|
return 0;
|
|
}
|
|
};
|
|
|
|
const getDimensionString = (length, width, height) => {
|
|
return `Custom ${length}in x ${width}in x ${height}in`;
|
|
};
|
|
|
|
/*
|
|
* populate fields in form
|
|
*/
|
|
const populateDataInFields = async (page, order) => {
|
|
try {
|
|
// Selector for all items in the order
|
|
const itemsSelector = "div.n3xi4S2p";
|
|
|
|
// Get all item elements
|
|
const itemElements = await page.$$(itemsSelector);
|
|
if (!itemElements || itemElements.length === 0) {
|
|
console.log(`No Items Found in Order ${order}`);
|
|
return;
|
|
}
|
|
// get orders with 1 item with quantity 1
|
|
console.log( `Item Quantity : ${itemElements.length}`)
|
|
if (itemElements.length > 1) {
|
|
console.log(`${order} : has more then 1 item`);
|
|
return;
|
|
}
|
|
|
|
let itemSkuQuantityList = [];
|
|
// loop over order items and get sku and quantities
|
|
for (const itemElement of itemElements) {
|
|
let itemSkuQuantity = {};
|
|
const sku = await itemElement
|
|
.$eval("span._3E6fOFxc:nth-of-type(2)", (skuEl) =>
|
|
skuEl.textContent.trim()
|
|
)
|
|
.catch(() => null); // Catch errors if the element doesn't exist
|
|
|
|
const quantity = await page
|
|
.$eval("span._3Fs-U187", (element) => {
|
|
// Extract the text content, split by "Qty:", and trim the result
|
|
const text = element.parentElement.textContent || "";
|
|
return text.replace("Qty:", "").trim();
|
|
})
|
|
.catch(() => 1);
|
|
|
|
// itemSkuQuantity["sku"] = sku.split("-")[0] ?? "";
|
|
itemSkuQuantity["sku"] = sku.substring(0, sku.lastIndexOf("-")) ?? "";
|
|
itemSkuQuantity["quantity"] = parseInt(quantity);
|
|
|
|
// check quantity should be 1 as well
|
|
if (parseInt(quantity) > 1) {
|
|
console.log("Item has more than quantity")
|
|
return;
|
|
}
|
|
|
|
itemSkuQuantityList.push(itemSkuQuantity);
|
|
console.log(`Order: ${order}, SKU: ${sku}, Quantity: ${quantity}`);
|
|
}
|
|
|
|
// date for daily pick pack
|
|
let date = utils.getFirstDayToCurrentMonth();
|
|
|
|
if (
|
|
itemSkuQuantityList !== undefined &&
|
|
itemSkuQuantityList.length !== 0
|
|
) {
|
|
// sku str
|
|
let skuStr = itemSkuQuantityList
|
|
.filter((item) => item.sku.trim() !== "")
|
|
.map((item) => item.sku)
|
|
.join(",");
|
|
|
|
// cosmos url
|
|
const dailyPickPackUrl = utils.getSkuDailyPickPack(skuStr, date);
|
|
console.log(dailyPickPackUrl);
|
|
|
|
// request cosmos
|
|
const axiosConfig = {
|
|
method: "get",
|
|
url: dailyPickPackUrl,
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
},
|
|
};
|
|
|
|
// get pick packs
|
|
const response = await axios(axiosConfig);
|
|
const pickPacks = response.data;
|
|
|
|
// sku map
|
|
const skuMapToPickPack = new Map(
|
|
pickPacks.map((item) => [item.sku, item])
|
|
);
|
|
// sku quality map
|
|
const skuMapToQuatityMap = new Map(
|
|
itemSkuQuantityList.map((item) => [item.sku, item])
|
|
);
|
|
// weight selector
|
|
const weightInputSelector =
|
|
'div[id="packageList[0].trackingInfoList[0].weight"] input:first-of-type';
|
|
await page.waitForSelector(weightInputSelector);
|
|
|
|
// calculate weight of order items
|
|
const totalCalWeight = calculateSkuWeight(
|
|
skuMapToPickPack,
|
|
skuMapToQuatityMap
|
|
);
|
|
|
|
// Type inside the input field
|
|
await page.type(weightInputSelector, String(totalCalWeight));
|
|
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
|
|
|
|
// dimension selector
|
|
const dimensionInputSelector =
|
|
'div[id="packageList[0].trackingInfoList[0].sizeInfo"] input:first-of-type';
|
|
await page.waitForSelector(dimensionInputSelector);
|
|
// click on add dimension
|
|
await page.click(dimensionInputSelector);
|
|
// wait for 2 seconds
|
|
await new Promise((resolve) => setTimeout(resolve, 2 * 1000));
|
|
|
|
let { length, width, height } = calculateDimensions(
|
|
skuMapToPickPack,
|
|
skuMapToQuatityMap
|
|
);
|
|
|
|
console.log(`length : ${length}`);
|
|
console.log(`width : ${width}`);
|
|
console.log(`height : ${height}`);
|
|
|
|
// check for dimension already exists
|
|
const listItemsSelector =
|
|
"body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_mediumDropdown_123.ST_customItem_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div._2A9Ayt3Y > ul > li";
|
|
const listItems = await page.$$(listItemsSelector);
|
|
console.log(`Dimensions count ${listItems.length}`);
|
|
// Loop through and extract text content or perform actions
|
|
for (const listItem of listItems) {
|
|
const text = await page.evaluate(
|
|
(el) => el.textContent.trim(),
|
|
listItem
|
|
);
|
|
if (text === getDimensionString(length, width, height)) {
|
|
console.log(`Clicked on list item with text: ${text}`);
|
|
await listItem.click();
|
|
return;
|
|
}
|
|
}
|
|
// click on Add dimensions
|
|
await page.click("div._3fps8VlR > div._1IbQdfUN");
|
|
|
|
// length input
|
|
const lengthSelector =
|
|
"#length > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input";
|
|
await page.waitForSelector(lengthSelector);
|
|
|
|
await page.type(lengthSelector, String(length));
|
|
// width input
|
|
const widthSelector =
|
|
"#width > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input";
|
|
await page.waitForSelector(widthSelector);
|
|
await page.type(widthSelector, String(width));
|
|
// height input
|
|
const heightSelector =
|
|
"#height > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input";
|
|
await page.waitForSelector(heightSelector);
|
|
await page.type(heightSelector, String(height));
|
|
|
|
// wait for 5 seconds
|
|
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
|
|
|
|
await page.mouse.click(100, 100, { button: "left" });
|
|
|
|
// click on save btn
|
|
const saveBtnSelector =
|
|
"body > div:nth-child(14) > div > div > div > div.MDL_bottom_123 > div.MDL_footer_123 > div > div._3yOxLjm0._2pgGmJ7w._1eT_m6dA > span._2ISpB3A2";
|
|
await page.waitForSelector(saveBtnSelector);
|
|
await page.click(saveBtnSelector);
|
|
// wait for 3 seconds
|
|
await new Promise((resolve) => setTimeout(resolve, 3 * 1000));
|
|
}
|
|
} catch (e) {
|
|
console.log(e);
|
|
}
|
|
};
|
|
|
|
// get orders
|
|
if (orders_list.length > 0) {
|
|
// goto every order page
|
|
for (const [index, order] of orders_list.entries()) {
|
|
try {
|
|
console.log(
|
|
`Syncing Order ${order} ( ${index + 1} / ${orders_list.length} )`
|
|
);
|
|
const orderPage = utils.getTemuOrderPage(order);
|
|
await page.goto(orderPage, {
|
|
waitUntil: ["domcontentloaded"],
|
|
});
|
|
await new Promise((resolve) => setTimeout(resolve, 1000));
|
|
|
|
// check for buy shipping button
|
|
const buyShippingSelector = "div._3yOxLjm0._2pgGmJ7w.IoqjAtdZ";
|
|
const buyShippingBtn = await page.$(buyShippingSelector);
|
|
if (!buyShippingBtn) {
|
|
console.log("No Buy Shipping Button found");
|
|
continue;
|
|
}
|
|
|
|
const responsePromise = checkShippingRates(page, 10_000);
|
|
// button exist
|
|
await buyShippingBtn.click();
|
|
console.log("Clicking on Buy Shipping Button");
|
|
|
|
let orderShippingRates = await responsePromise;
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, 2 * 1_000));
|
|
|
|
if (utils.isEmpty(orderShippingRates)) {
|
|
console.log(`Shipping Rates not found`);
|
|
// populate orders details to populate shipping rates
|
|
const promise = checkShippingRates(page, 20_000);
|
|
await populateDataInFields(page, order);
|
|
await new Promise((resolve) => setTimeout(resolve, 2 * 1000));
|
|
orderShippingRates = await promise;
|
|
}
|
|
// fields are already populate / save the shipping rates
|
|
mapResponseToRates(orderShippingRates, order);
|
|
|
|
// write the JSON data to a file
|
|
const outputFilePath = path.join(
|
|
config[environment].temu_orders_shipping_rates,
|
|
"/unprocessed",
|
|
`${order}.json`
|
|
);
|
|
if (rates.length > 1) {
|
|
// save to rates to file
|
|
await writeToFile(rates, outputFilePath);
|
|
}
|
|
|
|
// wait 10 seconds
|
|
await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
|
|
// reinitialize rates
|
|
rates = [];
|
|
} catch (e) {
|
|
console.log(e);
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`==========< ENDED ${luxon.DateTime.now()} >==========`);
|
|
await page.close();
|
|
await browser.close();
|
|
})();
|