From 756a291213a77112dd76631de8cbdf478b9c20f9 Mon Sep 17 00:00:00 2001 From: saif Date: Fri, 10 Jan 2025 11:21:44 +0500 Subject: [PATCH] added shipping rates sync functionality --- send-shipping-rates.js | 57 ++++ sync-shipping-rates.js | 587 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 644 insertions(+) create mode 100644 send-shipping-rates.js create mode 100644 sync-shipping-rates.js diff --git a/send-shipping-rates.js b/send-shipping-rates.js new file mode 100644 index 0000000..aee8968 --- /dev/null +++ b/send-shipping-rates.js @@ -0,0 +1,57 @@ +const axios = require("axios"); +const fs = require("fs"); +const path = require("path"); +const dotenv = require("dotenv").config({ path: __dirname + "/.env" }); +const { exit } = require("process"); + +(async function () { + /** + * load config data + */ + const config = JSON.parse(fs.readFileSync(__dirname + "/config.json")); + const environment = process.env["ENVIRONMENT"]; + + /** + * directory path + */ + let shippingRatesPath = config[environment].temu_orders_shipping_rates; + let unProcessedPath = shippingRatesPath + "/unprocessed"; + let processedPath = shippingRatesPath + "/processed"; + if (!fs.existsSync(processedPath)) { + fs.mkdirSync(processedPath); + } + + /** + * read all files in directory, send data to cosmos then move to processed + */ + const jsonFiles = fs + .readdirSync(unProcessedPath) + .filter((file) => path.extname(file).toLocaleLowerCase() === ".json"); + + if( jsonFiles.length === 0 ){ + console.log( `No Files Present at ${unProcessedPath}`) + } + for (const file of jsonFiles) { + try { + const filePath = path.join(unProcessedPath, file); + const orders = JSON.parse(fs.readFileSync(filePath, "utf-8")); + console.log(`Processing: ${filePath}`); + // send post request to cosmos + const axiosConfig = { + method: "get", + url: config[environment].cosmos_temu_order_shipping_rates, + headers: { + "Content-Type": "application/json", + }, + data: orders, // Add the orders object to the data field + }; + + const res = await axios(axiosConfig); + if (res["status"] == 200) { + fs.renameSync(filePath, path.join(processedPath, file)); + } + } catch (e) { + console.log(e); + } + } +})(); diff --git a/sync-shipping-rates.js b/sync-shipping-rates.js new file mode 100644 index 0000000..a947d6e --- /dev/null +++ b/sync-shipping-rates.js @@ -0,0 +1,587 @@ +const puppeteer = require("puppeteer"); +const axios = require("axios"); +const luxon = require("luxon"); +const { exit } = require("process"); +const fs = require("fs"); +const path = require("path"); +const dotenv = require("dotenv").config({ path: __dirname + "/.env" }); + +const utils = require("./utils"); + +(async function () { + console.log(`===========< STARTED ${luxon.DateTime.now()} >=========`); + + const syncDate = luxon.DateTime.now().toFormat("yyyy-MM-dd"); + /** + * loading config data + */ + const config = JSON.parse(fs.readFileSync(__dirname + "/config.json")); + const environment = process.env["ENVIRONMENT"]; + const cryptoConfig = utils.getCryptoConfig(); + let rates = []; + + const email = utils.decryptString( + process.env["temu-email"], + cryptoConfig.algo, + cryptoConfig.key, + cryptoConfig.iv + ); + const password = utils.decryptString( + process.env["temu-password"], + cryptoConfig.algo, + cryptoConfig.key, + cryptoConfig.iv + ); + + /* + * load cookies + */ + const loadPageCookies = async function (page) { + const cookiesFileName = `cookies.json`; + if (fs.existsSync(__dirname + `/cookies/${cookiesFileName}`)) { + const cookiesStr = fs.readFileSync( + __dirname + `/cookies/${cookiesFileName}` + ); + const cookies = JSON.parse(cookiesStr); + await page.setCookie(...cookies); + } + }; + + // launch browser and open page + const chromeProfilePath = path.resolve( + __dirname, + config[environment]["chrome_profile_path"] + ); + const browser = await puppeteer.launch( + utils.getBrowserConfig(chromeProfilePath, environment) + ); + const page = await browser.newPage(); + await loadPageCookies(page); + await page.setViewport({ + width: 1600, + height: 900, + }); + + // Inject CSS to show the cursor + await page.evaluate(() => { + const style = document.createElement("style"); + style.innerHTML = "* { cursor: auto !important; }"; + document.head.appendChild(style); + }); + + // save cookies on page load + const cookiesFileName = `cookies.json`; + page.on("load", async function () { + // save cookies + const cookies = await page.cookies(); + fs.writeFileSync( + __dirname + `/cookies/${cookiesFileName}`, + JSON.stringify(cookies, null, 2) + ); + }); + + /* + * goto login page + */ + const loginPage = config[environment]["temuLoginPage"]; + await page.goto(loginPage, { + waitUntil: ["domcontentloaded"], + }); + + await utils.tryTemuLogin(page, email, password, loginPage); + await new Promise((resolve) => setTimeout(resolve, 7000)); + + // goto orders request page + const UnshippedOrdersRequestPage = + config[environment]["temuUnshippedOrdersPage"]; + await page.goto(UnshippedOrdersRequestPage, { + waitUntil: ["domcontentloaded"], + }); + + const getOrdersFromPage = async (page) => { + let orderNumbers = []; + try { + const orderPOSelector = "div._3GLf87F3"; + const orderPoList = await page.$$(orderPOSelector); + console.log(`Total Orders On Page ${orderPoList.length}`); + if (orderPoList === undefined || orderPoList.length === 0) { + console.log("No Unshipped Orders Found !! "); + return; + } + + for (const element of orderPoList) { + try { + const orderNumber = await page.evaluate( + (el) => el.textContent.trim(), + element + ); + orderNumbers.push(orderNumber); + } catch (e) { + console.log(e); + } + } + return orderNumbers; + } catch (e) { + console.log(`Error in Crawling Orders ${e}`); + } + return orderNumbers; + }; + + // orders array + let orders_list = []; + const pagination = 10; + let total_items = 0; + let currentPage = 1; + + if (orders_list.length === 0) { + try { + // get total items + await page + .waitForSelector("li.PGT_totalText_123", { timeout: 5000 }) + .catch(() => {}); + const liText = await page.evaluate(() => { + const liElement = document.querySelector("li.PGT_totalText_123"); + return liElement ? liElement.textContent : null; + }); + + if (liText === null) { + total_items = 10; + } else { + total_items = parseInt(liText.split(" ")[1]); + console.log(`Total Items count : ${total_items}`); + } + + let total_pages = Math.ceil(total_items / pagination); + console.log(`Total Pages count : ${total_pages}`); + // crawl next pages + while (true) { + try { + console.log(`Crawling for page ${currentPage}`); + + await utils.tryTemuLogin(page, email, password, loginPage); + await new Promise((resolve) => setTimeout(resolve, 4000)); + + // load cookies + await loadPageCookies(page); + + // get orders + let orders = await getOrdersFromPage(page); + orders_list.push(...orders); + + // increment page + ++currentPage; + + // Evaluate the presence of both classes in the
  • element + const hasNextBtn = await page.evaluate(() => { + const liElement = document.querySelector( + "li.PGT_next_123.PGT_disabled_123" + ); + return liElement == null; + }); + + // break if doesn't have next button + if (!hasNextBtn) { + console.log("No next button"); + break; + } + + if (currentPage > total_pages) { + console.log("Last Page Reached"); + break; + } + + // goto next page + if (hasNextBtn) { + await page.evaluate(() => { + const liElement = document.querySelector("li.PGT_next_123"); + if (liElement) { + liElement.click(); + } + }); + } + + // wait + await new Promise((r) => setTimeout(r, 5000)); + } catch (e) { + console.log(e); + } + } + } catch (e) { + console.log(e); + } + } + + /** + * Capture response + */ + const checkShippingRates = async (page, timer) => { + return new Promise((resolve, reject) => { + // Timeout mechanism to resolve with an empty list after 5 seconds + const timeout = setTimeout(() => { + page.off("response", handleResponse); // Remove listener on timeout + resolve([]); // Resolve with an empty list + }, timer); + + const handleResponse = async (res) => { + try { + const req = res.request(); + if (req.url().includes("/query_shipping_provider_optional")) { + const resJson = await res.json(); + // Remove listener and clear timeout once response is captured + clearTimeout(timeout); + page.off("response", handleResponse); + resolve(resJson.result.online_channel_vo_list || []); + } + } catch (ex) { + // Remove listener and clear timeout on error + clearTimeout(timeout); + page.off("response", handleResponse); + reject(ex); + } + }; + + page.on("response", handleResponse); + }); + }; + + /* + * check for shipping rate in response + */ + + /* + * map response to rates + */ + const mapResponseToRates = (shippingRates, order) => { + for (const shippingRate of shippingRates) { + try { + let rate = {}; + rate["orderId"] = order; + rate["channel_id"] = shippingRate["channel_id"]; + rate["ship_product_name"] = shippingRate["ship_product_name"]; + rate["ship_company_id"] = shippingRate["ship_company_id"]; + rate["shipping_company_name"] = shippingRate["shipping_company_name"]; + rate["faraway_type"] = shippingRate["faraway_type"]; + rate["service_code"] = shippingRate["service_code"]; + rate["ship_logistics_type"] = shippingRate["ship_logistics_type"]; + rate["require_reservation"] = shippingRate["require_reservation"]; + // + rate["amount"] = + shippingRate["online_estimated_vo"]["charge_amount_si"] / 100_000; + rate["currency_type"] = + shippingRate["online_estimated_vo"]["currency_type"]; + rate["charge_amount_with_currency_str"] = + shippingRate["online_estimated_vo"][ + "currecharge_amount_with_currency_strncy_type" + ]; + rate["estimated_delivery_date"] = + shippingRate["online_estimated_vo"]["estimated_delivery_date"]; + rate["estimated_text"] = + shippingRate["online_estimated_vo"]["estimated_text"]; + rate["is_selected"] = false; + rates.push(rate); + } catch (e) { + console.log(e); + } + } + }; + + /* + * write rates to file + */ + const writeToFile = async (data, path) => { + fs.writeFileSync(path, JSON.stringify(data, null, 2)); + console.log(`Saved JSON to ${path}`); + }; + + /* + * check if dimension already already otherwise create new one + */ + const calculateDimensions = (dailyPickPackMap, skuToQuantityMap) => { + let length = 0.0; + let width = 0.0; + let height = 0.0; + for (const [sku, entity] of skuToQuantityMap) { + try { + const pickPack = dailyPickPackMap.get(sku); + length = pickPack.length; + width = pickPack.width; + height = height + pickPack.height * entity.quantity; + } catch (e) { + console.log(e); + } + } + return { + length: Math.ceil(length), + width: Math.ceil(width), + height: Math.ceil(height), + }; + }; + // length width height in inches + + /** + * calculate weight + */ + const calculateSkuWeight = (dailyPickPackMap, skuToQuantityMap) => { + try { + let weight = 0.0; + for (const [sku, entity] of skuToQuantityMap) { + const pickPack = dailyPickPackMap.get(sku); + weight = weight + pickPack.weight * entity.quantity; + } + return Math.round(weight); + } catch (e) { + console.log(e); + return 0; + } + }; + + const getDimensionString = (length, width, height) => { + return `Custom ${length}in x ${width}in x ${height}in`; + }; + + /* + * populate fields in form + */ + const populateDataInFields = async (page, order) => { + try { + // Selector for all items in the order + const itemsSelector = "div.n3xi4S2p"; + + // Get all item elements + const itemElements = await page.$$(itemsSelector); + if (!itemElements || itemElements.length === 0) { + console.log(`No Items Found in Order ${order}`); + return; + } + // get orders with 1 item with quantity 1 + console.log( `Item Quantity : ${itemElements.length}`) + if (itemElements.length > 1) { + console.log(`${order} : has more then 1 item`); + return; + } + + let itemSkuQuantityList = []; + // loop over order items and get sku and quantities + for (const itemElement of itemElements) { + let itemSkuQuantity = {}; + const sku = await itemElement + .$eval("span._3E6fOFxc:nth-of-type(2)", (skuEl) => + skuEl.textContent.trim() + ) + .catch(() => null); // Catch errors if the element doesn't exist + + const quantity = await page + .$eval("span._3Fs-U187", (element) => { + // Extract the text content, split by "Qty:", and trim the result + const text = element.parentElement.textContent || ""; + return text.replace("Qty:", "").trim(); + }) + .catch(() => 1); + + // itemSkuQuantity["sku"] = sku.split("-")[0] ?? ""; + itemSkuQuantity["sku"] = sku.substring(0, sku.lastIndexOf("-")) ?? ""; + itemSkuQuantity["quantity"] = parseInt(quantity); + + // check quantity should be 1 as well + if (parseInt(quantity) > 1) { + console.log("Item has more than quantity") + return; + } + + itemSkuQuantityList.push(itemSkuQuantity); + console.log(`Order: ${order}, SKU: ${sku}, Quantity: ${quantity}`); + } + + // date for daily pick pack + let date = utils.getFirstDayToCurrentMonth(); + + if ( + itemSkuQuantityList !== undefined && + itemSkuQuantityList.length !== 0 + ) { + // sku str + let skuStr = itemSkuQuantityList + .filter((item) => item.sku.trim() !== "") + .map((item) => item.sku) + .join(","); + + // cosmos url + const dailyPickPackUrl = utils.getSkuDailyPickPack(skuStr, date); + console.log(dailyPickPackUrl); + + // request cosmos + const axiosConfig = { + method: "get", + url: dailyPickPackUrl, + headers: { + "Content-Type": "application/json", + }, + }; + + // get pick packs + const response = await axios(axiosConfig); + const pickPacks = response.data; + + // sku map + const skuMapToPickPack = new Map( + pickPacks.map((item) => [item.sku, item]) + ); + // sku quality map + const skuMapToQuatityMap = new Map( + itemSkuQuantityList.map((item) => [item.sku, item]) + ); + // weight selector + const weightInputSelector = + 'div[id="packageList[0].trackingInfoList[0].weight"] input:first-of-type'; + await page.waitForSelector(weightInputSelector); + + // calculate weight of order items + const totalCalWeight = calculateSkuWeight( + skuMapToPickPack, + skuMapToQuatityMap + ); + + // Type inside the input field + await page.type(weightInputSelector, String(totalCalWeight)); + await new Promise((resolve) => setTimeout(resolve, 5 * 1000)); + + // dimension selector + const dimensionInputSelector = + 'div[id="packageList[0].trackingInfoList[0].sizeInfo"] input:first-of-type'; + await page.waitForSelector(dimensionInputSelector); + // click on add dimension + await page.click(dimensionInputSelector); + // wait for 2 seconds + await new Promise((resolve) => setTimeout(resolve, 2 * 1000)); + + let { length, width, height } = calculateDimensions( + skuMapToPickPack, + skuMapToQuatityMap + ); + + console.log(`length : ${length}`); + console.log(`width : ${width}`); + console.log(`height : ${height}`); + + // check for dimension already exists + const listItemsSelector = + "body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_mediumDropdown_123.ST_customItem_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div._2A9Ayt3Y > ul > li"; + const listItems = await page.$$(listItemsSelector); + console.log(`Dimensions count ${listItems.length}`); + // Loop through and extract text content or perform actions + for (const listItem of listItems) { + const text = await page.evaluate( + (el) => el.textContent.trim(), + listItem + ); + if (text === getDimensionString(length, width, height)) { + console.log(`Clicked on list item with text: ${text}`); + await listItem.click(); + return; + } + } + // click on Add dimensions + await page.click("div._3fps8VlR > div._1IbQdfUN"); + + // length input + const lengthSelector = + "#length > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input"; + await page.waitForSelector(lengthSelector); + + await page.type(lengthSelector, String(length)); + // width input + const widthSelector = + "#width > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input"; + await page.waitForSelector(widthSelector); + await page.type(widthSelector, String(width)); + // height input + const heightSelector = + "#height > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input"; + await page.waitForSelector(heightSelector); + await page.type(heightSelector, String(height)); + + // wait for 5 seconds + await new Promise((resolve) => setTimeout(resolve, 5 * 1000)); + + await page.mouse.click(100, 100, { button: "left" }); + + // click on save btn + const saveBtnSelector = + "body > div:nth-child(14) > div > div > div > div.MDL_bottom_123 > div.MDL_footer_123 > div > div._3yOxLjm0._2pgGmJ7w._1eT_m6dA > span._2ISpB3A2"; + await page.waitForSelector(saveBtnSelector); + await page.click(saveBtnSelector); + // wait for 3 seconds + await new Promise((resolve) => setTimeout(resolve, 3 * 1000)); + } + } catch (e) { + console.log(e); + } + }; + + // get orders + if (orders_list.length > 0) { + // goto every order page + for (const [index, order] of orders_list.entries()) { + try { + console.log( + `Syncing Order ${order} ( ${index + 1} / ${orders_list.length} )` + ); + const orderPage = utils.getTemuOrderPage(order); + await page.goto(orderPage, { + waitUntil: ["domcontentloaded"], + }); + await new Promise((resolve) => setTimeout(resolve, 1000)); + + // check for buy shipping button + const buyShippingSelector = "div._3yOxLjm0._2pgGmJ7w.IoqjAtdZ"; + const buyShippingBtn = await page.$(buyShippingSelector); + if (!buyShippingBtn) { + console.log("No Buy Shipping Button found"); + continue; + } + + const responsePromise = checkShippingRates(page, 10_000); + // button exist + await buyShippingBtn.click(); + console.log("Clicking on Buy Shipping Button"); + + let orderShippingRates = await responsePromise; + + await new Promise((resolve) => setTimeout(resolve, 2 * 1_000)); + + if (utils.isEmpty(orderShippingRates)) { + console.log(`Shipping Rates not found`); + // populate orders details to populate shipping rates + const promise = checkShippingRates(page, 20_000); + await populateDataInFields(page, order); + await new Promise((resolve) => setTimeout(resolve, 2 * 1000)); + orderShippingRates = await promise; + } + // fields are already populate / save the shipping rates + mapResponseToRates(orderShippingRates, order); + + // write the JSON data to a file + const outputFilePath = path.join( + config[environment].temu_orders_shipping_rates, + "/unprocessed", + `${order}.json` + ); + if (rates.length > 1) { + // save to rates to file + await writeToFile(rates, outputFilePath); + } + + // wait 10 seconds + await new Promise((resolve) => setTimeout(resolve, 5 * 1000)); + // reinitialize rates + rates = []; + } catch (e) { + console.log(e); + } + } + } + + console.log(`==========< ENDED ${luxon.DateTime.now()} >==========`); + await page.close(); + await browser.close(); +})();