4 changed files with 3 additions and 667 deletions
--- a/config.json
+++ b/config.json
@ -1,6 +1,6 @@
 {
    "dev": {
-        "temu_orders_shipping_rates" : "D:/Projects/Crawlers/temu-label-crawler/shipping_rates",
+        "temu_orders_shipping_rates" : "D:/Projects/Crawlers/temu-label-crawler/temu_orders_shipping_rates",
        "temu_orders_path" : "D:/Projects/Crawlers/temu-label-crawler",
 		"orders_path": "D:/Projects/Crawlers/amazon_crawler/data/orders",
        "voc_path":"D:/Projects/Crawlers/amazon_crawler/data/voc",
@ -13,7 +13,6 @@
        "localhost_return_comments_path":"http://192.168.91.11:8080/uind/rest/cosmos/amazon-voc/return-comments/edit",
        "cosmos_path_reviews" : "https://portal.utopiaindustries.pk/uind/rest/cosmos/amazon-reviews/edit",
        "cosmos_temu_orders" : "http://192.168.90.228:8081/cosmos/temu/order-upload-json",
        "cosmos_temu_order_shipping_rates" : "https://cosmos.utopiadeals.com/cosmos/order-shipping-rates-json",
        "shopify_products_path" : "D:/Projects/Crawlers/amazon_crawler/data/products",
        "shopify_variants_path" : "D:/Projects/Crawlers/amazon_crawler/data/variants",
        "amz_product_path" : "D:/Projects/Crawlers/amazon_crawler/data/products",
@ -35,8 +34,7 @@
        "cosmos_path_orders_progress": "https://portal.utopiaindustries.pk/uind/rest/cosmos/amazon-orders-progress/edit",
        "cosmos_path_vocs" : "https://portal.utopiaindustries.pk/uind/rest/cosmos/amazon-voc/edit",
        "cosmos_path_reviews" : "https://portal.utopiaindustries.pk/uind/rest/cosmos/amazon-reviews/edit",
-        "cosmos_temu_orders" : "https://cosmos.utopiadeals.com/cosmos/temu/order-upload-json",
+        "cosmos_temu_orders" : "https://cosmos.utopiadeals.com/cosmos/temu/order-upload-json", 
        "cosmos_temu_order_shipping_rates" : "https://cosmos.utopiadeals.com/cosmos/order-shipping-rates-json",
        "temuLoginPage" : "https://seller.temu.com/login.html",
        "temuUnshippedOrdersPage" : "https://seller.temu.com/orders.html?activeTab=2",
        "temuOrderPage" : "https://seller.temu.com/order-detail.html?parent_order_sn=",
--- a/send-shipping-rates.js
+++ b/send-shipping-rates.js
@ -1,57 +0,0 @@
 const axios = require("axios");
 const fs = require("fs");
 const path = require("path");
 const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
 const { exit } = require("process");
 (async function () {
  /**
   * load config data
   */
  const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
  const environment = process.env["ENVIRONMENT"];
  /**
   * directory path
   */
  let shippingRatesPath = config[environment].temu_orders_shipping_rates;
  let unProcessedPath = shippingRatesPath + "/unprocessed";
  let processedPath = shippingRatesPath + "/processed";
  if (!fs.existsSync(processedPath)) {
    fs.mkdirSync(processedPath);
  }
  /**
   * read all files in directory, send data to cosmos then move to processed
   */
  const jsonFiles = fs
    .readdirSync(unProcessedPath)
    .filter((file) => path.extname(file).toLocaleLowerCase() === ".json");
    if( jsonFiles.length === 0 ){
      console.log( `No Files Present at ${unProcessedPath}`)
    }
  for (const file of jsonFiles) {
    try {
      const filePath = path.join(unProcessedPath, file);
      const orders = JSON.parse(fs.readFileSync(filePath, "utf-8"));
      console.log(`Processing: ${filePath}`);
      // send post request to cosmos
      const axiosConfig = {
        method: "get",
        url: config[environment].cosmos_temu_order_shipping_rates,
        headers: {
          "Content-Type": "application/json",
        },
        data: orders, // Add the orders object to the data field
      };
      const res = await axios(axiosConfig);
      if (res["status"] == 200) {
        fs.renameSync(filePath, path.join(processedPath, file));
      }
    } catch (e) {
      console.log(e);
    }
  }
 })();
--- a/sync-shipping-rates.js
+++ b/sync-shipping-rates.js
@ -1,587 +0,0 @@
 const puppeteer = require("puppeteer");
 const axios = require("axios");
 const luxon = require("luxon");
 const { exit } = require("process");
 const fs = require("fs");
 const path = require("path");
 const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
 const utils = require("./utils");
 (async function () {
  console.log(`===========< STARTED  ${luxon.DateTime.now()} >=========`);
  const syncDate = luxon.DateTime.now().toFormat("yyyy-MM-dd");
  /**
   * loading config data
   */
  const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
  const environment = process.env["ENVIRONMENT"];
  const cryptoConfig = utils.getCryptoConfig();
  let rates = [];
  const email = utils.decryptString(
    process.env["temu-email"],
    cryptoConfig.algo,
    cryptoConfig.key,
    cryptoConfig.iv
  );
  const password = utils.decryptString(
    process.env["temu-password"],
    cryptoConfig.algo,
    cryptoConfig.key,
    cryptoConfig.iv
  );
  /*
   * load cookies
   */
  const loadPageCookies = async function (page) {
    const cookiesFileName = `cookies.json`;
    if (fs.existsSync(__dirname + `/cookies/${cookiesFileName}`)) {
      const cookiesStr = fs.readFileSync(
        __dirname + `/cookies/${cookiesFileName}`
      );
      const cookies = JSON.parse(cookiesStr);
      await page.setCookie(...cookies);
    }
  };
  // launch browser and open page
  const chromeProfilePath = path.resolve(
    __dirname,
    config[environment]["chrome_profile_path"]
  );
  const browser = await puppeteer.launch(
    utils.getBrowserConfig(chromeProfilePath, environment)
  );
  const page = await browser.newPage();
  await loadPageCookies(page);
  await page.setViewport({
    width: 1600,
    height: 900,
  });
  // Inject CSS to show the cursor
  await page.evaluate(() => {
    const style = document.createElement("style");
    style.innerHTML = "* { cursor: auto !important; }";
    document.head.appendChild(style);
  });
  // save cookies on page load
  const cookiesFileName = `cookies.json`;
  page.on("load", async function () {
    // save cookies
    const cookies = await page.cookies();
    fs.writeFileSync(
      __dirname + `/cookies/${cookiesFileName}`,
      JSON.stringify(cookies, null, 2)
    );
  });
  /*
   * goto login page
   */
  const loginPage = config[environment]["temuLoginPage"];
  await page.goto(loginPage, {
    waitUntil: ["domcontentloaded"],
  });
  await utils.tryTemuLogin(page, email, password, loginPage);
  await new Promise((resolve) => setTimeout(resolve, 7000));
  // goto orders request page
  const UnshippedOrdersRequestPage =
    config[environment]["temuUnshippedOrdersPage"];
  await page.goto(UnshippedOrdersRequestPage, {
    waitUntil: ["domcontentloaded"],
  });
  const getOrdersFromPage = async (page) => {
    let orderNumbers = [];
    try {
      const orderPOSelector = "div._3GLf87F3";
      const orderPoList = await page.$$(orderPOSelector);
      console.log(`Total Orders On Page ${orderPoList.length}`);
      if (orderPoList === undefined || orderPoList.length === 0) {
        console.log("No Unshipped Orders Found !! ");
        return;
      }
      for (const element of orderPoList) {
        try {
          const orderNumber = await page.evaluate(
            (el) => el.textContent.trim(),
            element
          );
          orderNumbers.push(orderNumber);
        } catch (e) {
          console.log(e);
        }
      }
      return orderNumbers;
    } catch (e) {
      console.log(`Error in Crawling Orders  ${e}`);
    }
    return orderNumbers;
  };
  // orders array
  let orders_list = [];
  const pagination = 10;
  let total_items = 0;
  let currentPage = 1;
  if (orders_list.length === 0) {
    try {
      // get total items
      await page
        .waitForSelector("li.PGT_totalText_123", { timeout: 5000 })
        .catch(() => {});
      const liText = await page.evaluate(() => {
        const liElement = document.querySelector("li.PGT_totalText_123");
        return liElement ? liElement.textContent : null;
      });
      if (liText === null) {
        total_items = 10;
      } else {
        total_items = parseInt(liText.split(" ")[1]);
        console.log(`Total Items count : ${total_items}`);
      }
      let total_pages = Math.ceil(total_items / pagination);
      console.log(`Total Pages count : ${total_pages}`);
      // crawl next pages
      while (true) {
        try {
          console.log(`Crawling for page ${currentPage}`);
          await utils.tryTemuLogin(page, email, password, loginPage);
          await new Promise((resolve) => setTimeout(resolve, 4000));
          // load cookies
          await loadPageCookies(page);
          // get orders
          let orders = await getOrdersFromPage(page);
          orders_list.push(...orders);
          // increment page
          ++currentPage;
          // Evaluate the presence of both classes in the <li> element
          const hasNextBtn = await page.evaluate(() => {
            const liElement = document.querySelector(
              "li.PGT_next_123.PGT_disabled_123"
            );
            return liElement == null;
          });
          // break if doesn't have next button
          if (!hasNextBtn) {
            console.log("No next button");
            break;
          }
          if (currentPage > total_pages) {
            console.log("Last Page Reached");
            break;
          }
          // goto next page
          if (hasNextBtn) {
            await page.evaluate(() => {
              const liElement = document.querySelector("li.PGT_next_123");
              if (liElement) {
                liElement.click();
              }
            });
          }
          // wait
          await new Promise((r) => setTimeout(r, 5000));
        } catch (e) {
          console.log(e);
        }
      }
    } catch (e) {
      console.log(e);
    }
  }
  /**
   * Capture response
   */
  const checkShippingRates = async (page, timer) => {
    return new Promise((resolve, reject) => {
      // Timeout mechanism to resolve with an empty list after 5 seconds
      const timeout = setTimeout(() => {
        page.off("response", handleResponse); // Remove listener on timeout
        resolve([]); // Resolve with an empty list
      }, timer);
      const handleResponse = async (res) => {
        try {
          const req = res.request();
          if (req.url().includes("/query_shipping_provider_optional")) {
            const resJson = await res.json();
            // Remove listener and clear timeout once response is captured
            clearTimeout(timeout);
            page.off("response", handleResponse);
            resolve(resJson.result.online_channel_vo_list || []);
          }
        } catch (ex) {
          // Remove listener and clear timeout on error
          clearTimeout(timeout);
          page.off("response", handleResponse);
          reject(ex);
        }
      };
      page.on("response", handleResponse);
    });
  };
  /*
   * check for shipping rate in response
   */
  /*
   * map response to rates
   */
  const mapResponseToRates = (shippingRates, order) => {
    for (const shippingRate of shippingRates) {
      try {
        let rate = {};
        rate["orderId"] = order;
        rate["channel_id"] = shippingRate["channel_id"];
        rate["ship_product_name"] = shippingRate["ship_product_name"];
        rate["ship_company_id"] = shippingRate["ship_company_id"];
        rate["shipping_company_name"] = shippingRate["shipping_company_name"];
        rate["faraway_type"] = shippingRate["faraway_type"];
        rate["service_code"] = shippingRate["service_code"];
        rate["ship_logistics_type"] = shippingRate["ship_logistics_type"];
        rate["require_reservation"] = shippingRate["require_reservation"];
        //
        rate["amount"] =
          shippingRate["online_estimated_vo"]["charge_amount_si"] / 100_000;
        rate["currency_type"] =
          shippingRate["online_estimated_vo"]["currency_type"];
        rate["charge_amount_with_currency_str"] =
          shippingRate["online_estimated_vo"][
            "currecharge_amount_with_currency_strncy_type"
          ];
        rate["estimated_delivery_date"] =
          shippingRate["online_estimated_vo"]["estimated_delivery_date"];
        rate["estimated_text"] =
          shippingRate["online_estimated_vo"]["estimated_text"];
        rate["is_selected"] = false;
        rates.push(rate);
      } catch (e) {
        console.log(e);
      }
    }
  };
  /*
   * write rates to file
   */
  const writeToFile = async (data, path) => {
    fs.writeFileSync(path, JSON.stringify(data, null, 2));
    console.log(`Saved JSON to ${path}`);
  };
  /*
   * check if dimension already already otherwise create new one
   */
  const calculateDimensions = (dailyPickPackMap, skuToQuantityMap) => {
    let length = 0.0;
    let width = 0.0;
    let height = 0.0;
    for (const [sku, entity] of skuToQuantityMap) {
      try {
        const pickPack = dailyPickPackMap.get(sku);
        length = pickPack.length;
        width = pickPack.width;
        height = height + pickPack.height * entity.quantity;
      } catch (e) {
        console.log(e);
      }
    }
    return {
      length: Math.ceil(length),
      width: Math.ceil(width),
      height: Math.ceil(height),
    };
  };
  // length width height in inches
  /**
   *  calculate weight
   */
  const calculateSkuWeight = (dailyPickPackMap, skuToQuantityMap) => {
    try {
      let weight = 0.0;
      for (const [sku, entity] of skuToQuantityMap) {
        const pickPack = dailyPickPackMap.get(sku);
        weight = weight + pickPack.weight * entity.quantity;
      }
      return Math.round(weight);
    } catch (e) {
      console.log(e);
      return 0;
    }
  };
  const getDimensionString = (length, width, height) => {
    return `Custom ${length}in x ${width}in x ${height}in`;
  };
  /*
   * populate fields in form
   */
  const populateDataInFields = async (page, order) => {
    try {
      // Selector for all items in the order
      const itemsSelector = "div.n3xi4S2p";
      // Get all item elements
      const itemElements = await page.$$(itemsSelector);
      if (!itemElements || itemElements.length === 0) {
        console.log(`No Items Found in Order ${order}`);
        return;
      }
      // get orders with 1 item with quantity 1
      console.log( `Item Quantity : ${itemElements.length}`)
      if (itemElements.length > 1) {
        console.log(`${order} : has more then 1 item`);
        return;
      }
      let itemSkuQuantityList = [];
      // loop over order items and get sku and quantities
      for (const itemElement of itemElements) {
        let itemSkuQuantity = {};
        const sku = await itemElement
          .$eval("span._3E6fOFxc:nth-of-type(2)", (skuEl) =>
            skuEl.textContent.trim()
          )
          .catch(() => null); // Catch errors if the element doesn't exist
        const quantity = await page
          .$eval("span._3Fs-U187", (element) => {
            // Extract the text content, split by "Qty:", and trim the result
            const text = element.parentElement.textContent || "";
            return text.replace("Qty:", "").trim();
          })
          .catch(() => 1);
        //  itemSkuQuantity["sku"] = sku.split("-")[0] ?? "";
        itemSkuQuantity["sku"] = sku.substring(0, sku.lastIndexOf("-")) ?? "";
        itemSkuQuantity["quantity"] = parseInt(quantity);
        // check quantity should be 1 as well
        if (parseInt(quantity) > 1) {
          console.log("Item has more than quantity")
          return;
        }
        itemSkuQuantityList.push(itemSkuQuantity);
        console.log(`Order: ${order}, SKU: ${sku}, Quantity: ${quantity}`);
      }
      // date for daily pick pack
      let date = utils.getFirstDayToCurrentMonth();
      if (
        itemSkuQuantityList !== undefined &&
        itemSkuQuantityList.length !== 0
      ) {
        // sku str
        let skuStr = itemSkuQuantityList
          .filter((item) => item.sku.trim() !== "")
          .map((item) => item.sku)
          .join(",");
        // cosmos url
        const dailyPickPackUrl = utils.getSkuDailyPickPack(skuStr, date);
        console.log(dailyPickPackUrl);
        // request cosmos
        const axiosConfig = {
          method: "get",
          url: dailyPickPackUrl,
          headers: {
            "Content-Type": "application/json",
          },
        };
        // get pick packs
        const response = await axios(axiosConfig);
        const pickPacks = response.data;
        // sku map
        const skuMapToPickPack = new Map(
          pickPacks.map((item) => [item.sku, item])
        );
        // sku quality map
        const skuMapToQuatityMap = new Map(
          itemSkuQuantityList.map((item) => [item.sku, item])
        );
        // weight selector
        const weightInputSelector =
          'div[id="packageList[0].trackingInfoList[0].weight"] input:first-of-type';
        await page.waitForSelector(weightInputSelector);
        // calculate weight of order items
        const totalCalWeight = calculateSkuWeight(
          skuMapToPickPack,
          skuMapToQuatityMap
        );
        // Type inside the input field
        await page.type(weightInputSelector, String(totalCalWeight));
        await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
        // dimension selector
        const dimensionInputSelector =
          'div[id="packageList[0].trackingInfoList[0].sizeInfo"] input:first-of-type';
        await page.waitForSelector(dimensionInputSelector);
        // click on add dimension
        await page.click(dimensionInputSelector);
        // wait for 2 seconds
        await new Promise((resolve) => setTimeout(resolve, 2 * 1000));
        let { length, width, height } = calculateDimensions(
          skuMapToPickPack,
          skuMapToQuatityMap
        );
        console.log(`length : ${length}`);
        console.log(`width : ${width}`);
        console.log(`height  : ${height}`);
        // check for dimension already exists
        const listItemsSelector =
          "body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_mediumDropdown_123.ST_customItem_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > div > div._2A9Ayt3Y > ul > li";
        const listItems = await page.$$(listItemsSelector);
        console.log(`Dimensions count  ${listItems.length}`);
        // Loop through and extract text content or perform actions
        for (const listItem of listItems) {
          const text = await page.evaluate(
            (el) => el.textContent.trim(),
            listItem
          );
          if (text === getDimensionString(length, width, height)) {
            console.log(`Clicked on list item with text: ${text}`);
            await listItem.click();
            return;
          }
        }
        // click on Add dimensions
        await page.click("div._3fps8VlR > div._1IbQdfUN");
        // length input
        const lengthSelector =
          "#length > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input";
        await page.waitForSelector(lengthSelector);
        await page.type(lengthSelector, String(length));
        // width input
        const widthSelector =
          "#width > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input";
        await page.waitForSelector(widthSelector);
        await page.type(widthSelector, String(width));
        // height input
        const heightSelector =
          "#height > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div.IPT_inputWrapper_123.IPTN_inputWrapper_123.IPT_collapseRight_123 > div > div.IPT_inputBlockCell_123 > input";
        await page.waitForSelector(heightSelector);
        await page.type(heightSelector, String(height));
        // wait for 5 seconds
        await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
        await page.mouse.click(100, 100, { button: "left" });
        // click on save btn
        const saveBtnSelector =
          "body > div:nth-child(14) > div > div > div > div.MDL_bottom_123 > div.MDL_footer_123 > div > div._3yOxLjm0._2pgGmJ7w._1eT_m6dA > span._2ISpB3A2";
        await page.waitForSelector(saveBtnSelector);
        await page.click(saveBtnSelector);
        // wait for 3 seconds
        await new Promise((resolve) => setTimeout(resolve, 3 * 1000));
      }
    } catch (e) {
      console.log(e);
    }
  };
  // get orders
  if (orders_list.length > 0) {
    // goto every order page
    for (const [index, order] of orders_list.entries()) {
      try {
        console.log(
          `Syncing Order ${order} ( ${index + 1} / ${orders_list.length} )`
        );
        const orderPage = utils.getTemuOrderPage(order);
        await page.goto(orderPage, {
          waitUntil: ["domcontentloaded"],
        });
        await new Promise((resolve) => setTimeout(resolve, 1000));
        // check for buy shipping button
        const buyShippingSelector = "div._3yOxLjm0._2pgGmJ7w.IoqjAtdZ";
        const buyShippingBtn = await page.$(buyShippingSelector);
        if (!buyShippingBtn) {
          console.log("No Buy Shipping Button found");
          continue;
        }
        const responsePromise = checkShippingRates(page, 10_000);
        // button exist
        await buyShippingBtn.click();
        console.log("Clicking on Buy Shipping Button");
        let orderShippingRates = await responsePromise;
        await new Promise((resolve) => setTimeout(resolve, 2 * 1_000));
        if (utils.isEmpty(orderShippingRates)) {
          console.log(`Shipping Rates not found`);
          // populate orders details to populate shipping rates
          const promise = checkShippingRates(page, 20_000);
          await populateDataInFields(page, order);
          await new Promise((resolve) => setTimeout(resolve, 2 * 1000));
          orderShippingRates = await promise;
        }
        // fields are already populate / save the shipping rates
        mapResponseToRates(orderShippingRates, order);
        // write the JSON data to a file
        const outputFilePath = path.join(
          config[environment].temu_orders_shipping_rates,
          "/unprocessed",
          `${order}.json`
        );
        if (rates.length > 1) {
          // save to rates to file
          await writeToFile(rates, outputFilePath);
        }
        // wait 10 seconds
        await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
        // reinitialize rates
        rates = [];
      } catch (e) {
        console.log(e);
      }
    }
  }
  console.log(`==========< ENDED  ${luxon.DateTime.now()}  >==========`);
  await page.close();
  await browser.close();
 })();
--- a/utils.js
+++ b/utils.js
@ -291,24 +291,6 @@ exports.getTemuOrdersPage = getTemuOrdersPage;
 * check list
 */
 const isEmpty =  function (arr) {
-  return arr === undefined || arr.length === 0;
+  return arr.length === 0;
 }
 exports.isEmpty = isEmpty;
 /**
 * daily pick pack from cosmos ( dimensions )
 */
 const getSkuDailyPickPack = function( skuStr, date){
  return `https://cosmos.utopiadeals.com/cosmos/daily-pick-packs-json?marketplace=AMAZON_USA&skus=${skuStr}&from=${date}&to=${date}`
 }
 exports.getSkuDailyPickPack = getSkuDailyPickPack;
 /*
 *
 */
 const getFirstDayToCurrentMonth = function( ){
  const now = new Date();
  const firstDay = new Date(now.getFullYear(), now.getMonth(), 2); 
  return firstDay.toISOString().split('T')[0];
 }
 exports.getFirstDayToCurrentMonth = getFirstDayToCurrentMonth;