temu-labels-crawler/sync-orders.js

const puppeteer = require("puppeteer");
const axios = require("axios");
const luxon = require("luxon");
const { exit } = require("process");
const fs = require("fs");
const path = require("path");
const xlsx = require("xlsx");
const dotenv = require("dotenv").config({ path: __dirname + "/.env" });

const utils = require("./utils");

(async function () {
  console.log(`===========< STARTED  ${luxon.DateTime.now()}  >=========`);

  const syncDate = luxon.DateTime.now().toFormat("yyyy-MM-dd");

  const screenshotDirectory = path.join(__dirname, `screenshots/${syncDate}`);

  if (!fs.existsSync(screenshotDirectory)) {
    fs.mkdirSync(screenshotDirectory, { recursive: true });
  }

  /**
   * loading config data
   */
  const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
  const environment = process.env["ENVIRONMENT"];
  const cryptoConfig = utils.getCryptoConfig();

  const email = utils.decryptString(
    process.env["temu-email"],
    cryptoConfig.algo,
    cryptoConfig.key,
    cryptoConfig.iv
  );
  const password = utils.decryptString(
    process.env["temu-password"],
    cryptoConfig.algo,
    cryptoConfig.key,
    cryptoConfig.iv
  );

  /*
   * load cookies
   */
  const loadPageCookies = async function (page) {
    const cookiesFileName = `cookies.json`;
    if (fs.existsSync(__dirname + `/cookies/${cookiesFileName}`)) {
      const cookiesStr = fs.readFileSync(
        __dirname + `/cookies/${cookiesFileName}`
      );
      const cookies = JSON.parse(cookiesStr);
      await page.setCookie(...cookies);
    }
  };

  // launch browser and open page
  const chromeProfilePath = path.resolve(
    __dirname,
    config[environment]["chrome_profile_path"]
  );
  const browser = await puppeteer.launch(
    utils.getBrowserConfig(chromeProfilePath, environment)
  );
  const page = await browser.newPage();
  await loadPageCookies(page);
  await page.setViewport({
    width: 1600,
    height: 900,
  });

  // Inject CSS to show the cursor
  await page.evaluate(() => {
    const style = document.createElement("style");
    style.innerHTML = "* { cursor: auto !important; }";
    document.head.appendChild(style);
  });

  // save cookies on page load
  const cookiesFileName = `cookies.json`;
  page.on("load", async function () {
    // save cookies
    const cookies = await page.cookies();
    fs.writeFileSync(
      __dirname + `/cookies/${cookiesFileName}`,
      JSON.stringify(cookies, null, 2)
    );
  });

  /*
   * configure download path of excel report
   */
  const generateTheDownloadPath = async function (page, downloadPath) {
    // create client session for configuration
    const client = await page.target().createCDPSession();
    // set the download path
    await client.send("Page.setDownloadBehavior", {
      behavior: "allow",
      downloadPath: downloadPath, // Change this to your desired download path.
    });
  };

  /*
   * goto login page
   */
  const loginPage = config[environment]["temuLoginPage"];
  await page.goto(loginPage, {
    waitUntil: ["domcontentloaded"],
  });

  await utils.tryTemuLogin(page, email, password, loginPage);
  await new Promise((resolve) => setTimeout(resolve, 7000));

  // goto orders request page
  const ordersRequestPage = config[environment]["temuOrderReportPage"];
  await page.goto(ordersRequestPage, {
    waitUntil: ["domcontentloaded"],
  });

  /*
   * request order report
   */
  const orderRequestSelector = "div._3yOxLjm0._2pgGmJ7w.IoqjAtdZ.vbGE0cGC";
  const element = await page.$(orderRequestSelector);
  if (element) {
    // wait for 5 sec
    await new Promise((resolve) => setTimeout(resolve, 5 * 1000));
    await element.click();
    console.log("Clicked on the Request Report btn");
    // take screenshot
    let screenshotPath = path.join(
      screenshotDirectory,
      `${luxon.DateTime.now().toMillis()}.png`
    );
    await page.screenshot({ path: screenshotPath, fullPage: true });

    // wait 10 secs
    await new Promise((resolve) => setTimeout(resolve, 10 * 1000));

    // take screenshot
    screenshotPath = path.join(
      screenshotDirectory,
      `${luxon.DateTime.now().toMillis()}.png`
    );
    await page.screenshot({ path: screenshotPath, fullPage: true });
    // wait 10 secs
    await new Promise((resolve) => setTimeout(resolve, 10 * 1000));

    const confirmBtnSelector = "div._3yOxLjm0._2pgGmJ7w._1eT_m6dA";
    const confirmBtnElement = await page.$(confirmBtnSelector);
    if (confirmBtnElement) {
      await new Promise((resolve) => setTimeout(resolve, 15 * 1000));
      await confirmBtnElement.click();
      console.log("Clicked on the Confirm Button");
      // wait for 3 seconds then reload page
      await new Promise((resolve) => setTimeout(resolve, 3 * 1000));

      // take screenshot
      screenshotPath = path.join(
        screenshotDirectory,
        `${luxon.DateTime.now().toMillis()}.png`
      );
      await page.screenshot({ path: screenshotPath, fullPage: true });

      // wait for 3 mins for request to be processed
      await new Promise((resolve) => setTimeout(resolve, 3 * 60 * 1000));
    }
  } else {
    console.log(" Request Btn doest exists");
  }

  // take screenshot
  var screenshotPath = path.join(
    screenshotDirectory,
    `${luxon.DateTime.now().toMillis()}.png`
  );
  await page.screenshot({ path: screenshotPath, fullPage: true });

  // download latest report
  await page.waitForSelector("div._2zs6hel0");

  // Select all divs with the class _2zs6hel0
  const divs = await page.$$("div._2zs6hel0");

  console.log(`Found ${divs.length} Buttons`);

  const downloadFolderPath = path.resolve(
    config[environment].temu_orders_path,
    `unprocessed/`
  );

  // download the latest report
  if (divs !== undefined && divs.length > 1) {
    generateTheDownloadPath(page, downloadFolderPath);
    console.log("Clicking On download Button");
    await new Promise((resolve) => setTimeout(resolve, 10 * 1000));
    // take screenshot
    var screenshotPath = path.join(
      screenshotDirectory,
      `${luxon.DateTime.now().toMillis()}.png`
    );
    await page.screenshot({ path: screenshotPath, fullPage: true });
    await divs[0].click();
  }

  // take screenshot
  var screenshotPath = path.join(
    screenshotDirectory,
    `${luxon.DateTime.now().toMillis()}.png`
  );
  await page.screenshot({ path: screenshotPath, fullPage: true });

  // wait 60 seconds
  await new Promise((resolve) => setTimeout(resolve, 60 * 1000));

  // generate json from excel mark it processed
  const unprocessedFolderPath = path.join(
    config[environment].temu_orders_path,
    "unprocessed"
  );

  // Function to read Excel files and convert to JSON
  const convertExcelToJson = () => {
    fs.readdir(unprocessedFolderPath, (err, files) => {
      if (err) {
        console.error("Error reading the directory:", err);
        return;
      }

      // Filter Excel files (xlsx or xls)
      const excelFiles = files.filter(
        (file) =>
          file.endsWith(".xlsx") ||
          file.endsWith(".xls") ||
          file.endsWith(".csv")
      );

      // Process each Excel file
      excelFiles.forEach((file) => {
        const filePath = path.join(unprocessedFolderPath, file);
        const workbook = xlsx.readFile(filePath);
        const sheetName = workbook.SheetNames[0]; // Use the first sheet
        const worksheet = workbook.Sheets[sheetName];

        // Convert the sheet to JSON
        let jsonData = xlsx.utils.sheet_to_json(worksheet);

        // Modify the column names by adding underscores
        jsonData = jsonData.map((row) => {
          const modifiedRow = {};
          Object.keys(row).forEach((key) => {
            const newKey = key.replace(/\s+/g, "_").toLowerCase();
            const value =
              row[key] === undefined || row[key] === "" || row[key] === null
                ? ""
                : row[key];
            modifiedRow[newKey] = value;
          });
          return modifiedRow;
        });

        // write the JSON data to a file
        const outputFile = path.join(
          config[environment].temu_orders_path,
          "data/unprocessed",
          `${luxon.DateTime.now().toMillis()}.json`
        );
        fs.writeFileSync(outputFile, JSON.stringify(jsonData, null, 2));
        console.log(`Saved JSON to ${outputFile}`);

        console.log(
          `Move Excel file ${filePath} to ${config[environment].temu_orders_path}/processed`
        );
        fs.renameSync(
          filePath,
          path.join(`${config[environment].temu_orders_path}/processed`, file)
        );
      });
    });
  };

  convertExcelToJson();
  console.log(`==========< ENDED  ${luxon.DateTime.now()}  >==========`);
  await page.close();
  await browser.close();
})();