add label download functionality

config.json

@@ -1,5 +1,6 @@
 {
     "dev": {
+        "temu_order_shipping_labels_path" : "D:/Projects/Crawlers/temu-label-crawler/shipping_labels",
         "temu_orders_shipping_rates" : "D:/Projects/Crawlers/temu-label-crawler/shipping_rates",
         "temu_orders_path" : "D:/Projects/Crawlers/temu-label-crawler",
 		"orders_path": "D:/Projects/Crawlers/amazon_crawler/data/orders",
@@ -25,6 +26,7 @@
         "temuUnshippedOrdersUrl" : "https://cosmos.utopiadeals.com/cosmos/temu/get-orders-by-status?status=Unshipped"
 	},
     "prod": {
+        "temu_order_shipping_labels_path" : "/mnt/AmazonReports/Temu/shipping_labels",
         "temu_orders_shipping_rates" : "/mnt/AmazonReports/Temu/shipping_rates",
         "temu_orders_path" : "/mnt/AmazonReports/Temu",
         "orders_path": "/mnt/AmazonReports/amazon-orders",

download-shipping-labels.js

@@ -0,0 +1,368 @@
+const puppeteer = require("puppeteer");
+const axios = require("axios");
+const luxon = require("luxon");
+const { exit } = require("process");
+const fs = require("fs");
+const path = require("path");
+const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
+
+const utils = require("./utils");
+
+(async function () {
+  console.log(
+    `===========< STARTED --- DOWNLOAD SHIPPING LABELS  ${luxon.DateTime.now()} >=========`
+  );
+
+  const syncDate = luxon.DateTime.now().toFormat("yyyy-MM-dd");
+
+  /**
+   * loading config data
+   */
+  const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
+  const environment = process.env["ENVIRONMENT"];
+  const cryptoConfig = utils.getCryptoConfig();
+  let rates = [];
+
+  const email = utils.decryptString(
+    process.env["temu-email"],
+    cryptoConfig.algo,
+    cryptoConfig.key,
+    cryptoConfig.iv
+  );
+  const password = utils.decryptString(
+    process.env["temu-password"],
+    cryptoConfig.algo,
+    cryptoConfig.key,
+    cryptoConfig.iv
+  );
+
+  /*
+   * load cookies
+   */
+  const loadPageCookies = async function (page) {
+    const cookiesFileName = `cookies.json`;
+    if (fs.existsSync(__dirname + `/cookies/${cookiesFileName}`)) {
+      const cookiesStr = fs.readFileSync(
+        __dirname + `/cookies/${cookiesFileName}`
+      );
+      const cookies = JSON.parse(cookiesStr);
+      await page.setCookie(...cookies);
+    }
+  };
+
+  // launch browser and open page
+  const chromeProfilePath = path.resolve(
+    __dirname,
+    config[environment]["chrome_profile_path"]
+  );
+  const browser = await puppeteer.launch(
+    utils.getBrowserConfig(chromeProfilePath, environment)
+  );
+  const page = await browser.newPage();
+  await loadPageCookies(page);
+  await page.setViewport({
+    width: 1600,
+    height: 900,
+  });
+
+  // Inject CSS to show the cursor
+  await page.evaluate(() => {
+    const style = document.createElement("style");
+    style.innerHTML = "* { cursor: auto !important; }";
+    document.head.appendChild(style);
+  });
+
+  // save cookies on page load
+  const cookiesFileName = `cookies.json`;
+  page.on("load", async function () {
+    // save cookies
+    const cookies = await page.cookies();
+    fs.writeFileSync(
+      __dirname + `/cookies/${cookiesFileName}`,
+      JSON.stringify(cookies, null, 2)
+    );
+  });
+
+  /*
+   * goto login page
+   */
+  const loginPage = config[environment]["temuLoginPage"];
+  await page.goto(loginPage, {
+    waitUntil: ["domcontentloaded"],
+  });
+
+  await utils.tryTemuLogin(page, email, password, loginPage);
+  await new Promise((resolve) => setTimeout(resolve, 7000));
+
+  // goto orders request page
+  const shippedOrdersRequestPage = config[environment]["temuShippedOrdersPage"];
+  await page.goto(shippedOrdersRequestPage, {
+    waitUntil: ["domcontentloaded"],
+  });
+
+  /*
+   * configure download path of excel report
+   */
+  const generateTheDownloadPath = async function (page, downloadPath) {
+    // create client session for configuration
+    const client = await page.target().createCDPSession();
+    // set the download path
+    await client.send("Page.setDownloadBehavior", {
+      behavior: "allow",
+      downloadPath: downloadPath, // Change this to your desired download path.
+    });
+  };
+
+  /*
+   *  data
+   */
+  const pagination = 100;
+  let total_items = 0;
+  let currentPage = 1;
+  let maxPage = 10;
+
+  /**
+   * Capture response
+   */
+  const checkShippingLabelResponse = async (page, timer) => {
+    return new Promise((resolve, reject) => {
+      // Timeout mechanism to resolve with an empty list after 5 seconds
+      const timeout = setTimeout(() => {
+        page.off("response", handleResponse); // Remove listener on timeout
+        resolve([]); // Resolve with an empty list
+      }, timer);
+
+      const handleResponse = async (res) => {
+        try {
+          const req = res.request();
+          if (req.url().includes("/batch_print_shipping_label")) {
+            const resJson = await res.json();
+            // Remove listener and clear timeout once response is captured
+            clearTimeout(timeout);
+            page.off("response", handleResponse);
+            resolve(resJson || []);
+          }
+        } catch (ex) {
+          // Remove listener and clear timeout on error
+          clearTimeout(timeout);
+          page.off("response", handleResponse);
+          reject(ex);
+        }
+      };
+
+      page.on("response", handleResponse);
+    });
+  };
+
+  const downloadFolderPath = path.resolve(
+    config[environment].temu_order_shipping_labels_path,
+    `unprocessed/`
+  );
+
+  const processedFolderPath = path.resolve(
+    config[environment].temu_order_shipping_labels_path,
+    `processed/`
+  );
+
+
+  /*
+   * download labels
+   */
+  const downloadAndUploadShippingLabels = async (page) => {
+    try {
+      let bIndex = 0;
+      // PO Selector
+      const orderPOSelector = "div._3GLf87F3";
+      const orderPoList = await page.$$(orderPOSelector);
+
+      // select all print buttons
+      const buttonSelector = "div._37Oz7wUV";
+      const buttons = await page.$$(buttonSelector);
+      for (const button of buttons) {
+        try {
+          const text = await page.evaluate(
+            (el) => el.textContent.trim(),
+            button
+          );
+          if (text === "Print documents") {
+            console.log(`Button : ${bIndex}`);
+            await button.click(); // Click the button
+            await new Promise((resolve) => setTimeout(resolve, 2000));
+            try {
+              // check modal apears
+              await page.waitForSelector("div._1sNAa0oS.r4S9S0F2", {
+                timeout: 3000,
+              });
+            } catch (ex) {
+              console.log(`label Not present for Index : ${bIndex}`);
+              bIndex++;
+              continue;
+            }
+            await new Promise((resolve) => setTimeout(resolve, 2000));
+            const responsePromise = checkShippingLabelResponse(page, 5_000);
+
+            // click on download document tab
+            const downloadDocumentSelector =
+              "body > div.MDL_outerWrapper_123.MDL_modal_123 > div > div > div > div.MDL_body_123.MDL_noHeader_123 > div._1sNAa0oS.r4S9S0F2 > div._1zajAtWr > div > div > div.TAB_topWrapper_123.TAB_lineWrapper_123.TAB_topLineBelow_123.TAB_top_123 > div > div:nth-child(2) > div > span";
+            await page.waitForSelector(downloadDocumentSelector);
+            await page.click(downloadDocumentSelector);
+            await new Promise((resolve) => setTimeout(resolve, 1_000));
+
+            // click on radio buttin
+            const radioButtonSelector =
+              "#downloadMethod > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > label.RD_outerWrapper_123.RDG_OrOuterWrapper_123.RDG_radio_123.RDG_large_123.RDG_active_123.bc-HocRadio_outerWrapper_123.RD_large_123.RD_active_123.undefined.bc-HocRadio_active_123.RD_radio_123 > div.RD_textWrapper_123.RD_prevRadio_123";
+            await page.waitForSelector(radioButtonSelector);
+            await page.click(downloadDocumentSelector);
+            await new Promise((resolve) => setTimeout(resolve, 1_000));
+
+            // click on select
+            // const selectSelector =
+            //   "#downLoadPreferredName > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div > div > div > div > div > div.IPT_inputBlockCell_123.ST_inputBlockCell_123";
+            // await page.waitForSelector(selectSelector);
+            // await page.click(selectSelector);
+            // await new Promise((resolve) => setTimeout(resolve, 1_000));
+
+            // // click on order option
+            // const optionSelector =
+            //   "body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_largeDropdown_123.ST_customItem_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > ul > div > div > li.cIL_item_123.cIL_large_123.cIL_active_123.ST_itemRendererLabel_123.ST_itemRendererLabelChecked_123 > div > span";
+            // await page.waitForSelector(optionSelector);
+            // await page.click(optionSelector);
+            // await new Promise((resolve) => setTimeout(resolve, 1_000));
+            generateTheDownloadPath(page, downloadFolderPath);
+            // click on Print button
+            const downloadDocumentBtnSelector =
+              "body > div.MDL_outerWrapper_123.MDL_modal_123 > div > div > div > div.MDL_body_123.MDL_noHeader_123 > div._1sNAa0oS.r4S9S0F2 > div.gpBGty5E._4PfVmBF- > div > div > div._2bhVasgL > div._3yOxLjm0._2pgGmJ7w._1eT_m6dA._2IBCXPHH > span._2ISpB3A2";
+            await page.waitForSelector(downloadDocumentBtnSelector);
+            // click on add dimension
+            await page.click(downloadDocumentBtnSelector);
+            console.log(`Click on download Document Btn`);
+
+            await new Promise((resolve) => setTimeout(resolve, 1_000));
+
+            // listen for api call
+            let labelResponse = await responsePromise;
+
+            // close modal
+            await new Promise((resolve) => setTimeout(resolve, 2_000));
+            const closeBtnSelector = "div._2hT9O4-z.Y9rEd1VR";
+            await page.waitForSelector(closeBtnSelector);
+            await page.click(closeBtnSelector);
+
+            console.log(bIndex);
+            const orderNumber = await page.evaluate(
+              (el) => el.textContent.trim(),
+              orderPoList[bIndex]
+            );
+            // Rename the file after download
+            const files = fs.readdirSync(downloadFolderPath);
+            console.log(`Order Number : ${orderNumber}`);
+            if (files.length > 0) {
+              const downloadedFilePath = path.join(
+                downloadFolderPath,
+                files[files.length - 1]
+              );
+              const renamedFilePath = path.join(
+                processedFolderPath,
+                `${orderNumber}.pdf`
+              );
+              fs.renameSync(downloadedFilePath, renamedFilePath);
+              console.log(`File downloaded and renamed to: ${renamedFilePath}`);
+            } else {
+              console.log("No file downloaded.");
+            }
+            bIndex++;
+            // extract the url and save in s3
+            // await s3Utils.downloadAndUploadToS3(labelUrl, poNumber);
+          }
+        } catch (e) {
+          console.log(e);
+        }
+      }
+    } catch (e) {
+      console.log(e);
+    }
+  };
+
+
+  try {
+    // get total items
+    await page
+      .waitForSelector("li.PGT_totalText_123", { timeout: 5000 })
+      .catch(() => {});
+    const liText = await page.evaluate(() => {
+      const liElement = document.querySelector("li.PGT_totalText_123");
+      return liElement ? liElement.textContent : null;
+    });
+
+    if (liText === null) {
+      total_items = 10;
+    } else {
+      total_items = parseInt(liText.split(" ")[1]);
+      console.log(`Total Items count : ${total_items}`);
+    }
+
+    let total_pages = Math.ceil(total_items / pagination);
+    console.log(`Total Pages count : ${total_pages}`);
+    // crawl next pages
+    while (true) {
+      try {
+        console.log(`Crawling for page ${currentPage}`);
+
+        await utils.tryTemuLogin(page, email, password, loginPage);
+        await new Promise((resolve) => setTimeout(resolve, 4_000));
+
+        // load cookies
+        await loadPageCookies(page);
+
+        // get labels from page
+        await downloadAndUploadShippingLabels(page);
+        await new Promise((resolve) => setTimeout(resolve, 20_000));
+
+        // increment page
+        ++currentPage;
+
+        // Evaluate the presence of both classes in the <li> element
+        const hasNextBtn = await page.evaluate(() => {
+          const liElement = document.querySelector(
+            "li.PGT_next_123.PGT_disabled_123"
+          );
+          return liElement == null;
+        });
+
+        // break if doesn't have next button
+        if (!hasNextBtn) {
+          console.log("No next button");
+          break;
+        }
+
+        if (currentPage > maxPage || currentPage > total_pages) {
+          console.log("Last Page Reached");
+          break;
+        }
+
+        // goto next page
+        if (hasNextBtn) {
+          await page.evaluate(() => {
+            const liElement = document.querySelector("li.PGT_next_123");
+            if (liElement) {
+              liElement.click();
+            }
+          });
+        }
+
+        // wait
+        await new Promise((r) => setTimeout(r, 5000));
+      } catch (e) {
+        console.log(e);
+      }
+    }
+  } catch (e) {
+    console.log(e);
+  }
+
+  console.log(
+    `==========< ENDED --- DOWNLOAD SHIPPING LABELS  ${luxon.DateTime.now()}  >==========`
+  );
+  await page.close();
+  await browser.close();
+})();

package.json

@@ -10,6 +10,8 @@
     "author": "",
     "license": "ISC",
     "dependencies": {
+        "@aws-sdk/client-s3": "^3.734.0",
+        "@aws-sdk/lib-storage": "^3.734.0",
         "axios": "^0.27.2",
         "base32-encode": "^2.0.0",
         "dotenv": "^16.0.3",

s3-service.js

@@ -0,0 +1,58 @@
+const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3");
+const { Upload } = require("@aws-sdk/lib-storage");
+const axios = require("axios");
+const fs = require("fs");
+
+// AWS S3 configuration
+const REGION = process.env["aws-region"];
+const BUCKET_NAME = process.env["aws-s3-bucket-name"];
+const ACCESS_KEY = process.env["s3-access-key"];
+const SECRET_KEY = process.env["s3-secret-key"];
+
+// Create an S3 client
+const s3Client = new S3Client({
+  region: REGION,
+  credentials: {
+    accessKeyId: ACCESS_KEY,
+    secretAccessKey: SECRET_KEY,
+  },
+});
+
+// Function to upload a file stream to S3
+const saveToS3 = async (fileStream, key) => {
+  try {
+    const uploadParams = {
+      Bucket: BUCKET_NAME,
+      Key: key,
+      Body: fileStream,
+      ContentType: "application/pdf",
+    };
+
+    const upload = new Upload({
+      client: s3Client,
+      params: uploadParams,
+    });
+
+    const result = await upload.done();
+    return result;
+  } catch (error) {
+    console.error("Error uploading to S3:", error.message);
+    throw error;
+  }
+};
+
+/*
+ * download and save to s3
+ */
+const downloadAndUploadToS3 = async (filePath, s3Key) => {
+  try {
+    const fileContent = fs.readFileSync(filePath);
+    const result = await saveToS3(fileContent, s3Key);
+    console.log(`${s3Key} File successfully uploaded to S3`);
+    return result;
+  } catch (error) {
+    console.error("Error in download and upload process:", error.message);
+  }
+};
+
+exports.downloadAndUploadToS3 = downloadAndUploadToS3;

send-shipping-labels-s3.js

@@ -0,0 +1,49 @@
+const axios = require("axios");
+const fs = require("fs");
+const path = require("path");
+const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
+const { exit } = require("process");
+const s3Utils = require("./s3-service");
+
+(async function () {
+  /**
+   * load config data
+   */
+  const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
+  const environment = process.env["ENVIRONMENT"];
+
+  /**
+   * directory path
+   */
+  let shippingLabelsPath = config[environment].temu_order_shipping_labels_path;
+  let processedPath = shippingLabelsPath + "/processed";
+  let syncedPath = shippingLabelsPath + "/synced";
+
+  if (!fs.existsSync(syncedPath)) {
+    fs.mkdirSync(syncedPath);
+  }
+
+  /**
+   * read all files in directory, send data to s3 then move to synced
+   */
+  const pdfFiles = fs
+    .readdirSync(processedPath)
+    .filter((file) => path.extname(file).toLocaleLowerCase() === ".pdf");
+
+  if (pdfFiles.length === 0) {
+    console.log(`No Files Present at ${processedPath}`);
+  }
+  for (const file of pdfFiles) {
+    try {
+      const filePath = path.join(processedPath, file);
+      console.log(`Processing: ${filePath}`);
+      // send pdf files to s3
+      const result = await s3Utils.downloadAndUploadToS3(filePath, file );
+      if (result['$metadata']['httpStatusCode'] == 200) {
+        fs.renameSync(filePath, path.join(syncedPath, file));
+      }
+    } catch (e) {
+      console.log(e);
+    }
+  }
+})();

utils.js

@@ -55,14 +55,16 @@ const getBrowserConfig = function (profilePath, env) {
       "--disable-setuid-sandbox",
       "--no-sandbox",
       "--no-zygote",
-      `--user-data-dir=${profilePath}`,
+      "--incognito",
     ];
+    //  `--user-data-dir=${profilePath}`,
     browserConfig["executablePath"] = "/usr/bin/chromium-browser";
   }
   // dev specific
   if (env === "dev") {
     browserConfig["headless"] = false;
-    browserConfig["args"] = [`--user-data-dir=${profilePath}`];
+    browserConfig["args"] =  ['--incognito']
+   // browserConfig["args"] = [`--user-data-dir=${profilePath}`];
   }
   return browserConfig;
 };