add label download functionality
parent
7144790a80
commit
4637342688
|
|
@ -1,5 +1,6 @@
|
|||
{
|
||||
"dev": {
|
||||
"temu_order_shipping_labels_path" : "D:/Projects/Crawlers/temu-label-crawler/shipping_labels",
|
||||
"temu_orders_shipping_rates" : "D:/Projects/Crawlers/temu-label-crawler/shipping_rates",
|
||||
"temu_orders_path" : "D:/Projects/Crawlers/temu-label-crawler",
|
||||
"orders_path": "D:/Projects/Crawlers/amazon_crawler/data/orders",
|
||||
|
|
@ -25,6 +26,7 @@
|
|||
"temuUnshippedOrdersUrl" : "https://cosmos.utopiadeals.com/cosmos/temu/get-orders-by-status?status=Unshipped"
|
||||
},
|
||||
"prod": {
|
||||
"temu_order_shipping_labels_path" : "/mnt/AmazonReports/Temu/shipping_labels",
|
||||
"temu_orders_shipping_rates" : "/mnt/AmazonReports/Temu/shipping_rates",
|
||||
"temu_orders_path" : "/mnt/AmazonReports/Temu",
|
||||
"orders_path": "/mnt/AmazonReports/amazon-orders",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,368 @@
|
|||
const puppeteer = require("puppeteer");
|
||||
const axios = require("axios");
|
||||
const luxon = require("luxon");
|
||||
const { exit } = require("process");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
|
||||
|
||||
const utils = require("./utils");
|
||||
|
||||
(async function () {
|
||||
console.log(
|
||||
`===========< STARTED --- DOWNLOAD SHIPPING LABELS ${luxon.DateTime.now()} >=========`
|
||||
);
|
||||
|
||||
const syncDate = luxon.DateTime.now().toFormat("yyyy-MM-dd");
|
||||
|
||||
/**
|
||||
* loading config data
|
||||
*/
|
||||
const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
|
||||
const environment = process.env["ENVIRONMENT"];
|
||||
const cryptoConfig = utils.getCryptoConfig();
|
||||
let rates = [];
|
||||
|
||||
const email = utils.decryptString(
|
||||
process.env["temu-email"],
|
||||
cryptoConfig.algo,
|
||||
cryptoConfig.key,
|
||||
cryptoConfig.iv
|
||||
);
|
||||
const password = utils.decryptString(
|
||||
process.env["temu-password"],
|
||||
cryptoConfig.algo,
|
||||
cryptoConfig.key,
|
||||
cryptoConfig.iv
|
||||
);
|
||||
|
||||
/*
|
||||
* load cookies
|
||||
*/
|
||||
const loadPageCookies = async function (page) {
|
||||
const cookiesFileName = `cookies.json`;
|
||||
if (fs.existsSync(__dirname + `/cookies/${cookiesFileName}`)) {
|
||||
const cookiesStr = fs.readFileSync(
|
||||
__dirname + `/cookies/${cookiesFileName}`
|
||||
);
|
||||
const cookies = JSON.parse(cookiesStr);
|
||||
await page.setCookie(...cookies);
|
||||
}
|
||||
};
|
||||
|
||||
// launch browser and open page
|
||||
const chromeProfilePath = path.resolve(
|
||||
__dirname,
|
||||
config[environment]["chrome_profile_path"]
|
||||
);
|
||||
const browser = await puppeteer.launch(
|
||||
utils.getBrowserConfig(chromeProfilePath, environment)
|
||||
);
|
||||
const page = await browser.newPage();
|
||||
await loadPageCookies(page);
|
||||
await page.setViewport({
|
||||
width: 1600,
|
||||
height: 900,
|
||||
});
|
||||
|
||||
// Inject CSS to show the cursor
|
||||
await page.evaluate(() => {
|
||||
const style = document.createElement("style");
|
||||
style.innerHTML = "* { cursor: auto !important; }";
|
||||
document.head.appendChild(style);
|
||||
});
|
||||
|
||||
// save cookies on page load
|
||||
const cookiesFileName = `cookies.json`;
|
||||
page.on("load", async function () {
|
||||
// save cookies
|
||||
const cookies = await page.cookies();
|
||||
fs.writeFileSync(
|
||||
__dirname + `/cookies/${cookiesFileName}`,
|
||||
JSON.stringify(cookies, null, 2)
|
||||
);
|
||||
});
|
||||
|
||||
/*
|
||||
* goto login page
|
||||
*/
|
||||
const loginPage = config[environment]["temuLoginPage"];
|
||||
await page.goto(loginPage, {
|
||||
waitUntil: ["domcontentloaded"],
|
||||
});
|
||||
|
||||
await utils.tryTemuLogin(page, email, password, loginPage);
|
||||
await new Promise((resolve) => setTimeout(resolve, 7000));
|
||||
|
||||
// goto orders request page
|
||||
const shippedOrdersRequestPage = config[environment]["temuShippedOrdersPage"];
|
||||
await page.goto(shippedOrdersRequestPage, {
|
||||
waitUntil: ["domcontentloaded"],
|
||||
});
|
||||
|
||||
/*
|
||||
* configure download path of excel report
|
||||
*/
|
||||
const generateTheDownloadPath = async function (page, downloadPath) {
|
||||
// create client session for configuration
|
||||
const client = await page.target().createCDPSession();
|
||||
// set the download path
|
||||
await client.send("Page.setDownloadBehavior", {
|
||||
behavior: "allow",
|
||||
downloadPath: downloadPath, // Change this to your desired download path.
|
||||
});
|
||||
};
|
||||
|
||||
/*
|
||||
* data
|
||||
*/
|
||||
const pagination = 100;
|
||||
let total_items = 0;
|
||||
let currentPage = 1;
|
||||
let maxPage = 10;
|
||||
|
||||
/**
|
||||
* Capture response
|
||||
*/
|
||||
const checkShippingLabelResponse = async (page, timer) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Timeout mechanism to resolve with an empty list after 5 seconds
|
||||
const timeout = setTimeout(() => {
|
||||
page.off("response", handleResponse); // Remove listener on timeout
|
||||
resolve([]); // Resolve with an empty list
|
||||
}, timer);
|
||||
|
||||
const handleResponse = async (res) => {
|
||||
try {
|
||||
const req = res.request();
|
||||
if (req.url().includes("/batch_print_shipping_label")) {
|
||||
const resJson = await res.json();
|
||||
// Remove listener and clear timeout once response is captured
|
||||
clearTimeout(timeout);
|
||||
page.off("response", handleResponse);
|
||||
resolve(resJson || []);
|
||||
}
|
||||
} catch (ex) {
|
||||
// Remove listener and clear timeout on error
|
||||
clearTimeout(timeout);
|
||||
page.off("response", handleResponse);
|
||||
reject(ex);
|
||||
}
|
||||
};
|
||||
|
||||
page.on("response", handleResponse);
|
||||
});
|
||||
};
|
||||
|
||||
const downloadFolderPath = path.resolve(
|
||||
config[environment].temu_order_shipping_labels_path,
|
||||
`unprocessed/`
|
||||
);
|
||||
|
||||
const processedFolderPath = path.resolve(
|
||||
config[environment].temu_order_shipping_labels_path,
|
||||
`processed/`
|
||||
);
|
||||
|
||||
|
||||
/*
|
||||
* download labels
|
||||
*/
|
||||
const downloadAndUploadShippingLabels = async (page) => {
|
||||
try {
|
||||
let bIndex = 0;
|
||||
// PO Selector
|
||||
const orderPOSelector = "div._3GLf87F3";
|
||||
const orderPoList = await page.$$(orderPOSelector);
|
||||
|
||||
// select all print buttons
|
||||
const buttonSelector = "div._37Oz7wUV";
|
||||
const buttons = await page.$$(buttonSelector);
|
||||
for (const button of buttons) {
|
||||
try {
|
||||
const text = await page.evaluate(
|
||||
(el) => el.textContent.trim(),
|
||||
button
|
||||
);
|
||||
if (text === "Print documents") {
|
||||
console.log(`Button : ${bIndex}`);
|
||||
await button.click(); // Click the button
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
try {
|
||||
// check modal apears
|
||||
await page.waitForSelector("div._1sNAa0oS.r4S9S0F2", {
|
||||
timeout: 3000,
|
||||
});
|
||||
} catch (ex) {
|
||||
console.log(`label Not present for Index : ${bIndex}`);
|
||||
bIndex++;
|
||||
continue;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
const responsePromise = checkShippingLabelResponse(page, 5_000);
|
||||
|
||||
// click on download document tab
|
||||
const downloadDocumentSelector =
|
||||
"body > div.MDL_outerWrapper_123.MDL_modal_123 > div > div > div > div.MDL_body_123.MDL_noHeader_123 > div._1sNAa0oS.r4S9S0F2 > div._1zajAtWr > div > div > div.TAB_topWrapper_123.TAB_lineWrapper_123.TAB_topLineBelow_123.TAB_top_123 > div > div:nth-child(2) > div > span";
|
||||
await page.waitForSelector(downloadDocumentSelector);
|
||||
await page.click(downloadDocumentSelector);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
|
||||
// click on radio buttin
|
||||
const radioButtonSelector =
|
||||
"#downloadMethod > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > label.RD_outerWrapper_123.RDG_OrOuterWrapper_123.RDG_radio_123.RDG_large_123.RDG_active_123.bc-HocRadio_outerWrapper_123.RD_large_123.RD_active_123.undefined.bc-HocRadio_active_123.RD_radio_123 > div.RD_textWrapper_123.RD_prevRadio_123";
|
||||
await page.waitForSelector(radioButtonSelector);
|
||||
await page.click(downloadDocumentSelector);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
|
||||
// click on select
|
||||
// const selectSelector =
|
||||
// "#downLoadPreferredName > div > div.Grid_row_123.Grid_rowHorizontal_123.Grid_rowJustifyStart_123.Form_itemContent_123.Form_itemContentCenter_123 > div > div > div > div > div > div > div > div > div.IPT_inputBlockCell_123.ST_inputBlockCell_123";
|
||||
// await page.waitForSelector(selectSelector);
|
||||
// await page.click(selectSelector);
|
||||
// await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
|
||||
// // click on order option
|
||||
// const optionSelector =
|
||||
// "body > div.PT_outerWrapper_123.PP_outerWrapper_123.ST_dropdown_123.ST_largeDropdown_123.ST_customItem_123.PT_dropdown_123.PT_portalBottomLeft_123.PT_inCustom_123.PP_dropdown_123 > div > div > div > div > ul > div > div > li.cIL_item_123.cIL_large_123.cIL_active_123.ST_itemRendererLabel_123.ST_itemRendererLabelChecked_123 > div > span";
|
||||
// await page.waitForSelector(optionSelector);
|
||||
// await page.click(optionSelector);
|
||||
// await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
generateTheDownloadPath(page, downloadFolderPath);
|
||||
// click on Print button
|
||||
const downloadDocumentBtnSelector =
|
||||
"body > div.MDL_outerWrapper_123.MDL_modal_123 > div > div > div > div.MDL_body_123.MDL_noHeader_123 > div._1sNAa0oS.r4S9S0F2 > div.gpBGty5E._4PfVmBF- > div > div > div._2bhVasgL > div._3yOxLjm0._2pgGmJ7w._1eT_m6dA._2IBCXPHH > span._2ISpB3A2";
|
||||
await page.waitForSelector(downloadDocumentBtnSelector);
|
||||
// click on add dimension
|
||||
await page.click(downloadDocumentBtnSelector);
|
||||
console.log(`Click on download Document Btn`);
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
|
||||
// listen for api call
|
||||
let labelResponse = await responsePromise;
|
||||
|
||||
// close modal
|
||||
await new Promise((resolve) => setTimeout(resolve, 2_000));
|
||||
const closeBtnSelector = "div._2hT9O4-z.Y9rEd1VR";
|
||||
await page.waitForSelector(closeBtnSelector);
|
||||
await page.click(closeBtnSelector);
|
||||
|
||||
console.log(bIndex);
|
||||
const orderNumber = await page.evaluate(
|
||||
(el) => el.textContent.trim(),
|
||||
orderPoList[bIndex]
|
||||
);
|
||||
// Rename the file after download
|
||||
const files = fs.readdirSync(downloadFolderPath);
|
||||
console.log(`Order Number : ${orderNumber}`);
|
||||
if (files.length > 0) {
|
||||
const downloadedFilePath = path.join(
|
||||
downloadFolderPath,
|
||||
files[files.length - 1]
|
||||
);
|
||||
const renamedFilePath = path.join(
|
||||
processedFolderPath,
|
||||
`${orderNumber}.pdf`
|
||||
);
|
||||
fs.renameSync(downloadedFilePath, renamedFilePath);
|
||||
console.log(`File downloaded and renamed to: ${renamedFilePath}`);
|
||||
} else {
|
||||
console.log("No file downloaded.");
|
||||
}
|
||||
bIndex++;
|
||||
// extract the url and save in s3
|
||||
// await s3Utils.downloadAndUploadToS3(labelUrl, poNumber);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
try {
|
||||
// get total items
|
||||
await page
|
||||
.waitForSelector("li.PGT_totalText_123", { timeout: 5000 })
|
||||
.catch(() => {});
|
||||
const liText = await page.evaluate(() => {
|
||||
const liElement = document.querySelector("li.PGT_totalText_123");
|
||||
return liElement ? liElement.textContent : null;
|
||||
});
|
||||
|
||||
if (liText === null) {
|
||||
total_items = 10;
|
||||
} else {
|
||||
total_items = parseInt(liText.split(" ")[1]);
|
||||
console.log(`Total Items count : ${total_items}`);
|
||||
}
|
||||
|
||||
let total_pages = Math.ceil(total_items / pagination);
|
||||
console.log(`Total Pages count : ${total_pages}`);
|
||||
// crawl next pages
|
||||
while (true) {
|
||||
try {
|
||||
console.log(`Crawling for page ${currentPage}`);
|
||||
|
||||
await utils.tryTemuLogin(page, email, password, loginPage);
|
||||
await new Promise((resolve) => setTimeout(resolve, 4_000));
|
||||
|
||||
// load cookies
|
||||
await loadPageCookies(page);
|
||||
|
||||
// get labels from page
|
||||
await downloadAndUploadShippingLabels(page);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20_000));
|
||||
|
||||
// increment page
|
||||
++currentPage;
|
||||
|
||||
// Evaluate the presence of both classes in the <li> element
|
||||
const hasNextBtn = await page.evaluate(() => {
|
||||
const liElement = document.querySelector(
|
||||
"li.PGT_next_123.PGT_disabled_123"
|
||||
);
|
||||
return liElement == null;
|
||||
});
|
||||
|
||||
// break if doesn't have next button
|
||||
if (!hasNextBtn) {
|
||||
console.log("No next button");
|
||||
break;
|
||||
}
|
||||
|
||||
if (currentPage > maxPage || currentPage > total_pages) {
|
||||
console.log("Last Page Reached");
|
||||
break;
|
||||
}
|
||||
|
||||
// goto next page
|
||||
if (hasNextBtn) {
|
||||
await page.evaluate(() => {
|
||||
const liElement = document.querySelector("li.PGT_next_123");
|
||||
if (liElement) {
|
||||
liElement.click();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// wait
|
||||
await new Promise((r) => setTimeout(r, 5000));
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`==========< ENDED --- DOWNLOAD SHIPPING LABELS ${luxon.DateTime.now()} >==========`
|
||||
);
|
||||
await page.close();
|
||||
await browser.close();
|
||||
})();
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -10,6 +10,8 @@
|
|||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.734.0",
|
||||
"@aws-sdk/lib-storage": "^3.734.0",
|
||||
"axios": "^0.27.2",
|
||||
"base32-encode": "^2.0.0",
|
||||
"dotenv": "^16.0.3",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3");
|
||||
const { Upload } = require("@aws-sdk/lib-storage");
|
||||
const axios = require("axios");
|
||||
const fs = require("fs");
|
||||
|
||||
// AWS S3 configuration
|
||||
const REGION = process.env["aws-region"];
|
||||
const BUCKET_NAME = process.env["aws-s3-bucket-name"];
|
||||
const ACCESS_KEY = process.env["s3-access-key"];
|
||||
const SECRET_KEY = process.env["s3-secret-key"];
|
||||
|
||||
// Create an S3 client
|
||||
const s3Client = new S3Client({
|
||||
region: REGION,
|
||||
credentials: {
|
||||
accessKeyId: ACCESS_KEY,
|
||||
secretAccessKey: SECRET_KEY,
|
||||
},
|
||||
});
|
||||
|
||||
// Function to upload a file stream to S3
|
||||
const saveToS3 = async (fileStream, key) => {
|
||||
try {
|
||||
const uploadParams = {
|
||||
Bucket: BUCKET_NAME,
|
||||
Key: key,
|
||||
Body: fileStream,
|
||||
ContentType: "application/pdf",
|
||||
};
|
||||
|
||||
const upload = new Upload({
|
||||
client: s3Client,
|
||||
params: uploadParams,
|
||||
});
|
||||
|
||||
const result = await upload.done();
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error("Error uploading to S3:", error.message);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
* download and save to s3
|
||||
*/
|
||||
const downloadAndUploadToS3 = async (filePath, s3Key) => {
|
||||
try {
|
||||
const fileContent = fs.readFileSync(filePath);
|
||||
const result = await saveToS3(fileContent, s3Key);
|
||||
console.log(`${s3Key} File successfully uploaded to S3`);
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error("Error in download and upload process:", error.message);
|
||||
}
|
||||
};
|
||||
|
||||
exports.downloadAndUploadToS3 = downloadAndUploadToS3;
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
const axios = require("axios");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
|
||||
const { exit } = require("process");
|
||||
const s3Utils = require("./s3-service");
|
||||
|
||||
(async function () {
|
||||
/**
|
||||
* load config data
|
||||
*/
|
||||
const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
|
||||
const environment = process.env["ENVIRONMENT"];
|
||||
|
||||
/**
|
||||
* directory path
|
||||
*/
|
||||
let shippingLabelsPath = config[environment].temu_order_shipping_labels_path;
|
||||
let processedPath = shippingLabelsPath + "/processed";
|
||||
let syncedPath = shippingLabelsPath + "/synced";
|
||||
|
||||
if (!fs.existsSync(syncedPath)) {
|
||||
fs.mkdirSync(syncedPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* read all files in directory, send data to s3 then move to synced
|
||||
*/
|
||||
const pdfFiles = fs
|
||||
.readdirSync(processedPath)
|
||||
.filter((file) => path.extname(file).toLocaleLowerCase() === ".pdf");
|
||||
|
||||
if (pdfFiles.length === 0) {
|
||||
console.log(`No Files Present at ${processedPath}`);
|
||||
}
|
||||
for (const file of pdfFiles) {
|
||||
try {
|
||||
const filePath = path.join(processedPath, file);
|
||||
console.log(`Processing: ${filePath}`);
|
||||
// send pdf files to s3
|
||||
const result = await s3Utils.downloadAndUploadToS3(filePath, file );
|
||||
if (result['$metadata']['httpStatusCode'] == 200) {
|
||||
fs.renameSync(filePath, path.join(syncedPath, file));
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
}
|
||||
}
|
||||
})();
|
||||
6
utils.js
6
utils.js
|
|
@ -55,14 +55,16 @@ const getBrowserConfig = function (profilePath, env) {
|
|||
"--disable-setuid-sandbox",
|
||||
"--no-sandbox",
|
||||
"--no-zygote",
|
||||
`--user-data-dir=${profilePath}`,
|
||||
"--incognito",
|
||||
];
|
||||
// `--user-data-dir=${profilePath}`,
|
||||
browserConfig["executablePath"] = "/usr/bin/chromium-browser";
|
||||
}
|
||||
// dev specific
|
||||
if (env === "dev") {
|
||||
browserConfig["headless"] = false;
|
||||
browserConfig["args"] = [`--user-data-dir=${profilePath}`];
|
||||
browserConfig["args"] = ['--incognito']
|
||||
// browserConfig["args"] = [`--user-data-dir=${profilePath}`];
|
||||
}
|
||||
return browserConfig;
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue