Psw crawler #1
10
config.json
10
config.json
|
|
@ -23,7 +23,10 @@
|
||||||
"temuShippedOrdersPage" : "https://seller.temu.com/orders.html?activeTab=4",
|
"temuShippedOrdersPage" : "https://seller.temu.com/orders.html?activeTab=4",
|
||||||
"temuOrderPage" : "https://seller.temu.com/order-detail.html?parent_order_sn=",
|
"temuOrderPage" : "https://seller.temu.com/order-detail.html?parent_order_sn=",
|
||||||
"temuOrderReportPage" : "https://seller.temu.com/order-reports.html",
|
"temuOrderReportPage" : "https://seller.temu.com/order-reports.html",
|
||||||
"temuUnshippedOrdersUrl" : "https://cosmos-jobs.utopiadeals.com/cosmos/temu/get-orders-by-status?status=Unshipped"
|
"temuUnshippedOrdersUrl" : "https://cosmos-jobs.utopiadeals.com/cosmos/temu/get-orders-by-status?status=Unshipped",
|
||||||
|
"psw_fi_invoices_path" : "D:/Frontend/temu-labels-crawler/fi-invoice-crawler",
|
||||||
|
"uind_psw_financial_invoices" : "http://192.168.90.188:8086/uind/psw/fi-invoices-upload-json",
|
||||||
|
"psw_url": "https://weboc.gov.pk/(S(hqaenkukmimyaagggrvvbiai))/Shared/FormE_Attachment_LookUp.aspx?t=1&URID=131380&CUT=1&TID=27472728&BID=4"
|
||||||
},
|
},
|
||||||
"prod": {
|
"prod": {
|
||||||
"temu_order_shipping_labels_path" : "/mnt/AmazonReports/Temu/shipping_labels",
|
"temu_order_shipping_labels_path" : "/mnt/AmazonReports/Temu/shipping_labels",
|
||||||
|
|
@ -45,6 +48,9 @@
|
||||||
"temuShippedOrdersPage" : "https://seller.temu.com/orders.html?activeTab=4",
|
"temuShippedOrdersPage" : "https://seller.temu.com/orders.html?activeTab=4",
|
||||||
"temuOrderPage" : "https://seller.temu.com/order-detail.html?parent_order_sn=",
|
"temuOrderPage" : "https://seller.temu.com/order-detail.html?parent_order_sn=",
|
||||||
"temuOrderReportPage" : "https://seller.temu.com/order-reports.html",
|
"temuOrderReportPage" : "https://seller.temu.com/order-reports.html",
|
||||||
"temuUnshippedOrdersUrl" : "https://cosmos-jobs.utopiadeals.com/cosmos/temu/get-orders-by-status?status=Unshipped"
|
"temuUnshippedOrdersUrl" : "https://cosmos-jobs.utopiadeals.com/cosmos/temu/get-orders-by-status?status=Unshipped",
|
||||||
|
"psw_fi_invoices_path" : "/mnt/AmazonReports/psw",
|
||||||
|
"uind_psw_financial_invoices" : "https://portal.utopiaindustries.pk/uind/psw/fi-invoices-upload-json",
|
||||||
|
"psw_url": "https://weboc.gov.pk/(S(hqaenkukmimyaagggrvvbiai))/Shared/FormE_Attachment_LookUp.aspx?t=1&URID=131380&CUT=1&TID=27472728&BID=4"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
const axios = require("axios");
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
|
||||||
|
const { exit } = require("process");
|
||||||
|
|
||||||
|
(async function () {
|
||||||
|
/**
|
||||||
|
* load config data
|
||||||
|
*/
|
||||||
|
const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
|
||||||
|
const environment = process.env["ENVIRONMENT"];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* directory path
|
||||||
|
*/
|
||||||
|
let ordersPath = config[environment].psw_fi_invoices_path;
|
||||||
|
let unProcessedPath = ordersPath + "/data/unprocessed";
|
||||||
|
let processedPath = ordersPath + "/data/processed";
|
||||||
|
|
||||||
|
if (!fs.existsSync(processedPath)) {
|
||||||
|
fs.mkdirSync(processedPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* read all files in directory, send data to uind then move to processed
|
||||||
|
*/
|
||||||
|
const jsonFiles = fs
|
||||||
|
.readdirSync(unProcessedPath)
|
||||||
|
.filter((file) => path.extname(file).toLocaleLowerCase() === ".json");
|
||||||
|
|
||||||
|
if( jsonFiles.length === 0 ){
|
||||||
|
console.log( `No Files Present at ${unProcessedPath}`)
|
||||||
|
}
|
||||||
|
for (const file of jsonFiles) {
|
||||||
|
try {
|
||||||
|
const filePath = path.join(unProcessedPath, file);
|
||||||
|
const orders = JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||||
|
console.log(`Processing: ${filePath}`);
|
||||||
|
// send post request to uind
|
||||||
|
const axiosConfig = {
|
||||||
|
method: "get",
|
||||||
|
url: config[environment].uind_psw_financial_invoices,
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
data: orders,
|
||||||
|
};
|
||||||
|
|
||||||
|
const res = await axios(axiosConfig);
|
||||||
|
if (res["status"] == 200) {
|
||||||
|
fs.renameSync(filePath, path.join(processedPath, file));
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
console.log(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
@ -0,0 +1,79 @@
|
||||||
|
const puppeteer = require("puppeteer");
|
||||||
|
const luxon = require("luxon");
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const dotenv = require("dotenv").config({ path: __dirname + "/.env" });
|
||||||
|
|
||||||
|
const utils = require("./utils");
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`===========< STARTED ${utils.getPakistanStandardTime(
|
||||||
|
luxon.DateTime.now()
|
||||||
|
)} >=========`
|
||||||
|
);
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* loading config data
|
||||||
|
*/
|
||||||
|
const config = JSON.parse(fs.readFileSync(__dirname + "/config.json"));
|
||||||
|
const environment = process.env["ENVIRONMENT"];
|
||||||
|
|
||||||
|
|
||||||
|
const browser = await puppeteer.launch({
|
||||||
|
headless: "new",
|
||||||
|
args: ["--no-sandbox", "--disable-setuid-sandbox"]
|
||||||
|
});
|
||||||
|
|
||||||
|
const page = await browser.newPage();
|
||||||
|
|
||||||
|
await page.goto(
|
||||||
|
config[environment].psw_url,
|
||||||
|
{
|
||||||
|
waitUntil: "domcontentloaded",
|
||||||
|
timeout: 0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
await page.waitForSelector("tbody tr.ItemStyle");
|
||||||
|
|
||||||
|
const data = await page.evaluate(() => {
|
||||||
|
const rows = document.querySelectorAll(
|
||||||
|
"tbody tr.ItemStyle, tbody tr.AlternatingItemStyle"
|
||||||
|
);
|
||||||
|
|
||||||
|
return Array.from(rows).map(row => {
|
||||||
|
const cells = row.querySelectorAll("td");
|
||||||
|
|
||||||
|
const finNo =
|
||||||
|
cells[0]?.querySelector("label")?.innerText.trim() ||
|
||||||
|
cells[0]?.innerText.trim();
|
||||||
|
|
||||||
|
return {
|
||||||
|
financialInstrumentNo: finNo,
|
||||||
|
date: cells[1]?.innerText.trim(),
|
||||||
|
authorizedDealer: cells[2]?.innerText.trim(),
|
||||||
|
consigneeName: cells[3]?.innerText.trim(),
|
||||||
|
invoiceValue: cells[4]?.innerText.trim(),
|
||||||
|
currency: cells[6]?.innerText.trim(),
|
||||||
|
status: cells[7]?.innerText.trim(),
|
||||||
|
balance: cells[8]?.innerText.trim()
|
||||||
|
};
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const jsonDir = path.join(config[environment].psw_fi_invoices_path, "data/unprocessed");
|
||||||
|
|
||||||
|
fs.mkdirSync(jsonDir, { recursive: true });
|
||||||
|
|
||||||
|
const outputFile = path.join(jsonDir, `${luxon.DateTime.now().toMillis()}.json`);
|
||||||
|
fs.writeFileSync(outputFile, JSON.stringify(data, null, 2));
|
||||||
|
console.log(`Saved JSON to ${outputFile}`);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
await page.close();
|
||||||
|
await browser.close();
|
||||||
|
})();
|
||||||
Loading…
Reference in New Issue