saif.haq
/
temu-labels-crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

increase timeouts

main
saif 2025-02-10 13:14:00 +05:00
parent efd4451fc2
commit 5dda4ff239
1 changed files with 18 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
sync-orders.js
View File

@ -222,31 +222,19 @@ const utils = require("./utils");
// Function to read Excel files and convert to JSON // Function to read Excel files and convert to JSON
const convertExcelToJson = async () => { const convertExcelToJson = async () => {
fs.readdir(unprocessedFolderPath, (err, files) => { try {
if (err) { const files = await fs.readdir(unprocessedFolderPath);
console.error("Error reading the directory:", err);
return;
}
// Filter Excel files (xlsx or xls)
const excelFiles = files.filter( const excelFiles = files.filter(
(file) => (file) => file.endsWith(".xlsx") || file.endsWith(".xls") || file.endsWith(".csv")
file.endsWith(".xlsx") ||
file.endsWith(".xls") ||
file.endsWith(".csv")
); );
// Process each Excel file for (const file of excelFiles) {
excelFiles.forEach((file) => {
const filePath = path.join(unprocessedFolderPath, file); const filePath = path.join(unprocessedFolderPath, file);
const workbook = xlsx.readFile(filePath); const workbook = xlsx.readFile(filePath);
const sheetName = workbook.SheetNames[0]; // Use the first sheet const sheetName = workbook.SheetNames[0];
const worksheet = workbook.Sheets[sheetName]; const worksheet = workbook.Sheets[sheetName];
// Convert the sheet to JSON
let jsonData = xlsx.utils.sheet_to_json(worksheet); let jsonData = xlsx.utils.sheet_to_json(worksheet);
// Modify the column names by adding underscores
jsonData = jsonData.map((row) => { jsonData = jsonData.map((row) => {
const modifiedRow = {}; const modifiedRow = {};
Object.keys(row).forEach((key) => { Object.keys(row).forEach((key) => {
@ -260,24 +248,21 @@ const utils = require("./utils");
return modifiedRow; return modifiedRow;
}); });
// write the JSON data to a file
const outputFile = path.join( const outputFile = path.join(
config[environment].temu_orders_path, config.environment.temu_orders_path,
"data/unprocessed", "data/unprocessed",
`${luxon.DateTime.now().toMillis()}.json` `${luxon.DateTime.now().toMillis()}.json`
); );
fs.writeFileSync(outputFile, JSON.stringify(jsonData, null, 2)); await fs.writeFile(outputFile, JSON.stringify(jsonData, null, 2));
console.log(`Saved JSON to ${outputFile}`); console.log(`Saved JSON to ${outputFile}`);
console.log( const processedPath = path.join(config.environment.temu_orders_path, "processed", file);
`Move Excel file ${filePath} to ${config[environment].temu_orders_path}/processed` await fs.rename(filePath, processedPath);
); console.log(`Moved Excel file ${filePath} to ${processedPath}`);
fs.renameSync( }
filePath, } catch (err) {
path.join(`${config[environment].temu_orders_path}/processed`, file) console.error("Error processing Excel files:", err);
); }
});
});
}; };
await convertExcelToJson(); await convertExcelToJson();