keyword_ranking_crawler/send-data.js

56 lines
1.7 KiB
JavaScript

const axios = require( 'axios' );
const fs = require( 'fs' );
const path = require( 'path' );
(async function() {
/**
* directory path
*/
let rootPath = '/mnt/AmazonReports/Amazon/keyword_ranking';
let processedPath = rootPath + '/processed';
if ( ! fs.existsSync( rootPath ) ) {
fs.mkdirSync( rootPath );
}
if ( ! fs.existsSync( processedPath ) ) {
fs.mkdirSync( processedPath );
}
/**
* read all files in directory, send data to cosmos then move to processed
*/
const jsonFiles = fs.readdirSync(rootPath)
.filter(file => path.extname(file).toLowerCase() === '.json')
.map(file => {
const filePath = path.join(rootPath, file);
const stats = fs.statSync(filePath);
return { file, birthtime: stats.birthtime };
})
.sort((a, b) => a.birthtime - b.birthtime) // ASCENDING order
.map(entry => entry.file); // extract filenames
for ( const file of jsonFiles ) {
try {
// read contents of the file
const filePath = path.join( rootPath, file );
const orders = JSON.parse( fs.readFileSync( filePath, 'utf-8' ) );
let payload = { progressList: orders };
console.log( `Processing: ${filePath}` );
// send post request to cosmos
let res = await axios.post( config[environment].cosmos_path_orders_progress, payload, {
headers: {
'Content-Type': 'application/json'
}
} );
if ( res['status'] == 200 ) {
fs.renameSync( filePath, path.join( processedPath, file ) )
}
} catch ( e ) {
console.log( e );
}
}
})();