minor chnages
parent
d1b07c5eae
commit
4cdad303a4
|
@ -220,6 +220,7 @@ const utils = require("./utils");
|
|||
);
|
||||
if (text === "Print documents") {
|
||||
console.log(`Button : ${bIndex}`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
await button.click(); // Click the button
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
try {
|
||||
|
@ -338,49 +339,8 @@ const utils = require("./utils");
|
|||
|
||||
while (true) {
|
||||
try {
|
||||
// crawl next pages
|
||||
const currentStatePage = JSON.parse(
|
||||
fs.readFileSync(__dirname + "/state.json")
|
||||
).last_page;
|
||||
if (currentStatePage > currentPage) {
|
||||
console.log(`Moving from ${currentPage} to ${currentStatePage}`);
|
||||
const hasNextBtn = await page.evaluate(() => {
|
||||
const liElement = document.querySelector(
|
||||
"li.PGT_next_123.PGT_disabled_123"
|
||||
);
|
||||
return liElement == null;
|
||||
});
|
||||
|
||||
// break if doesn't have next button
|
||||
if (!hasNextBtn) {
|
||||
console.log("No next button");
|
||||
break;
|
||||
}
|
||||
|
||||
if (currentPage > maxPage || currentPage > total_pages) {
|
||||
console.log("Last Page Reached");
|
||||
break;
|
||||
}
|
||||
|
||||
// goto next page
|
||||
if (hasNextBtn) {
|
||||
await page.evaluate(async () => {
|
||||
const liElement = document.querySelector("li.PGT_next_123");
|
||||
await new Promise((r) => setTimeout(r, 3000));
|
||||
if (liElement) {
|
||||
liElement.click();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
currentPage++;
|
||||
// wait
|
||||
continue;
|
||||
}
|
||||
|
||||
// update state
|
||||
await updateState(currentPage, luxon.DateTime.now().toISO());
|
||||
|
||||
// await updateState(currentPage, luxon.DateTime.now().toISO());
|
||||
console.log(`Crawling for page ${currentPage}`);
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 2_000));
|
||||
|
|
Loading…
Reference in New Issue