I've written a Node.js program that fetches a webpage url every X seconds, and compare the previous and new value of a specific html element. It will only save changes to a specific output file.
Note that the previous value record will be deleted after each run of this program, meaning that the first time you run this program it will always save the extracted text ( Because there's nothing to compare to )
This program uses node-fetch and jsdom npm packages.
fs is a build in package for Node.js.
If you are new to Node.js, you can follow this to install in your computer.
const fetch = require('node-fetch');
const jsdom = require('jsdom');
const fs = require('fs');
// Local previous extracted text variable to compare and determine changes
let prevExtractedText;
// The webpage URL to fetch from
const url = 'https://en.wikipedia.org/wiki/Node.js';
// Setting your file's output path
const outputFilepath = 'myfile.txt';
// Setting timeout every 5 seconds
const timeout = 5000;
const handleOnError = (err) => {
console.error(`! An error occurred: ${err.message}`);
process.exit(1);
}
const handleFetchAndSaveFile = async () => {
let html;
try {
console.log(`* Fetching ${url}...`);
const resp = await fetch(url);
console.log('* Converting response into html text...');
html = await resp.text();
} catch (err) {
handleOnError(err);
}
// Convert into DOM in Node.js enviroment
const dom = new jsdom.JSDOM(html);
// Example with element of id "footer-places-privacy"
const extractedText = dom.window.document.getElementById("footer-places-privacy").textContent;
console.log(`* Comparing previous extracted text (${prevExtractedText}) and current extracted text (${extractedText})`);
if (prevExtractedText !== extractedText) {
// Update prevExtractedText
prevExtractedText = extractedText;
console.log(`* Updating ${outputFilepath}...`);
try {
// Writing new extracted text into a file
await fs.appendFileSync(outputFilepath, extractedText);
console.log(`* ${outputFilepath} has been updated and saved.`);
} catch (err) {
handleOnError(err);
}
}
console.log('--------------------------------------------------')
}
console.log(`* Polling ${url} every ${timeout}ms`);
setInterval(handleFetchAndSaveFile, timeout);
Working demo: https://codesandbox.io/s/nodejs-webpage-polling-jqf6v?fontsize=14&hidenavigation=1&theme=dark