I am pretty new to this Jquery extraction. My problem right now is to make the links I extracted automatically clicked through code in Chrome console. I used the code below to get the link in the href, but the click(),trigger('click') function doesn't work in this situation. Can someone give some suggestions? Thanks in advance
$('.agent-info').find('a').href.trigger('click')
Use $(".agent-info a").trigger("click"); instead. This is a simple example:
$(document).on("click", ".agent-info a", function(){
$(this).text("It works!");
});
$(document).ready(function(){
$(".agent-info a").trigger("click");
});<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<div class="agent-info">
<a href="#">Click me 1</a>
<a href="#">Click me 2</a>
<a href="#">Click me 3</a>
</div>
window.location.href = $(this).attr('href'); in click event. @IrisZhanfurequest module to request these links and get body, then use cheerio to extract data. Of course Not in Browser! I have some prepared codes for that. I put them in another answer.@IrisZhanfuYou can use request and cheerio modules in NodeJs to load page contents and get all bodies.
let request = require('request'),
cheerio = require('cheerio'),
q = require('q')
You need a function which get url and load the page
function loadPage(url) {
var deferred = q.defer();
request.get({
url: url,
timeout: 60000
},
function (error, response, body) {
if (!error) {
deferred.resolve(body);
}
else {
deferred.reject(error);
}
}
);
return deferred.promise;
}
You need to find all links in the body. For that use a function like this
function extractLinks(body) {
var deferred = q.defer(),
$ = cheerio.load(body),
links = [];
$(/*Your selector*/).each(function() {
// add links to links array
});
deferred.resolve(links);
return deferred.promise;
}
Then you must again load body of each item in links array. So use the loadPage module again. And a function using cheerio to extract your data of new pages.
loadPage(firstUrl)
.then(function(result){
extractLinks(result)
.then(function(res){
var arr = [];
for(var r in res) {
arr.push(loadPage(res[r]));
q.all(arr)
.then()
.catch();
}
});
})
.catch(function(error) {
console.log(error);
});
$('.agent-info').find('a').trigger('click')