-
Notifications
You must be signed in to change notification settings - Fork 1
/
point_scraper.js
55 lines (40 loc) · 1.09 KB
/
point_scraper.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
var scraper = require('scraper');
var fs = require('fs');
var base = 'http://www.bwca.net/entry-points/?page=';
var writeFile = function(entryPoints) {
console.log(entryPoints)
fs.writeFile('entry_points.json', JSON.stringify(entryPoints), function (err) {
if (err) {
console.error(err);
} else {
console.log('Wrote entry points');
}
});
};
pages = ['1', '2', '3', '4'];
urls = [];
pages.forEach(function(page) {
urls.push(base + page);
});
console.log(urls)
entryPoints = {};
count = 0;
urls.forEach(function(url) {
scraper(url, function(err, jQuery) {
count++;
if(err) {
console.error(err);
return;
}
console.log('Got Url: ' + url);
jQuery('.jrContentTitle a').each(function(index, object) {
console.log(object);
var name = object.innerHTML;
var url = object.href;
entryPoints[name] = {'url': url};
});
if(count === 4) {
writeFile(entryPoints);
}
});
});