/
GMapScrape.js
72 lines (57 loc) · 1.75 KB
/
GMapScrape.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
const { fetch } = require("./fetch");
const { appendJson, readJson } = require("./writer");
const { BASE_URL, SEARCH_PREFIX, SEARCHES } = require("./env");
const RESULTS = [];
const NEXT_PAGES = []; // readJson("tokens-2021-08-05T16-54-33-331Z.json")["results"];
const DONE_PAGES = new Set();
const jsonFileName = new Date()
.toISOString()
.replace(/\W+/g, "-")
.concat(".json");
class GMapScrape {
do() {
SEARCHES.forEach((s) => {
return this.getPlacesSearch(s);
});
setTimeout(() => {
while (NEXT_PAGES.length) {
const x = this.getPlacesNext(NEXT_PAGES[0]);
}
console.log(jsonFileName);
}, 3000);
}
getPlacesSearch(search) {
const query = SEARCH_PREFIX.concat(search);
console.log(`🚀 > getPlacesSearch > query`, query);
const place = fetch(
BASE_URL.concat("&query=").concat(encodeURIComponent(query))
);
this.addResults(place);
}
getPlacesNext(pagetoken) {
console.log(`🚀 > getPlacesNext`, pagetoken.substr(-5), NEXT_PAGES.length);
if (DONE_PAGES.has(pagetoken)) {
NEXT_PAGES.shift();
return;
}
DONE_PAGES.add(pagetoken);
const place = fetch(BASE_URL.concat("&pagetoken=").concat(pagetoken));
this.addResults(place, true);
}
addResults(place, removeFirst = false) {
if (place) {
if ("next_page_token" in place && place.next_page_token) {
NEXT_PAGES.push(place.next_page_token);
appendJson("tokens-" + jsonFileName, place.next_page_token);
}
if ("results" in place && place.results.length) {
RESULTS.push([...place.results]);
appendJson(jsonFileName, place.results);
if (removeFirst) {
NEXT_PAGES.shift();
}
}
}
}
}
module.exports = { GMapScrape };