-
Notifications
You must be signed in to change notification settings - Fork 54
/
scraper.mjs
296 lines (269 loc) 路 9.52 KB
/
scraper.mjs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
import Agent from 'socks5-http-client/lib/Agent.js';
import fetch from 'node-fetch';
import crypto from 'crypto';
import lzma from 'lzma';
import fs from 'node:fs/promises';
import cheerio from 'cheerio';
import { Generator as RelicGenerator } from '@wfcd/relics';
import patchlogs from 'warframe-patchlogs';
import Progress from './progress.mjs';
import ModScraper from './wikia/scrapers/ModScraper.mjs';
import WeaponScraper from './wikia/scrapers/WeaponScraper.mjs';
import WarframeScraper from './wikia/scrapers/WarframeScraper.mjs';
import VersionScraper from './wikia/scrapers/VersionScraper.mjs';
import readJson from './readJson.mjs';
const exportCache = await readJson(new URL('../data/cache/.export.json', import.meta.url));
const locales = await readJson(new URL('../config/locales.json', import.meta.url));
const prod = process.env.NODE_ENV === 'production';
// eslint-disable-next-line no-control-regex
const sanitize = (str) => str.replace(/\\r|\r?\n|\x09/g, '').replace(/\\\\"/g, "'");
const agent = process.env.SOCK5_HOST
? new Agent({
socksHost: process.env.SOCKS5_HOST,
socksPort: process.env.SOCKS5_PORT,
socksUsername: process.env.SOCKS5_USER,
socksPassword: process.env.SOCKS5_PASS,
})
: undefined;
const get = async (url, disableProxy = !prod, compress = false) => {
const res = await fetch(url, {
agent: disableProxy ? undefined : agent,
});
return compress === false ? Uint8Array.from(await res.buffer()) : res.text();
};
const getJSON = async (url, disableProxy) => JSON.parse(sanitize(await get(url, disableProxy, true)));
/**
* Retrieves the base item data necessary for the parsing process
*/
class Scraper {
/**
* Get Endpoints from Warframe's origin file
* @param {boolean} [manifest] to fetch only the manifest or everything else
* @param {string} [locale] Locale to fetch data for
*/
async fetchEndpoints(manifest, locale) {
const origin = `https://content.warframe.com/PublicExport/index_${locale || 'en'}.txt.lzma`;
const raw = await get(origin, !prod);
const decompressed = lzma.decompress(raw);
const manifestRegex = /(\r?\n)?ExportManifest.*/;
// We either don't need the manifest, or *only* the manifest
if (manifest) {
return manifestRegex.exec(decompressed)[0].replace(/\r?\n/, '');
}
return decompressed.replace(manifestRegex, '').split(/\r?\n/g);
}
/**
* Fetch Warframe API data, split up by endpoint.
*/
async fetchResources() {
const endpoints = await this.fetchEndpoints();
const result = [];
const i18nEndpoints = {};
await Promise.all(
locales.map(async (locale) => {
i18nEndpoints[locale] = await this.fetchEndpoints(false, locale);
})
);
const totalEndpoints =
i18nEndpoints[Object.keys(i18nEndpoints)[0]].length * Object.keys(i18nEndpoints).length + endpoints.length;
const bar = new Progress('Fetching API Endpoints', totalEndpoints);
const fetchEndpoint = async (endpoint) => {
const category = endpoint.replace('Export', '').replace(/_[a-z]{2}\.json.*/, '');
const raw = await getJSON(`https://content.warframe.com/PublicExport/Manifest/${endpoint}`);
const data = raw ? raw[`Export${category}`] : undefined;
bar.tick();
if (category === 'Upgrades') {
const modSets = raw.ExportModSet.map((modSet) => ({
...modSet,
type: 'Mod Set',
}));
data.push(...modSets, ...raw.ExportAvionics, ...raw.ExportFocusUpgrades);
}
return { category, data };
};
await Promise.all(
endpoints.map(async (endpoint) => {
result.push(await fetchEndpoint(endpoint));
})
);
const i18n = {
en: result,
};
// Request i18n sequentially by locale to avoid getting randomly stuck in some computers
// It is roughly the same speed as the "all async" method but is always successfull
for (let i = 0; i < locales.length; i += 1) {
const locale = locales[i];
i18n[locale] = [];
await Promise.all(
i18nEndpoints[locale].map(async (endpoint) => {
i18n[locale].push(await fetchEndpoint(endpoint));
})
);
}
return i18n;
}
/**
* @typedef {Object} ImageManifestItem
* @property {string} uniqueName unique string
* identifier corresponding to a {@link module:warframe-items.Item.uniqueName|uniqueName}
* @property {string} textureLocation location of the image for the corresponding item
*/
/**
* @typedef {Object} ImageManifest
* @property {ImageManifestItem[]} Manifest
*/
/**
* Get the manifest of all available images.
* @returns {ImageManifest.Manifest}
*/
async fetchImageManifest() {
const bar = new Progress('Fetching Image Manifest', 1);
const endpoint = await this.fetchEndpoints(true);
const manifest = (await getJSON(`https://content.warframe.com/PublicExport/Manifest/${endpoint}`)).Manifest;
bar.tick();
return manifest;
}
/**
* Get official drop rates and check if they changed since last build.
* @returns {DropData}
*/
async fetchDropRates() {
const bar = new Progress('Fetching Drop Rates', 1);
const rates = await getJSON('https://drops.warframestat.us/data/all.slim.json', true);
const ratesHash = crypto.createHash('md5').update(JSON.stringify(rates)).digest('hex');
const changed = exportCache.DropChances.hash !== ratesHash;
// Update checksum
if (changed) {
exportCache.DropChances.hash = ratesHash;
await fs.writeFile(
new URL('../data/cache/.export.json', import.meta.url),
JSON.stringify(exportCache, undefined, 1)
);
}
bar.tick();
return {
rates,
changed,
};
}
/**
* @typedef {Object} PatchlogWrap
* @property {module:warframe-patchlogs.Patchlogs} patchlogs Warframe patchlogs
* @property {boolean} changed whether or not there's an update
*/
/**
* Get patchlogs from the forums
* @returns {PatchlogWrap}
*/
async fetchPatchLogs() {
const bar = new Progress('Fetching Patchlogs', 1);
const patchlogsHash = crypto.createHash('md5').update(JSON.stringify(patchlogs.posts)).digest('hex');
const changed = exportCache.Patchlogs.hash !== patchlogsHash;
if (changed) {
exportCache.Patchlogs.hash = patchlogsHash;
await fs.writeFile(
new URL('../data/cache/.export.json', import.meta.url),
JSON.stringify(exportCache, undefined, 1)
);
}
bar.tick();
return {
patchlogs,
changed,
};
}
/**
* @typedef {Object} WikiaData
* @property {Array<WikiaWeapon>} weapons
* @property {Array<WikiaWarframe>} warframes
* @property {Array<WikiaMods>} mods
* @property {Array<WikiaVersions>} versions
* @property {Array<WikiaDucats>} ducats
*/
/**
* Get additional data from wikia if it's not provided in the API
* @returns {WikiaData}
*/
async fetchWikiaData() {
const bar = new Progress('Fetching Wikia Data', 5);
const ducats = [];
const ducatsWikia = await get('http://warframe.wikia.com/wiki/Ducats/Prices/All');
const $ = cheerio.load(ducatsWikia);
$('.mw-content-text table tbody tr').each(function () {
const name = $(this).find('td:nth-of-type(1) a:nth-of-type(2)').text();
const value = $(this).find('td:nth-of-type(3)').attr('data-sort-value');
ducats.push({ name, ducats: Number.parseInt(value, 10) });
});
bar.tick();
const weapons = await new WeaponScraper().scrape();
bar.tick();
const warframes = await new WarframeScraper().scrape();
bar.tick();
const mods = await new ModScraper().scrape();
bar.tick();
const versions = await new VersionScraper().scrape();
bar.tick();
return {
weapons,
warframes,
mods,
versions,
ducats,
};
}
/**
* Formatted date string. Format: "YYYY MM DD"
* @typedef {string} OggDateStamp
*/
/**
* @typedef {Object} OggRelic
* @property {string<module:warframe-items.Rarity>} Rarity
* @property {string} Name relic name, format: "%RelicEra% %Identifier%"
* @property {boolean} Vaulted whether or not the relic is vaulted
*/
/**
* @typedef {Object} OggComponent
* @property {string} Name component name
* @property {Array<OggRelic>} Relics relics that can drop this component
* @property {number} Ducats sell price in ducats
*/
/**
* @typedef {Object} VaultDataItem
* @property {boolean} Vaulted whether the item is vaulted
* @property {string<OggDateStamp>} ReleaseDate Ogg-date-formatted date stamp
* @property {Array<OggComponent>} Components
* @property {string<OggDateStamp>} EstimatedVaultedDate estimated date for the vault date
* @property {string<OggDateStamp>} VaultedDate actual vault date
*/
/**
* @typedef {Object} VaultData
* @property {{code: number}} metadata
* @property {Array<VaultDataItem>} data
*/
/**
* Get (estimated) vault dates from ducats or plat.
* @returns {VaultData}
*/
async fetchVaultData() {
const bar = new Progress('Fetching Vault Data', 1);
const vaultData = (await getJSON('http://www.oggtechnologies.com/api/ducatsorplat/v2/MainItemData.json')).data;
bar.tick();
return vaultData;
}
/**
* Generate Relic Data from Titania Project
* @returns {Promise<Array<TitaniaRelic>>}
*/
async generateRelicData() {
const bar = new Progress('Generating Relic Data', 1);
const relicGenerator = new RelicGenerator();
try {
const relicData = await relicGenerator.generate();
bar.tick();
return relicData;
} catch (e) {
console.error(e);
}
}
}
export default new Scraper();