default.js
+
+
+```js
+new Crawler({
+ appId: 'YOUR_APP_ID',
+ apiKey: 'YOUR_API_KEY',
+ indexPrefix: 'crawler_',
+ rateLimit: 8,
+ startUrls: ['https://YOUR_WEBSITE_URL'],
+ renderJavaScript: false,
+ sitemaps: [],
+ ignoreCanonicalTo: false,
+ discoveryPatterns: ['https://YOUR_WEBSITE_URL/**'],
+ actions: [
+ {
+ indexName: 'YOUR_INDEX_NAME',
+ pathsToMatch: ['https://YOUR_WEBSITE_URL/**'],
+ recordExtractor: ({ helpers }) => {
+ return helpers.docsearch({
+ recordProps: {
+ lvl1: ['header h1', 'article h1', 'main h1', 'h1', 'head > title'],
+ content: ['article p, article li', 'main p, main li', 'p, li'],
+ lvl0: {
+ selectors: '',
+ defaultValue: 'Documentation',
+ },
+ lvl2: ['article h2', 'main h2', 'h2'],
+ lvl3: ['article h3', 'main h3', 'h3'],
+ lvl4: ['article h4', 'main h4', 'h4'],
+ lvl5: ['article h5', 'main h5', 'h5'],
+ lvl6: ['article h6', 'main h6', 'h6'],
+ },
+ aggregateContent: true,
+ recordVersion: 'v3',
+ });
+ },
+ },
+ ],
+ initialIndexSettings: {
+ YOUR_INDEX_NAME: {
+ attributesForFaceting: ['type', 'lang'],
+ attributesToRetrieve: [
+ 'hierarchy',
+ 'content',
+ 'anchor',
+ 'url',
+ 'url_without_anchor',
+ 'type',
+ ],
+ attributesToHighlight: ['hierarchy', 'content'],
+ attributesToSnippet: ['content:10'],
+ camelCaseAttributes: ['hierarchy', 'content'],
+ searchableAttributes: [
+ 'unordered(hierarchy.lvl0)',
+ 'unordered(hierarchy.lvl1)',
+ 'unordered(hierarchy.lvl2)',
+ 'unordered(hierarchy.lvl3)',
+ 'unordered(hierarchy.lvl4)',
+ 'unordered(hierarchy.lvl5)',
+ 'unordered(hierarchy.lvl6)',
+ 'content',
+ ],
+ distinct: true,
+ attributeForDistinct: 'url',
+ customRanking: [
+ 'desc(weight.pageRank)',
+ 'desc(weight.level)',
+ 'asc(weight.position)',
+ ],
+ ranking: [
+ 'words',
+ 'filters',
+ 'typo',
+ 'attribute',
+ 'proximity',
+ 'exact',
+ 'custom',
+ ],
+ highlightPreTag: '',
+ highlightPostTag: '',
+ minWordSizefor1Typo: 3,
+ minWordSizefor2Typos: 7,
+ allowTyposOnNumericTokens: false,
+ minProximity: 1,
+ ignorePlurals: true,
+ advancedSyntax: true,
+ attributeCriteriaComputedByMinProximity: true,
+ removeWordsIfNoResults: 'allOptional',
+ },
+ },
+});
+```
+
+
+
+
## Docusaurus v1 template