feat: scraping expo/react-native version from npm

expo-react-native.json

@@ -0,0 +1,37 @@
+{
+  "expoSdkVersions": [
+    "45.0.0",
+    "44.0.0",
+    "43.0.0"
+  ],
+  "reactNativeVersions": [
+    "0.68.2",
+    "0.68.1",
+    "0.68.0",
+    "0.67.4",
+    "0.67.3",
+    "0.67.2",
+    "0.67.1",
+    "0.67.0",
+    "0.66.4",
+    "0.66.3",
+    "0.64.3",
+    "0.66.2",
+    "0.66.1",
+    "0.66.0",
+    "0.65.1",
+    "0.65.0",
+    "0.64.2",
+    "0.64.1",
+    "0.62.3",
+    "0.64.0",
+    "0.63.4",
+    "0.63.3",
+    "0.63.2",
+    "0.63.1",
+    "0.63.0",
+    "0.62.2",
+    "0.62.1",
+    "0.62.0"
+  ]
+}

package.json

@@ -32,6 +32,7 @@
     "lodash": "^4.17.21",
     "next": "^12.1.5",
     "node-emoji": "^1.11.0",
+    "npm-registry-fetch": "^13.1.1",
     "react": "17.0.2",
     "react-content-loader": "^6.2.0",
     "react-dom": "17.0.2",
@@ -42,6 +43,7 @@
     "react-native-web": "^0.17.7",
     "react-native-web-hooks": "^3.0.2",
     "react-popper": "^2.3.0",
+    "semver": "^7.3.7",
     "use-debounce": "^7.0.1"
   },
   "devDependencies": {

scripts/build-and-score-data.js

@@ -2,13 +2,15 @@ import fs from 'fs';
 import jsonfile from 'jsonfile';
 import chunk from 'lodash/chunk';
 import path from 'path';
+import semver from 'semver';
 
 import debugGithubRepos from '../debug-github-repos.json';
+import dataVersions from '../expo-react-native.json';
 import githubRepos from '../react-native-libraries.json';
 import * as Strings from '../util/strings';
 import { calculateDirectoryScore, calculatePopularityScore } from './calculate-score';
 import { fetchGithubData, fetchGithubRateLimit, loadGitHubLicenses } from './fetch-github-data';
-import { fetchNpmData, fetchNpmDataBulk } from './fetch-npm-data';
+import { fetchNpmPackageJSON, fetchNpmData, fetchNpmDataBulk } from './fetch-npm-data';
 import fetchReadmeImages from './fetch-readme-images';
 
 // Uses debug-github-repos.json instead, so we have less repositories to crunch
@@ -43,6 +45,50 @@ const fillNpmName = project => {
 
 const invalidRepos = [];
 
+function checkVersion(data) {
+  const { npmPackage, version } = data;
+  return project => {
+    const pkgJSON = project.npm?.packageJSON ?? {};
+    const requiredVersion =
+      pkgJSON.dependencies?.[npmPackage] ?? pkgJSON.peerDependencies?.[npmPackage];
+    return requiredVersion ? semver.satisfies(version, requiredVersion) : false;
+  };
+}
+
+function buildPackageData(npmPackage, versions, data) {
+  return versions
+    .map(version => {
+      const filter = checkVersion({ npmPackage, version });
+
+      const packages = data.filter(filter).map(({ npmPkg: npmPackage, npm }) => ({
+        npmPackage,
+        versionRange: npm.packageJSON?.version,
+      }));
+      return { version, packages };
+    })
+    .map(async ({ version, packages }) => {
+      const dir = path.resolve('assets', npmPackage, version);
+      await fs.promises.mkdir(dir, {
+        recursive: true,
+      });
+
+      return new Promise((resolve, reject) => {
+        jsonfile.writeFile(
+          path.resolve(dir, 'data.json'),
+          { data: packages },
+          JSON_OPTIONS,
+          err => {
+            if (err) {
+              reject(err);
+            } else {
+              resolve();
+            }
+          }
+        );
+      });
+    });
+}
+
 const buildAndScoreData = async () => {
   console.log('** Loading data from GitHub');
   let data = await loadRepositoryDataAsync();
@@ -107,6 +153,22 @@ const buildAndScoreData = async () => {
         }
   );
 
+  console.log('\n** Fetch NPM Package JSON');
+  data = await Promise.all(
+    data.map(async project => {
+      const { npmPkg } = project;
+      const packageJSON = await fetchNpmPackageJSON(npmPkg);
+      return {
+        ...project,
+        npm: {
+          ...(project.npm || {}),
+          packageJSON,
+        },
+      };
+    })
+  );
+  await Promise.all(buildPackageData('react-native', dataVersions.reactNativeVersions, data));
+
   console.log('\n** Calculating Directory Score');
   data = data.map(project => {
     try {

scripts/fetch-npm-data.js

@@ -1,4 +1,5 @@
 import fetch from 'cross-fetch';
+import npmFetch from 'npm-registry-fetch';
 
 import { sleep } from './build-and-score-data';
 
@@ -44,6 +45,42 @@ export const fetchNpmDataBulk = async (namesArray, period = 'month', attemptsCou
   }
 };
 
+const interestedOfDependencies = ['expo', 'react-native'];
+
+const buildMiniPackageJSON = packageJson => {
+  const result = {
+    name: packageJson.name,
+    version: packageJson.version,
+  };
+  ['dependencies', 'peerDependencies'].forEach(depType => {
+    if (packageJson[depType]) {
+      result[depType] = {};
+      interestedOfDependencies.map(dep => {
+        const version = packageJson[depType][dep];
+        if (version) {
+          result[depType][dep] = version;
+        }
+      });
+    }
+  });
+
+  return result;
+};
+
+export const fetchNpmPackageJSON = async npmPackage => {
+  const manifest = await npmFetch.json(npmPackage);
+
+  const latestVersion = manifest?.['dist-tags']?.['latest'];
+  if (latestVersion) {
+    const packageJSON = manifest.versions[latestVersion];
+    if (packageJSON) {
+      return buildMiniPackageJSON(packageJSON);
+    }
+  }
+
+  return null;
+};
+
 export const fetchNpmData = async (pkgData, attemptsCount = 0) => {
   const { npmPkg } = pkgData;