bootstrap: make snapshot reproducible

src/api/environment.cc

@@ -458,7 +458,13 @@ Environment* CreateEnvironment(
   if (use_snapshot) {
     context = Context::FromSnapshot(isolate,
                                     SnapshotData::kNodeMainContextIndex,
-                                    {DeserializeNodeInternalFields, env})
+                                    v8::DeserializeInternalFieldsCallback(
+                                        DeserializeNodeInternalFields, env),
+                                    nullptr,
+                                    MaybeLocal<Value>(),
+                                    nullptr,
+                                    v8::DeserializeContextDataCallback(
+                                        DeserializeNodeContextData, env))
                   .ToLocalChecked();
 
     CHECK(!context.IsEmpty());

src/blob_serializer_deserializer-inl.h

@@ -238,7 +238,8 @@ size_t BlobSerializer<Impl>::WriteVector(const std::vector<T>& data) {
   if (is_debug) {
     std::string str = std::is_arithmetic_v<T> ? "" : ToStr(data);
     std::string name = GetName<T>();
-    Debug("\nWriteVector<%s>() (%d-byte), count=%d: %s\n",
+    Debug("\nAt 0x%x: WriteVector<%s>() (%d-byte), count=%d: %s\n",
+          sink.size(),
           name.c_str(),
           sizeof(T),
           data.size(),
@@ -270,7 +271,10 @@ size_t BlobSerializer<Impl>::WriteVector(const std::vector<T>& data) {
 template <typename Impl>
 size_t BlobSerializer<Impl>::WriteStringView(std::string_view data,
                                              StringLogMode mode) {
-  Debug("WriteStringView(), length=%zu: %p\n", data.size(), data.data());
+  Debug("At 0x%x: WriteStringView(), length=%zu: %p\n",
+        sink.size(),
+        data.size(),
+        data.data());
   size_t written_total = WriteArithmetic<size_t>(data.size());
 
   size_t length = data.size();
@@ -294,17 +298,27 @@ size_t BlobSerializer<Impl>::WriteString(const std::string& data) {
   return WriteStringView(data, StringLogMode::kAddressAndContent);
 }
 
+static size_t kPreviewCount = 16;
+
 // Helper for writing an array of numeric types.
 template <typename Impl>
 template <typename T>
 size_t BlobSerializer<Impl>::WriteArithmetic(const T* data, size_t count) {
   static_assert(std::is_arithmetic_v<T>, "Arithmetic type");
   DCHECK_GT(count, 0);  // Should not write contents for vectors of size 0.
   if (is_debug) {
-    std::string str =
-        "{ " + std::to_string(data[0]) + (count > 1 ? ", ... }" : " }");
+    size_t preview_count = count < kPreviewCount ? count : kPreviewCount;
+    std::string str = "{ ";
+    for (size_t i = 0; i < preview_count; ++i) {
+      str += (std::to_string(data[i]) + ",");
+    }
+    if (count > preview_count) {
+      str += "...";
+    }
+    str += "}";
     std::string name = GetName<T>();
-    Debug("Write<%s>() (%zu-byte), count=%zu: %s",
+    Debug("At 0x%x: Write<%s>() (%zu-byte), count=%zu: %s",
+          sink.size(),
           name.c_str(),
           sizeof(T),
           count,

src/node.cc

@@ -1279,18 +1279,24 @@ ExitCode GenerateAndWriteSnapshotData(const SnapshotData** snapshot_data_ptr,
       return exit_code;
     }
   } else {
+    std::optional<std::string> builder_script_content;
     // Otherwise, load and run the specified builder script.
     std::unique_ptr<SnapshotData> generated_data =
         std::make_unique<SnapshotData>();
-    std::string builder_script_content;
-    int r = ReadFileSync(&builder_script_content, builder_script.c_str());
-    if (r != 0) {
-      FPrintF(stderr,
-              "Cannot read builder script %s for building snapshot. %s: %s",
-              builder_script,
-              uv_err_name(r),
-              uv_strerror(r));
-      return ExitCode::kGenericUserError;
+    if (builder_script != "node:generate_default_snapshot") {
+      builder_script_content = std::string();
+      int r = ReadFileSync(&(builder_script_content.value()),
+                           builder_script.c_str());
+      if (r != 0) {
+        FPrintF(stderr,
+                "Cannot read builder script %s for building snapshot. %s: %s\n",
+                builder_script,
+                uv_err_name(r),
+                uv_strerror(r));
+        return ExitCode::kGenericUserError;
+      }
+    } else {
+      snapshot_config.builder_script_path = std::nullopt;
     }
 
     exit_code = node::SnapshotBuilder::Generate(generated_data.get(),

src/node_snapshotable.cc

@@ -603,7 +603,7 @@ std::vector<char> SnapshotData::ToBlob() const {
   written_total += w.WriteArithmetic<uint32_t>(kMagic);
   w.Debug("Write metadata\n");
   written_total += w.Write<SnapshotMetadata>(metadata);
-
+  w.Debug("Write snapshot blob\n");
   written_total += w.Write<v8::StartupData>(v8_snapshot_blob_data);
   w.Debug("Write isolate_data_indices\n");
   written_total += w.Write<IsolateDataSerializeInfo>(isolate_data_info);
@@ -1155,8 +1155,11 @@ ExitCode SnapshotBuilder::CreateSnapshot(SnapshotData* out,
     CHECK_EQ(index, SnapshotData::kNodeVMContextIndex);
     index = creator->AddContext(base_context);
     CHECK_EQ(index, SnapshotData::kNodeBaseContextIndex);
-    index = creator->AddContext(main_context,
-                                {SerializeNodeContextInternalFields, env});
+    index = creator->AddContext(
+        main_context,
+        v8::SerializeInternalFieldsCallback(SerializeNodeContextInternalFields,
+                                            env),
+        v8::SerializeContextDataCallback(SerializeNodeContextData, env));
     CHECK_EQ(index, SnapshotData::kNodeMainContextIndex);
   }
 
@@ -1255,6 +1258,17 @@ std::string SnapshotableObject::GetTypeName() const {
   }
 }
 
+void DeserializeNodeContextData(Local<Context> holder,
+                                int index,
+                                StartupData payload,
+                                void* callback_data) {
+  DCHECK(index == ContextEmbedderIndex::kEnvironment ||
+         index == ContextEmbedderIndex::kRealm ||
+         index == ContextEmbedderIndex::kContextTag);
+  // This is a no-op for now. We will reset all the pointers in
+  // Environment::AssignToContext() via the realm constructor.
+}
+
 void DeserializeNodeInternalFields(Local<Object> holder,
                                    int index,
                                    StartupData payload,
@@ -1320,6 +1334,44 @@ void DeserializeNodeInternalFields(Local<Object> holder,
   }
 }
 
+StartupData SerializeNodeContextData(Local<Context> holder,
+                                     int index,
+                                     void* callback_data) {
+  DCHECK(index == ContextEmbedderIndex::kEnvironment ||
+         index == ContextEmbedderIndex::kContextifyContext ||
+         index == ContextEmbedderIndex::kRealm ||
+         index == ContextEmbedderIndex::kContextTag);
+  void* data = holder->GetAlignedPointerFromEmbedderData(index);
+  per_process::Debug(DebugCategory::MKSNAPSHOT,
+                     "Serialize context data, index=%d, holder=%p, ptr=%p\n",
+                     static_cast<int>(index),
+                     *holder,
+                     data);
+  // Serialization of contextify context is not yet supported.
+  if (index == ContextEmbedderIndex::kContextifyContext) {
+    DCHECK_NULL(data);
+    return {nullptr, 0};
+  }
+
+  // We need to use use new[] because V8 calls delete[] on the returned data.
+  int size = sizeof(ContextEmbedderIndex);
+  char* result = new char[size];
+  ContextEmbedderIndex* index_data =
+      reinterpret_cast<ContextEmbedderIndex*>(result);
+  *index_data = static_cast<ContextEmbedderIndex>(index);
+
+  // For now we just reset all of them in Environment::AssignToContext()
+  switch (index) {
+    case ContextEmbedderIndex::kEnvironment:
+    case ContextEmbedderIndex::kContextifyContext:
+    case ContextEmbedderIndex::kRealm:
+    case ContextEmbedderIndex::kContextTag:
+      return StartupData{result, size};
+    default:
+      UNREACHABLE();
+  }
+}
+
 StartupData SerializeNodeContextInternalFields(Local<Object> holder,
                                                int index,
                                                void* callback_data) {

src/node_snapshotable.h

@@ -126,10 +126,17 @@ class SnapshotableObject : public BaseObject {
 v8::StartupData SerializeNodeContextInternalFields(v8::Local<v8::Object> holder,
                                                    int index,
                                                    void* env);
+v8::StartupData SerializeNodeContextData(v8::Local<v8::Context> holder,
+                                         int index,
+                                         void* env);
 void DeserializeNodeInternalFields(v8::Local<v8::Object> holder,
                                    int index,
                                    v8::StartupData payload,
                                    void* env);
+void DeserializeNodeContextData(v8::Local<v8::Context> holder,
+                                int index,
+                                v8::StartupData payload,
+                                void* env);
 void SerializeSnapshotableObjects(Realm* realm,
                                   v8::SnapshotCreator* creator,
                                   RealmSerializeInfo* info);

test/parallel/test-snapshot-reproducible.js

@@ -0,0 +1,53 @@
+'use strict';
+
+require('../common');
+const { spawnSyncAndExitWithoutError } = require('../common/child_process');
+const tmpdir = require('../common/tmpdir');
+const fs = require('fs');
+const assert = require('assert');
+const fixtures = require('../common/fixtures');
+
+function generateSnapshot() {
+  tmpdir.refresh();
+
+  spawnSyncAndExitWithoutError(
+    process.execPath,
+    [
+      '--random_seed=42',
+      '--predictable',
+      '--build-snapshot',
+      'node:generate_default_snapshot',
+    ],
+    {
+      cwd: tmpdir.path
+    }
+  );
+  const blobPath = tmpdir.resolve('snapshot.blob');
+  return fs.readFileSync(blobPath);
+}
+
+const buf1 = generateSnapshot();
+const buf2 = generateSnapshot();
+const diff = [];
+let offset = 0;
+const step = 16;
+do {
+  const length = Math.min(buf1.length - offset, step);
+  const slice1 = buf1.slice(offset, offset + length).toString('hex');
+  const slice2 = buf2.slice(offset, offset + length).toString('hex');
+  if (slice1 != slice2) {
+    diff.push({offset, slice1, slice2});
+  }
+  offset += length;
+} while (offset < buf1.length);
+
+assert.strictEqual(offset, buf1.length);
+if (offset < buf2.length) {
+  const length = Math.min(buf2.length - offset, step);
+  const slice2 = buf2.slice(offset, offset + length).toString('hex');
+  diff.push({offset, slice1: '', slice2});
+  offset += length;
+} while (offset < buf2.length);
+
+assert.deepStrictEqual(diff, [], 'Built-in snapshot should not change in different builds.');
+assert.strictEqual(buf1.length, buf2.length);