Skip to content

Commit

Permalink
src: back snapshot I/O with a std::vector sink
Browse files Browse the repository at this point in the history
  • Loading branch information
@joyeecheung
joyeecheung committed Jan 31, 2023
1 parent b8b92ca commit 3748eb0
Showing 1 changed file with 74 additions and 55 deletions.
129 changes: 74 additions & 55 deletions src/node_snapshotable.cc
View file
Expand Up @@ -2,6 +2,7 @@
#include "node_snapshotable.h"
#include <iostream>
#include <sstream>
#include <vector>
#include "base_object-inl.h"
#include "debug_utils-inl.h"
#include "env-inl.h"
Expand Down Expand Up @@ -135,11 +136,10 @@ std::ostream& operator<<(std::ostream& output, const EnvSerializeInfo& i) {
return output;
}

class FileIO {
class SnapshotSerializerDeserializer {
public:
explicit FileIO(FILE* file)
: f(file),
is_debug(per_process::enabled_debug_list.enabled(
explicit SnapshotSerializerDeserializer()
: is_debug(per_process::enabled_debug_list.enabled(
DebugCategory::MKSNAPSHOT)) {}

template <typename... Args>
Expand Down Expand Up @@ -181,14 +181,14 @@ class FileIO {
return name;
}

FILE* f = nullptr;
bool is_debug = false;
};

class FileReader : public FileIO {
class SnapshotDeserializer : public SnapshotSerializerDeserializer {
public:
explicit FileReader(FILE* file) : FileIO(file) {}
~FileReader() {}
explicit SnapshotDeserializer(const std::vector<char>* s)
: SnapshotSerializerDeserializer(), sink(s) {}
~SnapshotDeserializer() {}

// Helper for reading numeric types.
template <typename T>
Expand Down Expand Up @@ -233,19 +233,19 @@ class FileReader : public FileIO {

CHECK_GT(length, 0); // There should be no empty strings.
MallocedBuffer<char> buf(length + 1);
size_t r = fread(buf.data, 1, length + 1, f);
CHECK_EQ(r, length + 1);
memcpy(buf.data, sink->data() + read_total, length + 1);
std::string result(buf.data, length); // This creates a copy of buf.data.

if (is_debug) {
Debug("\"%s\", read %d bytes\n", result.c_str(), r);
Debug("\"%s\", read %zu bytes\n", result.c_str(), length + 1);
}

read_total += r;
read_total += length + 1;
return result;
}

size_t read_total = 0;
const std::vector<char>* sink = nullptr;

private:
// Helper for reading an array of numeric types.
Expand All @@ -258,15 +258,15 @@ class FileReader : public FileIO {
Debug("Read<%s>()(%d-byte), count=%d: ", name.c_str(), sizeof(T), count);
}

size_t r = fread(out, sizeof(T), count, f);
CHECK_EQ(r, count);
size_t size = sizeof(T) * count;
memcpy(out, sink->data() + read_total, size);

if (is_debug) {
std::string str =
"{ " + std::to_string(out[0]) + (count > 1 ? ", ... }" : " }");
Debug("%s, read %d bytes\n", str.c_str(), r);
Debug("%s, read %zu bytes\n", str.c_str(), size);
}
read_total += r;
read_total += size;
}

// Helper for reading numeric vectors.
Expand Down Expand Up @@ -300,10 +300,15 @@ class FileReader : public FileIO {
}
};

class FileWriter : public FileIO {
class SnapshotSerializer : public SnapshotSerializerDeserializer {
public:
explicit FileWriter(FILE* file) : FileIO(file) {}
~FileWriter() {}
explicit SnapshotSerializer() : SnapshotSerializerDeserializer() {
// Currently the snapshot blob built with an empty script is around 4MB.
// So use that as the default sink size.
sink.reserve(4 * 1024 * 1024);
}
~SnapshotSerializer() {}
std::vector<char> sink;

// Helper for writing numeric types.
template <typename T>
Expand Down Expand Up @@ -349,15 +354,15 @@ class FileWriter : public FileIO {
size_t written_total = Write<size_t>(data.size());
if (is_debug) {
std::string str = ToStr(data);
Debug("WriteString(), length=%d: \"%s\"\n", data.size(), data.c_str());
Debug("WriteString(), length=%zu: \"%s\"\n", data.size(), data.c_str());
}

size_t r = fwrite(data.c_str(), 1, data.size() + 1, f);
CHECK_EQ(r, data.size() + 1);
written_total += r;
// Write the null-terminated string.
sink.insert(sink.end(), data.c_str(), data.c_str() + data.size() + 1);
written_total += data.size();

if (is_debug) {
Debug("WriteString() wrote %d bytes\n", written_total);
Debug("WriteString() wrote %zu bytes\n", written_total);
}

return written_total;
Expand All @@ -372,20 +377,21 @@ class FileWriter : public FileIO {
std::string str =
"{ " + std::to_string(data[0]) + (count > 1 ? ", ... }" : " }");
std::string name = GetName<T>();
Debug("Write<%s>() (%d-byte), count=%d: %s",
Debug("Write<%s>() (%zu-byte), count=%zu: %s",
name.c_str(),
sizeof(T),
count,
str.c_str());
}

size_t r = fwrite(data, sizeof(T), count, f);
CHECK_EQ(r, count);
size_t size = sizeof(T) * count;
const char* pos = reinterpret_cast<const char*>(data);
sink.insert(sink.end(), pos, pos + (sizeof(T) * count));

if (is_debug) {
Debug(", wrote %d bytes\n", r);
Debug(", wrote %zu bytes\n", size);
}
return r;
return size;
}

// Helper for writing numeric vectors.
Expand Down Expand Up @@ -418,19 +424,19 @@ class FileWriter : public FileIO {
// [ 4/8 bytes ] length
// [ |length| bytes ] contents
template <>
std::string FileReader::Read() {
std::string SnapshotDeserializer::Read() {
return ReadString();
}
template <>
size_t FileWriter::Write(const std::string& data) {
size_t SnapshotSerializer::Write(const std::string& data) {
return WriteString(data);
}

// Layout of v8::StartupData
// [ 4/8 bytes ] raw_size
// [ |raw_size| bytes ] contents
template <>
v8::StartupData FileReader::Read() {
v8::StartupData SnapshotDeserializer::Read() {
Debug("Read<v8::StartupData>()\n");

int raw_size = Read<int>();
Expand All @@ -445,7 +451,7 @@ v8::StartupData FileReader::Read() {
}

template <>
size_t FileWriter::Write(const v8::StartupData& data) {
size_t SnapshotSerializer::Write(const v8::StartupData& data) {
Debug("\nWrite<v8::StartupData>() size=%d\n", data.raw_size);

CHECK_GT(data.raw_size, 0); // There should be no startup data of size 0.
Expand All @@ -462,7 +468,7 @@ size_t FileWriter::Write(const v8::StartupData& data) {
// [ 4/8 bytes ] length of module code cache
// [ ... ] |length| bytes of module code cache
template <>
builtins::CodeCacheInfo FileReader::Read() {
builtins::CodeCacheInfo SnapshotDeserializer::Read() {
Debug("Read<builtins::CodeCacheInfo>()\n");

builtins::CodeCacheInfo result{ReadString(), ReadVector<uint8_t>()};
Expand All @@ -475,7 +481,7 @@ builtins::CodeCacheInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const builtins::CodeCacheInfo& data) {
size_t SnapshotSerializer::Write(const builtins::CodeCacheInfo& data) {
Debug("\nWrite<builtins::CodeCacheInfo>() id = %s"
", size=%d\n",
data.id.c_str(),
Expand All @@ -495,7 +501,7 @@ size_t FileWriter::Write(const builtins::CodeCacheInfo& data) {
// [ 4/8 bytes ] index in the snapshot blob, can be used with
// GetDataFromSnapshotOnce().
template <>
PropInfo FileReader::Read() {
PropInfo SnapshotDeserializer::Read() {
Debug("Read<PropInfo>()\n");

PropInfo result;
Expand All @@ -512,7 +518,7 @@ PropInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const PropInfo& data) {
size_t SnapshotSerializer::Write(const PropInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<PropInfo>() %s\n", str.c_str());
Expand All @@ -535,7 +541,7 @@ size_t FileWriter::Write(const PropInfo& data) {
// [ ... ] snapshot indices of each element in
// native_execution_async_resources
template <>
AsyncHooks::SerializeInfo FileReader::Read() {
AsyncHooks::SerializeInfo SnapshotDeserializer::Read() {
Debug("Read<AsyncHooks::SerializeInfo>()\n");

AsyncHooks::SerializeInfo result;
Expand All @@ -553,7 +559,7 @@ AsyncHooks::SerializeInfo FileReader::Read() {
return result;
}
template <>
size_t FileWriter::Write(const AsyncHooks::SerializeInfo& data) {
size_t SnapshotSerializer::Write(const AsyncHooks::SerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<AsyncHooks::SerializeInfo>() %s\n", str.c_str());
Expand All @@ -573,7 +579,7 @@ size_t FileWriter::Write(const AsyncHooks::SerializeInfo& data) {
// Layout of TickInfo::SerializeInfo
// [ 4/8 bytes ] snapshot index of fields
template <>
TickInfo::SerializeInfo FileReader::Read() {
TickInfo::SerializeInfo SnapshotDeserializer::Read() {
Debug("Read<TickInfo::SerializeInfo>()\n");

TickInfo::SerializeInfo result;
Expand All @@ -588,7 +594,7 @@ TickInfo::SerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const TickInfo::SerializeInfo& data) {
size_t SnapshotSerializer::Write(const TickInfo::SerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<TickInfo::SerializeInfo>() %s\n", str.c_str());
Expand All @@ -603,7 +609,7 @@ size_t FileWriter::Write(const TickInfo::SerializeInfo& data) {
// Layout of TickInfo::SerializeInfo
// [ 4/8 bytes ] snapshot index of fields
template <>
ImmediateInfo::SerializeInfo FileReader::Read() {
ImmediateInfo::SerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT,
"Read<ImmediateInfo::SerializeInfo>()\n");

Expand All @@ -617,7 +623,7 @@ ImmediateInfo::SerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const ImmediateInfo::SerializeInfo& data) {
size_t SnapshotSerializer::Write(const ImmediateInfo::SerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<ImmediateInfo::SerializeInfo>() %s\n", str.c_str());
Expand All @@ -635,7 +641,7 @@ size_t FileWriter::Write(const ImmediateInfo::SerializeInfo& data) {
// [ 4/8 bytes ] snapshot index of milestones
// [ 4/8 bytes ] snapshot index of observers
template <>
performance::PerformanceState::SerializeInfo FileReader::Read() {
performance::PerformanceState::SerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT,
"Read<PerformanceState::SerializeInfo>()\n");

Expand All @@ -651,7 +657,7 @@ performance::PerformanceState::SerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(
size_t SnapshotSerializer::Write(
const performance::PerformanceState::SerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Expand All @@ -673,7 +679,7 @@ size_t FileWriter::Write(
// [ 4/8 bytes ] length of template_values vector
// [ ... ] |length| of PropInfo data
template <>
IsolateDataSerializeInfo FileReader::Read() {
IsolateDataSerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT,
"Read<IsolateDataSerializeInfo>()\n");

Expand All @@ -688,7 +694,7 @@ IsolateDataSerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const IsolateDataSerializeInfo& data) {
size_t SnapshotSerializer::Write(const IsolateDataSerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<IsolateDataSerializeInfo>() %s\n", str.c_str());
Expand All @@ -702,7 +708,7 @@ size_t FileWriter::Write(const IsolateDataSerializeInfo& data) {
}

template <>
RealmSerializeInfo FileReader::Read() {
RealmSerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT, "Read<RealmSerializeInfo>()\n");
RealmSerializeInfo result;
result.builtins = ReadVector<std::string>();
Expand All @@ -713,7 +719,7 @@ RealmSerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const RealmSerializeInfo& data) {
size_t SnapshotSerializer::Write(const RealmSerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("\nWrite<RealmSerializeInfo>() %s\n", str.c_str());
Expand All @@ -730,7 +736,7 @@ size_t FileWriter::Write(const RealmSerializeInfo& data) {
}

template <>
EnvSerializeInfo FileReader::Read() {
EnvSerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT, "Read<EnvSerializeInfo>()\n");
EnvSerializeInfo result;
result.async_hooks = Read<AsyncHooks::SerializeInfo>();
Expand All @@ -747,7 +753,7 @@ EnvSerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const EnvSerializeInfo& data) {
size_t SnapshotSerializer::Write(const EnvSerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("\nWrite<EnvSerializeInfo>() %s\n", str.c_str());
Expand Down Expand Up @@ -780,7 +786,7 @@ size_t FileWriter::Write(const EnvSerializeInfo& data) {
// [ ... ] |length| bytes of node platform
// [ 4 bytes ] v8 cache version tag
template <>
SnapshotMetadata FileReader::Read() {
SnapshotMetadata SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT, "Read<SnapshotMetadata>()\n");

SnapshotMetadata result;
Expand All @@ -798,7 +804,7 @@ SnapshotMetadata FileReader::Read() {
}

template <>
size_t FileWriter::Write(const SnapshotMetadata& data) {
size_t SnapshotSerializer::Write(const SnapshotMetadata& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("\nWrite<SnapshotMetadata>() %s\n", str.c_str());
Expand Down Expand Up @@ -833,7 +839,7 @@ size_t FileWriter::Write(const SnapshotMetadata& data) {
// [ ... ] code_cache

void SnapshotData::ToBlob(FILE* out) const {
FileWriter w(out);
SnapshotSerializer w;
w.Debug("SnapshotData::ToBlob()\n");

size_t written_total = 0;
Expand All @@ -850,11 +856,24 @@ void SnapshotData::ToBlob(FILE* out) const {
written_total += w.Write<EnvSerializeInfo>(env_info);
w.Debug("Write code_cache\n");
written_total += w.WriteVector<builtins::CodeCacheInfo>(code_cache);
size_t num_written = fwrite(w.sink.data(), w.sink.size(), 1, out);
CHECK_EQ(num_written, 1);
w.Debug("SnapshotData::ToBlob() Wrote %d bytes\n", written_total);
}

bool SnapshotData::FromBlob(SnapshotData* out, FILE* in) {
FileReader r(in);
CHECK_EQ(ftell(in), 0);
int err = fseek(in, 0, SEEK_END);
CHECK_EQ(err, 0);
size_t size = ftell(in);
err = fseek(in, 0, SEEK_SET);
CHECK_EQ(err, 0);

std::vector<char> sink(size);
size_t num_read = fread(sink.data(), size, 1, in);
CHECK_EQ(num_read, 1);

SnapshotDeserializer r(&sink);
r.Debug("SnapshotData::FromBlob()\n");

DCHECK_EQ(out->data_ownership, SnapshotData::DataOwnership::kOwned);
Expand Down

0 comments on commit 3748eb0

Please sign in to comment.