Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

src: back snapshot I/O with a std::vector sink #46463

Merged
merged 2 commits into from Feb 3, 2023
Merged

src: back snapshot I/O with a std::vector sink #46463

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d21be77
src: back snapshot I/O with a std::vector sink
joyeecheung Jan 31, 2023
1bc6a14
fixup! src: back snapshot I/O with a std::vector sink
joyeecheung Feb 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
129 changes: 74 additions & 55 deletions src/node_snapshotable.cc
View file
Open in desktop
Expand Up @@ -2,6 +2,7 @@
#include "node_snapshotable.h"
#include <iostream>
#include <sstream>
#include <vector>
#include "base_object-inl.h"
#include "debug_utils-inl.h"
#include "env-inl.h"
Expand Down Expand Up @@ -135,11 +136,10 @@ std::ostream& operator<<(std::ostream& output, const EnvSerializeInfo& i) {
return output;
}

class FileIO {
class SnapshotSerializerDeserializer {
public:
explicit FileIO(FILE* file)
: f(file),
is_debug(per_process::enabled_debug_list.enabled(
SnapshotSerializerDeserializer()
: is_debug(per_process::enabled_debug_list.enabled(
DebugCategory::MKSNAPSHOT)) {}

template <typename... Args>
Expand Down Expand Up @@ -181,14 +181,14 @@ class FileIO {
return name;
}

FILE* f = nullptr;
bool is_debug = false;
};

class FileReader : public FileIO {
class SnapshotDeserializer : public SnapshotSerializerDeserializer {
public:
explicit FileReader(FILE* file) : FileIO(file) {}
~FileReader() {}
explicit SnapshotDeserializer(const std::vector<char>* s)
addaleax marked this conversation as resolved.
Show resolved Hide resolved
: SnapshotSerializerDeserializer(), sink(s) {}
~SnapshotDeserializer() {}

// Helper for reading numeric types.
template <typename T>
Expand Down Expand Up @@ -233,19 +233,19 @@ class FileReader : public FileIO {

CHECK_GT(length, 0); // There should be no empty strings.
MallocedBuffer<char> buf(length + 1);
size_t r = fread(buf.data, 1, length + 1, f);
CHECK_EQ(r, length + 1);
memcpy(buf.data, sink->data() + read_total, length + 1);
std::string result(buf.data, length); // This creates a copy of buf.data.

if (is_debug) {
Debug("\"%s\", read %d bytes\n", result.c_str(), r);
Debug("\"%s\", read %zu bytes\n", result.c_str(), length + 1);
}

read_total += r;
read_total += length + 1;
return result;
}

size_t read_total = 0;
const std::vector<char>* sink = nullptr;
joyeecheung marked this conversation as resolved.
Show resolved Hide resolved

private:
// Helper for reading an array of numeric types.
Expand All @@ -258,15 +258,15 @@ class FileReader : public FileIO {
Debug("Read<%s>()(%d-byte), count=%d: ", name.c_str(), sizeof(T), count);
}

size_t r = fread(out, sizeof(T), count, f);
CHECK_EQ(r, count);
size_t size = sizeof(T) * count;
memcpy(out, sink->data() + read_total, size);

if (is_debug) {
std::string str =
"{ " + std::to_string(out[0]) + (count > 1 ? ", ... }" : " }");
Debug("%s, read %d bytes\n", str.c_str(), r);
Debug("%s, read %zu bytes\n", str.c_str(), size);
}
read_total += r;
read_total += size;
}

// Helper for reading numeric vectors.
Expand Down Expand Up @@ -300,10 +300,15 @@ class FileReader : public FileIO {
}
};

class FileWriter : public FileIO {
class SnapshotSerializer : public SnapshotSerializerDeserializer {
public:
explicit FileWriter(FILE* file) : FileIO(file) {}
~FileWriter() {}
SnapshotSerializer() : SnapshotSerializerDeserializer() {
// Currently the snapshot blob built with an empty script is around 4MB.
// So use that as the default sink size.
sink.reserve(4 * 1024 * 1024);
}
~SnapshotSerializer() {}
std::vector<char> sink;

// Helper for writing numeric types.
template <typename T>
Expand Down Expand Up @@ -349,15 +354,15 @@ class FileWriter : public FileIO {
size_t written_total = Write<size_t>(data.size());
if (is_debug) {
std::string str = ToStr(data);
Debug("WriteString(), length=%d: \"%s\"\n", data.size(), data.c_str());
Debug("WriteString(), length=%zu: \"%s\"\n", data.size(), data.c_str());
}

size_t r = fwrite(data.c_str(), 1, data.size() + 1, f);
CHECK_EQ(r, data.size() + 1);
written_total += r;
// Write the null-terminated string.
sink.insert(sink.end(), data.c_str(), data.c_str() + data.size() + 1);
written_total += data.size();
joyeecheung marked this conversation as resolved.
Show resolved Hide resolved

if (is_debug) {
Debug("WriteString() wrote %d bytes\n", written_total);
Debug("WriteString() wrote %zu bytes\n", written_total);
}

return written_total;
Expand All @@ -372,20 +377,21 @@ class FileWriter : public FileIO {
std::string str =
"{ " + std::to_string(data[0]) + (count > 1 ? ", ... }" : " }");
std::string name = GetName<T>();
Debug("Write<%s>() (%d-byte), count=%d: %s",
Debug("Write<%s>() (%zu-byte), count=%zu: %s",
name.c_str(),
sizeof(T),
count,
str.c_str());
}

size_t r = fwrite(data, sizeof(T), count, f);
CHECK_EQ(r, count);
size_t size = sizeof(T) * count;
const char* pos = reinterpret_cast<const char*>(data);
sink.insert(sink.end(), pos, pos + (sizeof(T) * count));
joyeecheung marked this conversation as resolved.
Show resolved Hide resolved

if (is_debug) {
Debug(", wrote %d bytes\n", r);
Debug(", wrote %zu bytes\n", size);
}
return r;
return size;
}

// Helper for writing numeric vectors.
Expand Down Expand Up @@ -418,19 +424,19 @@ class FileWriter : public FileIO {
// [ 4/8 bytes ] length
// [ |length| bytes ] contents
template <>
std::string FileReader::Read() {
std::string SnapshotDeserializer::Read() {
return ReadString();
}
template <>
size_t FileWriter::Write(const std::string& data) {
size_t SnapshotSerializer::Write(const std::string& data) {
return WriteString(data);
}

// Layout of v8::StartupData
// [ 4/8 bytes ] raw_size
// [ |raw_size| bytes ] contents
template <>
v8::StartupData FileReader::Read() {
v8::StartupData SnapshotDeserializer::Read() {
Debug("Read<v8::StartupData>()\n");

int raw_size = Read<int>();
Expand All @@ -445,7 +451,7 @@ v8::StartupData FileReader::Read() {
}

template <>
size_t FileWriter::Write(const v8::StartupData& data) {
size_t SnapshotSerializer::Write(const v8::StartupData& data) {
Debug("\nWrite<v8::StartupData>() size=%d\n", data.raw_size);

CHECK_GT(data.raw_size, 0); // There should be no startup data of size 0.
Expand All @@ -462,7 +468,7 @@ size_t FileWriter::Write(const v8::StartupData& data) {
// [ 4/8 bytes ] length of module code cache
// [ ... ] |length| bytes of module code cache
template <>
builtins::CodeCacheInfo FileReader::Read() {
builtins::CodeCacheInfo SnapshotDeserializer::Read() {
Debug("Read<builtins::CodeCacheInfo>()\n");

builtins::CodeCacheInfo result{ReadString(), ReadVector<uint8_t>()};
Expand All @@ -475,7 +481,7 @@ builtins::CodeCacheInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const builtins::CodeCacheInfo& data) {
size_t SnapshotSerializer::Write(const builtins::CodeCacheInfo& data) {
Debug("\nWrite<builtins::CodeCacheInfo>() id = %s"
", size=%d\n",
data.id.c_str(),
Expand All @@ -495,7 +501,7 @@ size_t FileWriter::Write(const builtins::CodeCacheInfo& data) {
// [ 4/8 bytes ] index in the snapshot blob, can be used with
// GetDataFromSnapshotOnce().
template <>
PropInfo FileReader::Read() {
PropInfo SnapshotDeserializer::Read() {
Debug("Read<PropInfo>()\n");

PropInfo result;
Expand All @@ -512,7 +518,7 @@ PropInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const PropInfo& data) {
size_t SnapshotSerializer::Write(const PropInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<PropInfo>() %s\n", str.c_str());
Expand All @@ -535,7 +541,7 @@ size_t FileWriter::Write(const PropInfo& data) {
// [ ... ] snapshot indices of each element in
// native_execution_async_resources
template <>
AsyncHooks::SerializeInfo FileReader::Read() {
AsyncHooks::SerializeInfo SnapshotDeserializer::Read() {
Debug("Read<AsyncHooks::SerializeInfo>()\n");

AsyncHooks::SerializeInfo result;
Expand All @@ -553,7 +559,7 @@ AsyncHooks::SerializeInfo FileReader::Read() {
return result;
}
template <>
size_t FileWriter::Write(const AsyncHooks::SerializeInfo& data) {
size_t SnapshotSerializer::Write(const AsyncHooks::SerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<AsyncHooks::SerializeInfo>() %s\n", str.c_str());
Expand All @@ -573,7 +579,7 @@ size_t FileWriter::Write(const AsyncHooks::SerializeInfo& data) {
// Layout of TickInfo::SerializeInfo
// [ 4/8 bytes ] snapshot index of fields
template <>
TickInfo::SerializeInfo FileReader::Read() {
TickInfo::SerializeInfo SnapshotDeserializer::Read() {
Debug("Read<TickInfo::SerializeInfo>()\n");

TickInfo::SerializeInfo result;
Expand All @@ -588,7 +594,7 @@ TickInfo::SerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const TickInfo::SerializeInfo& data) {
size_t SnapshotSerializer::Write(const TickInfo::SerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<TickInfo::SerializeInfo>() %s\n", str.c_str());
Expand All @@ -603,7 +609,7 @@ size_t FileWriter::Write(const TickInfo::SerializeInfo& data) {
// Layout of TickInfo::SerializeInfo
// [ 4/8 bytes ] snapshot index of fields
template <>
ImmediateInfo::SerializeInfo FileReader::Read() {
ImmediateInfo::SerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT,
"Read<ImmediateInfo::SerializeInfo>()\n");

Expand All @@ -617,7 +623,7 @@ ImmediateInfo::SerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const ImmediateInfo::SerializeInfo& data) {
size_t SnapshotSerializer::Write(const ImmediateInfo::SerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<ImmediateInfo::SerializeInfo>() %s\n", str.c_str());
Expand All @@ -635,7 +641,7 @@ size_t FileWriter::Write(const ImmediateInfo::SerializeInfo& data) {
// [ 4/8 bytes ] snapshot index of milestones
// [ 4/8 bytes ] snapshot index of observers
template <>
performance::PerformanceState::SerializeInfo FileReader::Read() {
performance::PerformanceState::SerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT,
"Read<PerformanceState::SerializeInfo>()\n");

Expand All @@ -651,7 +657,7 @@ performance::PerformanceState::SerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(
size_t SnapshotSerializer::Write(
const performance::PerformanceState::SerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Expand All @@ -673,7 +679,7 @@ size_t FileWriter::Write(
// [ 4/8 bytes ] length of template_values vector
// [ ... ] |length| of PropInfo data
template <>
IsolateDataSerializeInfo FileReader::Read() {
IsolateDataSerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT,
"Read<IsolateDataSerializeInfo>()\n");

Expand All @@ -688,7 +694,7 @@ IsolateDataSerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const IsolateDataSerializeInfo& data) {
size_t SnapshotSerializer::Write(const IsolateDataSerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("Write<IsolateDataSerializeInfo>() %s\n", str.c_str());
Expand All @@ -702,7 +708,7 @@ size_t FileWriter::Write(const IsolateDataSerializeInfo& data) {
}

template <>
RealmSerializeInfo FileReader::Read() {
RealmSerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT, "Read<RealmSerializeInfo>()\n");
RealmSerializeInfo result;
result.builtins = ReadVector<std::string>();
Expand All @@ -713,7 +719,7 @@ RealmSerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const RealmSerializeInfo& data) {
size_t SnapshotSerializer::Write(const RealmSerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("\nWrite<RealmSerializeInfo>() %s\n", str.c_str());
Expand All @@ -730,7 +736,7 @@ size_t FileWriter::Write(const RealmSerializeInfo& data) {
}

template <>
EnvSerializeInfo FileReader::Read() {
EnvSerializeInfo SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT, "Read<EnvSerializeInfo>()\n");
EnvSerializeInfo result;
result.async_hooks = Read<AsyncHooks::SerializeInfo>();
Expand All @@ -747,7 +753,7 @@ EnvSerializeInfo FileReader::Read() {
}

template <>
size_t FileWriter::Write(const EnvSerializeInfo& data) {
size_t SnapshotSerializer::Write(const EnvSerializeInfo& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("\nWrite<EnvSerializeInfo>() %s\n", str.c_str());
Expand Down Expand Up @@ -780,7 +786,7 @@ size_t FileWriter::Write(const EnvSerializeInfo& data) {
// [ ... ] |length| bytes of node platform
// [ 4 bytes ] v8 cache version tag
template <>
SnapshotMetadata FileReader::Read() {
SnapshotMetadata SnapshotDeserializer::Read() {
per_process::Debug(DebugCategory::MKSNAPSHOT, "Read<SnapshotMetadata>()\n");

SnapshotMetadata result;
Expand All @@ -798,7 +804,7 @@ SnapshotMetadata FileReader::Read() {
}

template <>
size_t FileWriter::Write(const SnapshotMetadata& data) {
size_t SnapshotSerializer::Write(const SnapshotMetadata& data) {
if (is_debug) {
std::string str = ToStr(data);
Debug("\nWrite<SnapshotMetadata>() %s\n", str.c_str());
Expand Down Expand Up @@ -833,7 +839,7 @@ size_t FileWriter::Write(const SnapshotMetadata& data) {
// [ ... ] code_cache

void SnapshotData::ToBlob(FILE* out) const {
FileWriter w(out);
SnapshotSerializer w;
w.Debug("SnapshotData::ToBlob()\n");

size_t written_total = 0;
Expand All @@ -850,11 +856,24 @@ void SnapshotData::ToBlob(FILE* out) const {
written_total += w.Write<EnvSerializeInfo>(env_info);
w.Debug("Write code_cache\n");
written_total += w.WriteVector<builtins::CodeCacheInfo>(code_cache);
size_t num_written = fwrite(w.sink.data(), w.sink.size(), 1, out);
CHECK_EQ(num_written, 1);
w.Debug("SnapshotData::ToBlob() Wrote %d bytes\n", written_total);
}

bool SnapshotData::FromBlob(SnapshotData* out, FILE* in) {
FileReader r(in);
CHECK_EQ(ftell(in), 0);
int err = fseek(in, 0, SEEK_END);
CHECK_EQ(err, 0);
size_t size = ftell(in);
err = fseek(in, 0, SEEK_SET);
CHECK_EQ(err, 0);

std::vector<char> sink(size);
size_t num_read = fread(sink.data(), size, 1, in);
CHECK_EQ(num_read, 1);

SnapshotDeserializer r(&sink);
r.Debug("SnapshotData::FromBlob()\n");

DCHECK_EQ(out->data_ownership, SnapshotData::DataOwnership::kOwned);
Expand Down