/
liftoff-compiler.cc
6397 lines (5746 loc) Β· 265 KB
/
liftoff-compiler.cc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/wasm/baseline/liftoff-compiler.h"
#include "src/base/enum-set.h"
#include "src/base/optional.h"
#include "src/base/platform/wrappers.h"
#include "src/codegen/assembler-inl.h"
// TODO(clemensb): Remove dependences on compiler stuff.
#include "src/codegen/external-reference.h"
#include "src/codegen/interface-descriptors-inl.h"
#include "src/codegen/machine-type.h"
#include "src/codegen/macro-assembler-inl.h"
#include "src/compiler/linkage.h"
#include "src/compiler/wasm-compiler.h"
#include "src/logging/counters.h"
#include "src/logging/log.h"
#include "src/objects/smi.h"
#include "src/tracing/trace-event.h"
#include "src/utils/ostreams.h"
#include "src/utils/utils.h"
#include "src/wasm/baseline/liftoff-assembler.h"
#include "src/wasm/baseline/liftoff-register.h"
#include "src/wasm/function-body-decoder-impl.h"
#include "src/wasm/function-compiler.h"
#include "src/wasm/memory-tracing.h"
#include "src/wasm/object-access.h"
#include "src/wasm/simd-shuffle.h"
#include "src/wasm/wasm-debug.h"
#include "src/wasm/wasm-engine.h"
#include "src/wasm/wasm-linkage.h"
#include "src/wasm/wasm-objects.h"
#include "src/wasm/wasm-opcodes-inl.h"
namespace v8 {
namespace internal {
namespace wasm {
constexpr auto kRegister = LiftoffAssembler::VarState::kRegister;
constexpr auto kIntConst = LiftoffAssembler::VarState::kIntConst;
constexpr auto kStack = LiftoffAssembler::VarState::kStack;
namespace {
#define __ asm_.
#define TRACE(...) \
do { \
if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \
} while (false)
#define WASM_INSTANCE_OBJECT_FIELD_OFFSET(name) \
ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
template <int expected_size, int actual_size>
struct assert_field_size {
static_assert(expected_size == actual_size,
"field in WasmInstance does not have the expected size");
static constexpr int size = actual_size;
};
#define WASM_INSTANCE_OBJECT_FIELD_SIZE(name) \
FIELD_SIZE(WasmInstanceObject::k##name##Offset)
#define LOAD_INSTANCE_FIELD(dst, name, load_size, pinned) \
__ LoadFromInstance(dst, LoadInstanceIntoRegister(pinned, dst), \
WASM_INSTANCE_OBJECT_FIELD_OFFSET(name), \
assert_field_size<WASM_INSTANCE_OBJECT_FIELD_SIZE(name), \
load_size>::size);
#define LOAD_TAGGED_PTR_INSTANCE_FIELD(dst, name, pinned) \
static_assert(WASM_INSTANCE_OBJECT_FIELD_SIZE(name) == kTaggedSize, \
"field in WasmInstance does not have the expected size"); \
__ LoadTaggedPointerFromInstance(dst, LoadInstanceIntoRegister(pinned, dst), \
WASM_INSTANCE_OBJECT_FIELD_OFFSET(name));
#ifdef V8_CODE_COMMENTS
#define CODE_COMMENT(str) \
do { \
__ RecordComment(str); \
} while (false)
#else
#define CODE_COMMENT(str) ((void)0)
#endif
constexpr LoadType::LoadTypeValue kPointerLoadType =
kSystemPointerSize == 8 ? LoadType::kI64Load : LoadType::kI32Load;
constexpr ValueKind kPointerKind = LiftoffAssembler::kPointerKind;
constexpr ValueKind kSmiKind = LiftoffAssembler::kSmiKind;
constexpr ValueKind kTaggedKind = LiftoffAssembler::kTaggedKind;
// Used to construct fixed-size signatures: MakeSig::Returns(...).Params(...);
using MakeSig = FixedSizeSignature<ValueKind>;
#if V8_TARGET_ARCH_ARM64
// On ARM64, the Assembler keeps track of pointers to Labels to resolve
// branches to distant targets. Moving labels would confuse the Assembler,
// thus store the label on the heap and keep a unique_ptr.
class MovableLabel {
public:
MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(MovableLabel);
MovableLabel() : label_(new Label()) {}
Label* get() { return label_.get(); }
private:
std::unique_ptr<Label> label_;
};
#else
// On all other platforms, just store the Label directly.
class MovableLabel {
public:
MOVE_ONLY_WITH_DEFAULT_CONSTRUCTORS(MovableLabel);
Label* get() { return &label_; }
private:
Label label_;
};
#endif
compiler::CallDescriptor* GetLoweredCallDescriptor(
Zone* zone, compiler::CallDescriptor* call_desc) {
return kSystemPointerSize == 4
? compiler::GetI32WasmCallDescriptor(zone, call_desc)
: call_desc;
}
constexpr LiftoffCondition GetCompareCondition(WasmOpcode opcode) {
switch (opcode) {
case kExprI32Eq:
return kEqual;
case kExprI32Ne:
return kUnequal;
case kExprI32LtS:
return kSignedLessThan;
case kExprI32LtU:
return kUnsignedLessThan;
case kExprI32GtS:
return kSignedGreaterThan;
case kExprI32GtU:
return kUnsignedGreaterThan;
case kExprI32LeS:
return kSignedLessEqual;
case kExprI32LeU:
return kUnsignedLessEqual;
case kExprI32GeS:
return kSignedGreaterEqual;
case kExprI32GeU:
return kUnsignedGreaterEqual;
default:
UNREACHABLE();
}
}
// Builds a {DebugSideTable}.
class DebugSideTableBuilder {
using Entry = DebugSideTable::Entry;
using Value = Entry::Value;
public:
enum AssumeSpilling {
// All register values will be spilled before the pc covered by the debug
// side table entry. Register slots will be marked as stack slots in the
// generated debug side table entry.
kAssumeSpilling,
// Register slots will be written out as they are.
kAllowRegisters,
// Register slots cannot appear since we already spilled.
kDidSpill
};
class EntryBuilder {
public:
explicit EntryBuilder(int pc_offset, int stack_height,
std::vector<Value> changed_values)
: pc_offset_(pc_offset),
stack_height_(stack_height),
changed_values_(std::move(changed_values)) {}
Entry ToTableEntry() {
return Entry{pc_offset_, stack_height_, std::move(changed_values_)};
}
void MinimizeBasedOnPreviousStack(const std::vector<Value>& last_values) {
auto dst = changed_values_.begin();
auto end = changed_values_.end();
for (auto src = dst; src != end; ++src) {
if (src->index < static_cast<int>(last_values.size()) &&
*src == last_values[src->index]) {
continue;
}
if (dst != src) *dst = *src;
++dst;
}
changed_values_.erase(dst, end);
}
int pc_offset() const { return pc_offset_; }
void set_pc_offset(int new_pc_offset) { pc_offset_ = new_pc_offset; }
private:
int pc_offset_;
int stack_height_;
std::vector<Value> changed_values_;
};
// Adds a new entry in regular code.
void NewEntry(int pc_offset,
base::Vector<DebugSideTable::Entry::Value> values) {
entries_.emplace_back(pc_offset, static_cast<int>(values.size()),
GetChangedStackValues(last_values_, values));
}
// Adds a new entry for OOL code, and returns a pointer to a builder for
// modifying that entry.
EntryBuilder* NewOOLEntry(base::Vector<DebugSideTable::Entry::Value> values) {
constexpr int kNoPcOffsetYet = -1;
ool_entries_.emplace_back(kNoPcOffsetYet, static_cast<int>(values.size()),
GetChangedStackValues(last_ool_values_, values));
return &ool_entries_.back();
}
void SetNumLocals(int num_locals) {
DCHECK_EQ(-1, num_locals_);
DCHECK_LE(0, num_locals);
num_locals_ = num_locals;
}
std::unique_ptr<DebugSideTable> GenerateDebugSideTable() {
DCHECK_LE(0, num_locals_);
// Connect {entries_} and {ool_entries_} by removing redundant stack
// information from the first {ool_entries_} entry (based on
// {last_values_}).
if (!entries_.empty() && !ool_entries_.empty()) {
ool_entries_.front().MinimizeBasedOnPreviousStack(last_values_);
}
std::vector<Entry> entries;
entries.reserve(entries_.size() + ool_entries_.size());
for (auto& entry : entries_) entries.push_back(entry.ToTableEntry());
for (auto& entry : ool_entries_) entries.push_back(entry.ToTableEntry());
DCHECK(std::is_sorted(
entries.begin(), entries.end(),
[](Entry& a, Entry& b) { return a.pc_offset() < b.pc_offset(); }));
return std::make_unique<DebugSideTable>(num_locals_, std::move(entries));
}
private:
static std::vector<Value> GetChangedStackValues(
std::vector<Value>& last_values,
base::Vector<DebugSideTable::Entry::Value> values) {
std::vector<Value> changed_values;
int old_stack_size = static_cast<int>(last_values.size());
last_values.resize(values.size());
int index = 0;
for (const auto& value : values) {
if (index >= old_stack_size || last_values[index] != value) {
changed_values.push_back(value);
last_values[index] = value;
}
++index;
}
return changed_values;
}
int num_locals_ = -1;
// Keep a snapshot of the stack of the last entry, to generate a delta to the
// next entry.
std::vector<Value> last_values_;
std::vector<EntryBuilder> entries_;
// Keep OOL code entries separate so we can do proper delta-encoding (more
// entries might be added between the existing {entries_} and the
// {ool_entries_}). Store the entries in a list so the pointer is not
// invalidated by adding more entries.
std::vector<Value> last_ool_values_;
std::list<EntryBuilder> ool_entries_;
};
void CheckBailoutAllowed(LiftoffBailoutReason reason, const char* detail,
const CompilationEnv* env) {
// Decode errors are ok.
if (reason == kDecodeError) return;
// Missing CPU features are also generally OK for now.
if (reason == kMissingCPUFeature) return;
// --liftoff-only ensures that tests actually exercise the Liftoff path
// without bailing out. Bailing out due to (simulated) lack of CPU support
// is okay though (see above).
if (FLAG_liftoff_only) {
FATAL("--liftoff-only: treating bailout as fatal error. Cause: %s", detail);
}
// If --enable-testing-opcode-in-wasm is set, we are expected to bailout with
// "testing opcode".
if (FLAG_enable_testing_opcode_in_wasm &&
strcmp(detail, "testing opcode") == 0) {
return;
}
// Some externally maintained architectures don't fully implement Liftoff yet.
#if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_MIPS64 || V8_TARGET_ARCH_S390X || \
V8_TARGET_ARCH_PPC || V8_TARGET_ARCH_PPC64 || V8_TARGET_ARCH_LOONG64
return;
#endif
#if V8_TARGET_ARCH_ARM
// Allow bailout for missing ARMv7 support.
if (!CpuFeatures::IsSupported(ARMv7) && reason == kUnsupportedArchitecture) {
return;
}
#endif
#define LIST_FEATURE(name, ...) kFeature_##name,
constexpr WasmFeatures kExperimentalFeatures{
FOREACH_WASM_EXPERIMENTAL_FEATURE_FLAG(LIST_FEATURE)};
#undef LIST_FEATURE
// Bailout is allowed if any experimental feature is enabled.
if (env->enabled_features.contains_any(kExperimentalFeatures)) return;
// Otherwise, bailout is not allowed.
FATAL("Liftoff bailout should not happen. Cause: %s\n", detail);
}
class LiftoffCompiler {
public:
// TODO(clemensb): Make this a template parameter.
static constexpr Decoder::ValidateFlag validate = Decoder::kBooleanValidation;
using Value = ValueBase<validate>;
struct ElseState {
MovableLabel label;
LiftoffAssembler::CacheState state;
};
struct TryInfo {
TryInfo() = default;
LiftoffAssembler::CacheState catch_state;
Label catch_label;
bool catch_reached = false;
bool in_handler = false;
};
struct Control : public ControlBase<Value, validate> {
std::unique_ptr<ElseState> else_state;
LiftoffAssembler::CacheState label_state;
MovableLabel label;
std::unique_ptr<TryInfo> try_info;
// Number of exceptions on the stack below this control.
int num_exceptions = 0;
MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(Control);
template <typename... Args>
explicit Control(Args&&... args) V8_NOEXCEPT
: ControlBase(std::forward<Args>(args)...) {}
};
using FullDecoder = WasmFullDecoder<validate, LiftoffCompiler>;
using ValueKindSig = LiftoffAssembler::ValueKindSig;
class MostlySmallValueKindSig : public Signature<ValueKind> {
public:
MostlySmallValueKindSig(Zone* zone, const FunctionSig* sig)
: Signature<ValueKind>(sig->return_count(), sig->parameter_count(),
MakeKinds(inline_storage_, zone, sig)) {}
private:
static constexpr size_t kInlineStorage = 8;
static ValueKind* MakeKinds(ValueKind* storage, Zone* zone,
const FunctionSig* sig) {
const size_t size = sig->parameter_count() + sig->return_count();
if (V8_UNLIKELY(size > kInlineStorage)) {
storage = zone->NewArray<ValueKind>(size);
}
std::transform(sig->all().begin(), sig->all().end(), storage,
[](ValueType type) { return type.kind(); });
return storage;
}
ValueKind inline_storage_[kInlineStorage];
};
// For debugging, we need to spill registers before a trap or a stack check to
// be able to inspect them.
struct SpilledRegistersForInspection : public ZoneObject {
struct Entry {
int offset;
LiftoffRegister reg;
ValueKind kind;
};
ZoneVector<Entry> entries;
explicit SpilledRegistersForInspection(Zone* zone) : entries(zone) {}
};
struct OutOfLineSafepointInfo {
ZoneVector<int> slots;
LiftoffRegList spills;
explicit OutOfLineSafepointInfo(Zone* zone) : slots(zone) {}
};
struct OutOfLineCode {
MovableLabel label;
MovableLabel continuation;
WasmCode::RuntimeStubId stub;
WasmCodePosition position;
LiftoffRegList regs_to_save;
Register cached_instance;
OutOfLineSafepointInfo* safepoint_info;
uint32_t pc; // for trap handler.
// These two pointers will only be used for debug code:
SpilledRegistersForInspection* spilled_registers;
DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder;
// Named constructors:
static OutOfLineCode Trap(
WasmCode::RuntimeStubId s, WasmCodePosition pos,
SpilledRegistersForInspection* spilled_registers,
OutOfLineSafepointInfo* safepoint_info, uint32_t pc,
DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder) {
DCHECK_LT(0, pos);
return {
{}, // label
{}, // continuation
s, // stub
pos, // position
{}, // regs_to_save
no_reg, // cached_instance
safepoint_info, // safepoint_info
pc, // pc
spilled_registers, // spilled_registers
debug_sidetable_entry_builder // debug_side_table_entry_builder
};
}
static OutOfLineCode StackCheck(
WasmCodePosition pos, LiftoffRegList regs_to_save,
Register cached_instance, SpilledRegistersForInspection* spilled_regs,
OutOfLineSafepointInfo* safepoint_info,
DebugSideTableBuilder::EntryBuilder* debug_sidetable_entry_builder) {
return {
{}, // label
{}, // continuation
WasmCode::kWasmStackGuard, // stub
pos, // position
regs_to_save, // regs_to_save
cached_instance, // cached_instance
safepoint_info, // safepoint_info
0, // pc
spilled_regs, // spilled_registers
debug_sidetable_entry_builder // debug_side_table_entry_builder
};
}
};
LiftoffCompiler(compiler::CallDescriptor* call_descriptor,
CompilationEnv* env, Zone* compilation_zone,
std::unique_ptr<AssemblerBuffer> buffer,
DebugSideTableBuilder* debug_sidetable_builder,
ForDebugging for_debugging, int func_index,
base::Vector<const int> breakpoints = {},
int dead_breakpoint = 0, int32_t* max_steps = nullptr,
int32_t* nondeterminism = nullptr)
: asm_(std::move(buffer)),
descriptor_(
GetLoweredCallDescriptor(compilation_zone, call_descriptor)),
env_(env),
debug_sidetable_builder_(debug_sidetable_builder),
for_debugging_(for_debugging),
func_index_(func_index),
out_of_line_code_(compilation_zone),
source_position_table_builder_(compilation_zone),
protected_instructions_(compilation_zone),
compilation_zone_(compilation_zone),
safepoint_table_builder_(compilation_zone_),
next_breakpoint_ptr_(breakpoints.begin()),
next_breakpoint_end_(breakpoints.end()),
dead_breakpoint_(dead_breakpoint),
handlers_(compilation_zone),
max_steps_(max_steps),
nondeterminism_(nondeterminism) {
if (breakpoints.empty()) {
next_breakpoint_ptr_ = next_breakpoint_end_ = nullptr;
}
}
bool did_bailout() const { return bailout_reason_ != kSuccess; }
LiftoffBailoutReason bailout_reason() const { return bailout_reason_; }
void GetCode(CodeDesc* desc) {
asm_.GetCode(nullptr, desc, &safepoint_table_builder_,
handler_table_offset_);
}
std::unique_ptr<AssemblerBuffer> ReleaseBuffer() {
return asm_.ReleaseBuffer();
}
base::OwnedVector<uint8_t> GetSourcePositionTable() {
return source_position_table_builder_.ToSourcePositionTableVector();
}
base::OwnedVector<uint8_t> GetProtectedInstructionsData() const {
return base::OwnedVector<uint8_t>::Of(base::Vector<const uint8_t>::cast(
base::VectorOf(protected_instructions_)));
}
uint32_t GetTotalFrameSlotCountForGC() const {
return __ GetTotalFrameSlotCountForGC();
}
void unsupported(FullDecoder* decoder, LiftoffBailoutReason reason,
const char* detail) {
DCHECK_NE(kSuccess, reason);
if (did_bailout()) return;
bailout_reason_ = reason;
TRACE("unsupported: %s\n", detail);
decoder->errorf(decoder->pc_offset(), "unsupported liftoff operation: %s",
detail);
UnuseLabels(decoder);
CheckBailoutAllowed(reason, detail, env_);
}
bool DidAssemblerBailout(FullDecoder* decoder) {
if (decoder->failed() || !__ did_bailout()) return false;
unsupported(decoder, __ bailout_reason(), __ bailout_detail());
return true;
}
V8_INLINE bool CheckSupportedType(FullDecoder* decoder, ValueKind kind,
const char* context) {
if (V8_LIKELY(supported_types_.contains(kind))) return true;
return MaybeBailoutForUnsupportedType(decoder, kind, context);
}
V8_NOINLINE bool MaybeBailoutForUnsupportedType(FullDecoder* decoder,
ValueKind kind,
const char* context) {
DCHECK(!supported_types_.contains(kind));
// Lazily update {supported_types_}; then check again.
if (CpuFeatures::SupportsWasmSimd128()) supported_types_.Add(kS128);
if (supported_types_.contains(kind)) return true;
LiftoffBailoutReason bailout_reason;
switch (kind) {
case kS128:
bailout_reason = kMissingCPUFeature;
break;
case kRef:
case kOptRef:
case kRtt:
case kRttWithDepth:
case kI8:
case kI16:
bailout_reason = kRefTypes;
break;
default:
UNREACHABLE();
}
base::EmbeddedVector<char, 128> buffer;
SNPrintF(buffer, "%s %s", name(kind), context);
unsupported(decoder, bailout_reason, buffer.begin());
return false;
}
void UnuseLabels(FullDecoder* decoder) {
#ifdef DEBUG
auto Unuse = [](Label* label) {
label->Unuse();
label->UnuseNear();
};
// Unuse all labels now, otherwise their destructor will fire a DCHECK error
// if they where referenced before.
uint32_t control_depth = decoder ? decoder->control_depth() : 0;
for (uint32_t i = 0; i < control_depth; ++i) {
Control* c = decoder->control_at(i);
Unuse(c->label.get());
if (c->else_state) Unuse(c->else_state->label.get());
if (c->try_info != nullptr) Unuse(&c->try_info->catch_label);
}
for (auto& ool : out_of_line_code_) Unuse(ool.label.get());
#endif
}
void StartFunction(FullDecoder* decoder) {
if (FLAG_trace_liftoff && !FLAG_trace_wasm_decoder) {
StdoutStream{} << "hint: add --trace-wasm-decoder to also see the wasm "
"instructions being decoded\n";
}
int num_locals = decoder->num_locals();
__ set_num_locals(num_locals);
for (int i = 0; i < num_locals; ++i) {
ValueKind kind = decoder->local_type(i).kind();
__ set_local_kind(i, kind);
}
}
constexpr static LiftoffRegList RegsUnusedByParams() {
LiftoffRegList regs = kGpCacheRegList;
for (auto reg : kGpParamRegisters) {
regs.clear(reg);
}
return regs;
}
// Returns the number of inputs processed (1 or 2).
uint32_t ProcessParameter(ValueKind kind, uint32_t input_idx) {
const bool needs_pair = needs_gp_reg_pair(kind);
const ValueKind reg_kind = needs_pair ? kI32 : kind;
const RegClass rc = reg_class_for(reg_kind);
auto LoadToReg = [this, reg_kind, rc](compiler::LinkageLocation location,
LiftoffRegList pinned) {
if (location.IsRegister()) {
DCHECK(!location.IsAnyRegister());
return LiftoffRegister::from_external_code(rc, reg_kind,
location.AsRegister());
}
DCHECK(location.IsCallerFrameSlot());
// For reference type parameters we have to use registers that were not
// used for parameters because some reference type stack parameters may
// get processed before some value type register parameters.
static constexpr auto kRegsUnusedByParams = RegsUnusedByParams();
LiftoffRegister reg = is_reference(reg_kind)
? __ GetUnusedRegister(kRegsUnusedByParams)
: __ GetUnusedRegister(rc, pinned);
__ LoadCallerFrameSlot(reg, -location.AsCallerFrameSlot(), reg_kind);
return reg;
};
LiftoffRegister reg =
LoadToReg(descriptor_->GetInputLocation(input_idx), {});
if (needs_pair) {
LiftoffRegister reg2 =
LoadToReg(descriptor_->GetInputLocation(input_idx + 1),
LiftoffRegList::ForRegs(reg));
reg = LiftoffRegister::ForPair(reg.gp(), reg2.gp());
}
__ PushRegister(kind, reg);
return needs_pair ? 2 : 1;
}
void StackCheck(FullDecoder* decoder, WasmCodePosition position) {
CODE_COMMENT("stack check");
if (!FLAG_wasm_stack_checks || !env_->runtime_exception_support) return;
// Loading the limit address can change the stack state, hence do this
// before storing information about registers.
Register limit_address = __ GetUnusedRegister(kGpReg, {}).gp();
LOAD_INSTANCE_FIELD(limit_address, StackLimitAddress, kSystemPointerSize,
{});
LiftoffRegList regs_to_save = __ cache_state()->used_registers;
// The cached instance will be reloaded separately.
if (__ cache_state()->cached_instance != no_reg) {
DCHECK(regs_to_save.has(__ cache_state()->cached_instance));
regs_to_save.clear(__ cache_state()->cached_instance);
}
SpilledRegistersForInspection* spilled_regs = nullptr;
OutOfLineSafepointInfo* safepoint_info =
compilation_zone_->New<OutOfLineSafepointInfo>(compilation_zone_);
__ cache_state()->GetTaggedSlotsForOOLCode(
&safepoint_info->slots, &safepoint_info->spills,
for_debugging_
? LiftoffAssembler::CacheState::SpillLocation::kStackSlots
: LiftoffAssembler::CacheState::SpillLocation::kTopOfStack);
if (V8_UNLIKELY(for_debugging_)) {
// When debugging, we do not just push all registers to the stack, but we
// spill them to their proper stack locations such that we can inspect
// them.
// The only exception is the cached memory start, which we just push
// before the stack check and pop afterwards.
regs_to_save = {};
if (__ cache_state()->cached_mem_start != no_reg) {
regs_to_save.set(__ cache_state()->cached_mem_start);
}
spilled_regs = GetSpilledRegistersForInspection();
}
out_of_line_code_.push_back(OutOfLineCode::StackCheck(
position, regs_to_save, __ cache_state()->cached_instance, spilled_regs,
safepoint_info, RegisterOOLDebugSideTableEntry(decoder)));
OutOfLineCode& ool = out_of_line_code_.back();
__ StackCheck(ool.label.get(), limit_address);
__ bind(ool.continuation.get());
}
bool SpillLocalsInitially(FullDecoder* decoder, uint32_t num_params) {
int actual_locals = __ num_locals() - num_params;
DCHECK_LE(0, actual_locals);
constexpr int kNumCacheRegisters = NumRegs(kLiftoffAssemblerGpCacheRegs);
// If we have many locals, we put them on the stack initially. This avoids
// having to spill them on merge points. Use of these initial values should
// be rare anyway.
if (actual_locals > kNumCacheRegisters / 2) return true;
// If there are locals which are not i32 or i64, we also spill all locals,
// because other types cannot be initialized to constants.
for (uint32_t param_idx = num_params; param_idx < __ num_locals();
++param_idx) {
ValueKind kind = __ local_kind(param_idx);
if (kind != kI32 && kind != kI64) return true;
}
return false;
}
void TierUpFunction(FullDecoder* decoder) {
__ CallRuntimeStub(WasmCode::kWasmTriggerTierUp);
DefineSafepoint();
}
void TraceFunctionEntry(FullDecoder* decoder) {
CODE_COMMENT("trace function entry");
__ SpillAllRegisters();
source_position_table_builder_.AddPosition(
__ pc_offset(), SourcePosition(decoder->position()), false);
__ CallRuntimeStub(WasmCode::kWasmTraceEnter);
DefineSafepoint();
}
void StartFunctionBody(FullDecoder* decoder, Control* block) {
for (uint32_t i = 0; i < __ num_locals(); ++i) {
if (!CheckSupportedType(decoder, __ local_kind(i), "param")) return;
}
// Parameter 0 is the instance parameter.
uint32_t num_params =
static_cast<uint32_t>(decoder->sig_->parameter_count());
__ CodeEntry();
__ EnterFrame(StackFrame::WASM);
__ set_has_frame(true);
pc_offset_stack_frame_construction_ = __ PrepareStackFrame();
// {PrepareStackFrame} is the first platform-specific assembler method.
// If this failed, we can bail out immediately, avoiding runtime overhead
// and potential failures because of other unimplemented methods.
// A platform implementing {PrepareStackFrame} must ensure that we can
// finish compilation without errors even if we hit unimplemented
// LiftoffAssembler methods.
if (DidAssemblerBailout(decoder)) return;
// Input 0 is the call target, the instance is at 1.
constexpr int kInstanceParameterIndex = 1;
// Check that {kWasmInstanceRegister} matches our call descriptor.
DCHECK_EQ(kWasmInstanceRegister,
Register::from_code(
descriptor_->GetInputLocation(kInstanceParameterIndex)
.AsRegister()));
__ cache_state()->SetInstanceCacheRegister(kWasmInstanceRegister);
if (for_debugging_) __ ResetOSRTarget();
// Process parameters.
if (num_params) CODE_COMMENT("process parameters");
// Input 0 is the code target, 1 is the instance. First parameter at 2.
uint32_t input_idx = kInstanceParameterIndex + 1;
for (uint32_t param_idx = 0; param_idx < num_params; ++param_idx) {
input_idx += ProcessParameter(__ local_kind(param_idx), input_idx);
}
int params_size = __ TopSpillOffset();
DCHECK_EQ(input_idx, descriptor_->InputCount());
// Initialize locals beyond parameters.
if (num_params < __ num_locals()) CODE_COMMENT("init locals");
if (SpillLocalsInitially(decoder, num_params)) {
bool has_refs = false;
for (uint32_t param_idx = num_params; param_idx < __ num_locals();
++param_idx) {
ValueKind kind = __ local_kind(param_idx);
has_refs |= is_reference(kind);
__ PushStack(kind);
}
int spill_size = __ TopSpillOffset() - params_size;
__ FillStackSlotsWithZero(params_size, spill_size);
// Initialize all reference type locals with ref.null.
if (has_refs) {
Register null_ref_reg = __ GetUnusedRegister(kGpReg, {}).gp();
LoadNullValue(null_ref_reg, {});
for (uint32_t local_index = num_params; local_index < __ num_locals();
++local_index) {
ValueKind kind = __ local_kind(local_index);
if (is_reference(kind)) {
__ Spill(__ cache_state()->stack_state[local_index].offset(),
LiftoffRegister(null_ref_reg), kind);
}
}
}
} else {
for (uint32_t param_idx = num_params; param_idx < __ num_locals();
++param_idx) {
ValueKind kind = __ local_kind(param_idx);
// Anything which is not i32 or i64 requires spilling.
DCHECK(kind == kI32 || kind == kI64);
__ PushConstant(kind, int32_t{0});
}
}
DCHECK_EQ(__ num_locals(), __ cache_state()->stack_height());
if (V8_UNLIKELY(debug_sidetable_builder_)) {
debug_sidetable_builder_->SetNumLocals(__ num_locals());
}
// The function-prologue stack check is associated with position 0, which
// is never a position of any instruction in the function.
StackCheck(decoder, 0);
if (env_->dynamic_tiering == DynamicTiering::kEnabled) {
// TODO(arobin): Avoid spilling registers unconditionally.
__ SpillAllRegisters();
CODE_COMMENT("dynamic tiering");
LiftoffRegList pinned;
// Get the number of calls array address.
LiftoffRegister array_address =
pinned.set(__ GetUnusedRegister(kGpReg, pinned));
LOAD_INSTANCE_FIELD(array_address.gp(), NumLiftoffFunctionCallsArray,
kSystemPointerSize, pinned);
// Compute the correct offset in the array.
uint32_t offset =
kInt32Size * declared_function_index(env_->module, func_index_);
// Get the number of calls and update it.
LiftoffRegister old_number_of_calls =
pinned.set(__ GetUnusedRegister(kGpReg, pinned));
LiftoffRegister new_number_of_calls =
pinned.set(__ GetUnusedRegister(kGpReg, pinned));
__ Load(old_number_of_calls, array_address.gp(), no_reg, offset,
LoadType::kI32Load, pinned);
__ emit_i32_addi(new_number_of_calls.gp(), old_number_of_calls.gp(), 1);
__ Store(array_address.gp(), no_reg, offset, new_number_of_calls,
StoreType::kI32Store, pinned);
// Emit the runtime call if necessary.
Label no_tierup;
// Check if the number of calls is a power of 2.
__ emit_i32_and(old_number_of_calls.gp(), old_number_of_calls.gp(),
new_number_of_calls.gp());
__ emit_cond_jump(kNotEqualZero, &no_tierup, kI32,
old_number_of_calls.gp());
TierUpFunction(decoder);
// After the runtime call, the instance cache register is clobbered (we
// reset it already in {SpillAllRegisters} above, but then we still access
// the instance afterwards).
__ cache_state()->ClearCachedInstanceRegister();
__ bind(&no_tierup);
}
if (FLAG_trace_wasm) TraceFunctionEntry(decoder);
}
void GenerateOutOfLineCode(OutOfLineCode* ool) {
CODE_COMMENT(
(std::string("OOL: ") + GetRuntimeStubName(ool->stub)).c_str());
__ bind(ool->label.get());
const bool is_stack_check = ool->stub == WasmCode::kWasmStackGuard;
// Only memory OOB traps need a {pc}, but not unconditionally. Static OOB
// accesses do not need protected instruction information, hence they also
// do not set {pc}.
DCHECK_IMPLIES(ool->stub != WasmCode::kThrowWasmTrapMemOutOfBounds,
ool->pc == 0);
if (env_->bounds_checks == kTrapHandler && ool->pc != 0) {
uint32_t pc = static_cast<uint32_t>(__ pc_offset());
DCHECK_EQ(pc, __ pc_offset());
protected_instructions_.emplace_back(
trap_handler::ProtectedInstructionData{ool->pc, pc});
}
if (!env_->runtime_exception_support) {
// We cannot test calls to the runtime in cctest/test-run-wasm.
// Therefore we emit a call to C here instead of a call to the runtime.
// In this mode, we never generate stack checks.
DCHECK(!is_stack_check);
__ CallTrapCallbackForTesting();
__ LeaveFrame(StackFrame::WASM);
__ DropStackSlotsAndRet(
static_cast<uint32_t>(descriptor_->ParameterSlotCount()));
return;
}
if (!ool->regs_to_save.is_empty()) {
__ PushRegisters(ool->regs_to_save);
}
if (V8_UNLIKELY(ool->spilled_registers != nullptr)) {
for (auto& entry : ool->spilled_registers->entries) {
// We should not push and spill the same register.
DCHECK(!ool->regs_to_save.has(entry.reg));
__ Spill(entry.offset, entry.reg, entry.kind);
}
}
source_position_table_builder_.AddPosition(
__ pc_offset(), SourcePosition(ool->position), true);
__ CallRuntimeStub(ool->stub);
Safepoint safepoint = safepoint_table_builder_.DefineSafepoint(&asm_);
if (ool->safepoint_info) {
for (auto index : ool->safepoint_info->slots) {
safepoint.DefinePointerSlot(index);
}
int total_frame_size = __ GetTotalFrameSize();
LiftoffRegList gp_regs = ool->regs_to_save & kGpCacheRegList;
// {total_frame_size} is the highest offset from the FP that is used to
// store a value. The offset of the first spill slot should therefore be
// {(total_frame_size / kSystemPointerSize) + 1}. However, spill slots
// don't start at offset '0' but at offset '-1' (or
// {-kSystemPointerSize}). Therefore we have to add another '+ 1' to the
// index of the first spill slot.
int index = (total_frame_size / kSystemPointerSize) + 2;
__ RecordSpillsInSafepoint(safepoint, gp_regs,
ool->safepoint_info->spills, index);
}
DCHECK_EQ(!debug_sidetable_builder_, !ool->debug_sidetable_entry_builder);
if (V8_UNLIKELY(ool->debug_sidetable_entry_builder)) {
ool->debug_sidetable_entry_builder->set_pc_offset(__ pc_offset());
}
DCHECK_EQ(ool->continuation.get()->is_bound(), is_stack_check);
if (is_stack_check) {
MaybeOSR();
}
if (!ool->regs_to_save.is_empty()) __ PopRegisters(ool->regs_to_save);
if (is_stack_check) {
if (V8_UNLIKELY(ool->spilled_registers != nullptr)) {
DCHECK(for_debugging_);
for (auto& entry : ool->spilled_registers->entries) {
__ Fill(entry.reg, entry.offset, entry.kind);
}
}
if (ool->cached_instance != no_reg) {
__ LoadInstanceFromFrame(ool->cached_instance);
}
__ emit_jump(ool->continuation.get());
} else {
__ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
}
}
void FinishFunction(FullDecoder* decoder) {
if (DidAssemblerBailout(decoder)) return;
__ AlignFrameSize();
#if DEBUG
int frame_size = __ GetTotalFrameSize();
#endif
for (OutOfLineCode& ool : out_of_line_code_) {
GenerateOutOfLineCode(&ool);
}
DCHECK_EQ(frame_size, __ GetTotalFrameSize());
__ PatchPrepareStackFrame(pc_offset_stack_frame_construction_,
&safepoint_table_builder_);
__ FinishCode();
safepoint_table_builder_.Emit(&asm_, __ GetTotalFrameSlotCountForGC());
// Emit the handler table.
if (!handlers_.empty()) {
handler_table_offset_ = HandlerTable::EmitReturnTableStart(&asm_);
for (auto& handler : handlers_) {
HandlerTable::EmitReturnEntry(&asm_, handler.pc_offset,
handler.handler.get()->pos());
}
}
__ MaybeEmitOutOfLineConstantPool();
// The previous calls may have also generated a bailout.
DidAssemblerBailout(decoder);
DCHECK_EQ(num_exceptions_, 0);
}
void OnFirstError(FullDecoder* decoder) {
if (!did_bailout()) bailout_reason_ = kDecodeError;
UnuseLabels(decoder);
asm_.AbortCompilation();
}
V8_NOINLINE void EmitDebuggingInfo(FullDecoder* decoder, WasmOpcode opcode) {
DCHECK(for_debugging_);
if (!WasmOpcodes::IsBreakable(opcode)) return;
bool has_breakpoint = false;
if (next_breakpoint_ptr_) {
if (*next_breakpoint_ptr_ == 0) {
// A single breakpoint at offset 0 indicates stepping.
DCHECK_EQ(next_breakpoint_ptr_ + 1, next_breakpoint_end_);
has_breakpoint = true;
} else {
while (next_breakpoint_ptr_ != next_breakpoint_end_ &&