[GR-51491] Adapt to ICStubs removal.

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/Assembler.java

@@ -33,7 +33,7 @@
 import jdk.vm.ci.code.Register;
 import jdk.vm.ci.code.StackSlot;
 import jdk.vm.ci.code.TargetDescription;
-import jdk.vm.ci.code.site.Infopoint;
+import jdk.vm.ci.code.site.Call;
 
 /**
  * The platform-independent base class for the assembler.
@@ -271,11 +271,15 @@ public void maybeEmitIndirectTargetMarker(CompilationResultBuilder crb, Label la
 
     /**
      * Some platforms might require special post call code emission.
-     *
-     * @param infopoint The infopoint assoicated with the call if any
      */
-    public void postCallNop(Infopoint infopoint) {
-        ensureUniquePC();
+    public void postCallNop(Call call) {
+        if (call.debugInfo != null) {
+            // The nop inserted after a call is only required to distinguish
+            // debug info associated with the call from debug info associated
+            // with an instruction after the call. If the call has no debug
+            // info, the extra nop is not required.
+            ensureUniquePC();
+        }
     }
 
     public void reset() {

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/aarch64/AArch64MacroAssembler.java

@@ -1985,12 +1985,17 @@ public void illegal() {
      */
     @Override
     public void align(int modulus) {
+        align(modulus, position());
+    }
+
+    /**
+     * Ensure that the code at {@code target} bytes offset from the current {@link #position()} is
+     * aligned according to {@code modulus}.
+     */
+    public void align(int modulus, int target) {
         assert modulus > 0 && (modulus & 0b11) == 0 : "Modulus has to be a positive multiple of 4.";
-        if (position() % modulus == 0) {
-            return;
-        }
-        int offset = modulus - position() % modulus;
-        for (int i = 0; i < offset; i += 4) {
+        int delta = target - position();
+        while ((position() + delta) % modulus != 0) {
             nop();
         }
     }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64Assembler.java

@@ -4952,8 +4952,16 @@ public void nullCheck(AMD64Address address) {
 
     @Override
     public void align(int modulus) {
-        if (position() % modulus != 0) {
-            nop(modulus - (position() % modulus));
+        align(modulus, position());
+    }
+
+    /**
+     * Ensure that the code at {@code target} bytes offset from the current {@link #position()} is
+     * aligned according to {@code modulus}.
+     */
+    public void align(int modulus, int target) {
+        if (target % modulus != 0) {
+            nop(modulus - (target % modulus));
         }
     }
 

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -644,6 +644,9 @@ public boolean supportJVMTIVThreadNotification() {
         return jvmtiVThreadStart != 0L && jvmtiVThreadEnd != 0L && jvmtiVThreadMount != 0L && jvmtiVThreadUnmount != 0L;
     }
 
+    // JDK-8322630
+    public final int icSpeculatedKlassOffset = getFieldOffset("CompiledICData::_speculated_klass", Integer.class, "uintptr_t", Integer.MAX_VALUE, JDK >= 23);
+
     public final int jvmciCountersSize = getFlag("JVMCICounterSize", Integer.class);
 
     // JDK-8231756, GR-16685

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotBackend.java

@@ -27,6 +27,7 @@
 import static java.lang.reflect.Modifier.isStatic;
 import static jdk.vm.ci.aarch64.AArch64.lr;
 import static jdk.vm.ci.aarch64.AArch64.r10;
+import static jdk.vm.ci.aarch64.AArch64.rscratch1;
 import static jdk.vm.ci.aarch64.AArch64.sp;
 import static jdk.vm.ci.aarch64.AArch64.zr;
 import static jdk.vm.ci.code.ValueUtil.asRegister;
@@ -353,25 +354,23 @@ public CompilationResultBuilder newCompilationResultBuilder(LIRGenerationResult
 
     @Override
     public void emitCode(CompilationResultBuilder crb, ResolvedJavaMethod installedCodeOwner, EntryPointDecorator entryPointDecorator) {
-        Label verifiedStub = new Label();
         crb.buildLabelOffsets();
         try {
-            emitCode(crb, installedCodeOwner, verifiedStub, entryPointDecorator);
+            emitCodeHelper(crb, installedCodeOwner, entryPointDecorator);
         } catch (BranchTargetOutOfBoundsException e) {
             // A branch estimation was wrong, now retry with conservative label ranges, this
             // should always work
             crb.setConservativeLabelRanges();
             crb.resetForEmittingCode();
-            verifiedStub.reset();
-            emitCode(crb, installedCodeOwner, verifiedStub, entryPointDecorator);
+            emitCodeHelper(crb, installedCodeOwner, entryPointDecorator);
         }
     }
 
-    private void emitCode(CompilationResultBuilder crb, ResolvedJavaMethod installedCodeOwner, Label verifiedStub, EntryPointDecorator entryPointDecorator) {
+    private void emitCodeHelper(CompilationResultBuilder crb, ResolvedJavaMethod installedCodeOwner, EntryPointDecorator entryPointDecorator) {
         AArch64MacroAssembler masm = (AArch64MacroAssembler) crb.asm;
         FrameMap frameMap = crb.frameMap;
         RegisterConfig regConfig = frameMap.getRegisterConfig();
-        emitCodePrefix(crb, installedCodeOwner, masm, regConfig, verifiedStub);
+        emitCodePrefix(crb, installedCodeOwner, masm, regConfig);
 
         if (entryPointDecorator != null) {
             entryPointDecorator.emitEntryPoint(crb);
@@ -380,33 +379,60 @@ private void emitCode(CompilationResultBuilder crb, ResolvedJavaMethod installed
         emitCodeSuffix(crb, masm);
     }
 
-    private void emitCodePrefix(CompilationResultBuilder crb, ResolvedJavaMethod installedCodeOwner, AArch64MacroAssembler masm, RegisterConfig regConfig, Label verifiedStub) {
+    private void emitCodePrefix(CompilationResultBuilder crb, ResolvedJavaMethod installedCodeOwner, AArch64MacroAssembler masm, RegisterConfig regConfig) {
+        Label verifiedStub = new Label();
         HotSpotProviders providers = getProviders();
         if (installedCodeOwner != null && !isStatic(installedCodeOwner.getModifiers())) {
-            crb.recordMark(HotSpotMarkId.UNVERIFIED_ENTRY);
-            CallingConvention cc = regConfig.getCallingConvention(HotSpotCallingConventionType.JavaCallee, null, new JavaType[]{providers.getMetaAccess().lookupJavaType(Object.class)}, this);
-            // See definition of IC_Klass in c1_LIRAssembler_aarch64.cpp
-            // equal to scratch(1) careful!
-            Register inlineCacheKlass = AArch64HotSpotRegisterConfig.inlineCacheRegister;
+            JavaType[] parameterTypes = {providers.getMetaAccess().lookupJavaType(Object.class)};
+            CallingConvention cc = regConfig.getCallingConvention(HotSpotCallingConventionType.JavaCallee, null, parameterTypes, this);
             Register receiver = asRegister(cc.getArgument(0));
             int size = config.useCompressedClassPointers ? 32 : 64;
             AArch64Address klassAddress = masm.makeAddress(size, receiver, config.hubOffset);
+            if (config.icSpeculatedKlassOffset == Integer.MAX_VALUE) {
+                crb.recordMark(HotSpotMarkId.UNVERIFIED_ENTRY);
+                Register klass = rscratch1;
+                if (config.useCompressedClassPointers) {
+                    masm.ldr(32, klass, klassAddress);
+                    AArch64HotSpotMove.decodeKlassPointer(masm, klass, klass, config.getKlassEncoding());
+                } else {
+                    masm.ldr(64, klass, klassAddress);
+                }
+                // c1_LIRAssembler_aarch64.cpp: const Register IC_Klass = rscratch2;
+                Register inlineCacheKlass = AArch64HotSpotRegisterConfig.inlineCacheRegister;
+                masm.cmp(64, inlineCacheKlass, klass);
 
-            // Are r10 and r11 available scratch registers here? One would hope so.
-            Register klass = r10;
-            if (config.useCompressedClassPointers) {
-                masm.ldr(32, klass, klassAddress);
-                AArch64HotSpotMove.decodeKlassPointer(masm, klass, klass, config.getKlassEncoding());
+                masm.branchConditionally(AArch64Assembler.ConditionFlag.EQ, verifiedStub);
+                AArch64Call.directJmp(crb, masm, getForeignCalls().lookupForeignCall(IC_MISS_HANDLER));
             } else {
-                masm.ldr(64, klass, klassAddress);
+
+                // JDK-8322630 (removed ICStubs)
+                Register data = AArch64HotSpotRegisterConfig.inlineCacheRegister;
+                Register tmp1 = rscratch1;
+                Register tmp2 = r10; // Safe to use R10 as scratch register in method prologue
+                ForeignCallLinkage icMissHandler = getForeignCalls().lookupForeignCall(IC_MISS_HANDLER);
+
+                // Size of IC check sequence checked with a guarantee below.
+                int inlineCacheCheckSize = AArch64Call.isNearCall(icMissHandler) ? 20 : 32;
+                masm.align(config.codeEntryAlignment, masm.position() + inlineCacheCheckSize);
+
+                int startICCheck = masm.position();
+                crb.recordMark(HotSpotMarkId.UNVERIFIED_ENTRY);
+                AArch64Address icSpeculatedKlass = masm.makeAddress(size, data, config.icSpeculatedKlassOffset);
+
+                masm.ldr(size, tmp1, klassAddress);
+                masm.ldr(size, tmp2, icSpeculatedKlass);
+                masm.cmp(size, tmp1, tmp2);
+
+                masm.branchConditionally(AArch64Assembler.ConditionFlag.EQ, verifiedStub);
+                AArch64Call.directJmp(crb, masm, icMissHandler);
+
+                int actualInlineCacheCheckSize = masm.position() - startICCheck;
+                if (actualInlineCacheCheckSize != inlineCacheCheckSize) {
+                    // Code emission pattern has changed: adjust `inlineCacheCheckSize`
+                    // initialization above accordingly.
+                    throw new GraalError("%s != %s", actualInlineCacheCheckSize, inlineCacheCheckSize);
+                }
             }
-            masm.cmp(64, inlineCacheKlass, klass);
-            /*
-             * Conditional jumps have a much lower range than unconditional ones, which can be a
-             * problem because the miss handler could be out of range.
-             */
-            masm.branchConditionally(AArch64Assembler.ConditionFlag.EQ, verifiedStub);
-            AArch64Call.directJmp(crb, masm, getForeignCalls().lookupForeignCall(IC_MISS_HANDLER));
         }
         masm.align(config.codeEntryAlignment);
         masm.bind(verifiedStub);

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMacroAssembler.java

@@ -42,11 +42,10 @@ public AArch64HotSpotMacroAssembler(TargetDescription target, GraalHotSpotVMConf
     }
 
     @Override
-    public void postCallNop(Infopoint infopoint) {
-        if (config.continuationsEnabled && infopoint instanceof Call) {
+    public void postCallNop(Call call) {
+        if (config.continuationsEnabled) {
             // Support for loom requires custom nops after call sites that might deopt
-            Call call = (Call) infopoint;
-            if (call.debugInfo != null && call.reason == InfopointReason.CALL) {
+            if (call.debugInfo != null) {
                 // Expected post call nop pattern taken from
                 // src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp in
                 // MacroAssembler::post_call_nop(). JVMCI will add a relocation during installation.
@@ -60,6 +59,6 @@ public void postCallNop(Infopoint infopoint) {
                 return;
             }
         }
-        super.postCallNop(infopoint);
+        super.postCallNop(call);
     }
 }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotBackend.java

@@ -317,31 +317,65 @@ public void emitCode(CompilationResultBuilder crb, ResolvedJavaMethod installedC
     public void emitCodePrefix(ResolvedJavaMethod installedCodeOwner, CompilationResultBuilder crb, AMD64MacroAssembler asm, RegisterConfig regConfig) {
         HotSpotProviders providers = getProviders();
         if (installedCodeOwner != null && !installedCodeOwner.isStatic()) {
-            crb.recordMark(HotSpotMarkId.UNVERIFIED_ENTRY);
-            CallingConvention cc = regConfig.getCallingConvention(HotSpotCallingConventionType.JavaCallee, null, new JavaType[]{providers.getMetaAccess().lookupJavaType(Object.class)}, this);
-            Register inlineCacheKlass = rax; // see definition of IC_Klass in
-            // c1_LIRAssembler_x86.cpp
+            JavaType[] parameterTypes = {providers.getMetaAccess().lookupJavaType(Object.class)};
+            CallingConvention cc = regConfig.getCallingConvention(HotSpotCallingConventionType.JavaCallee, null, parameterTypes, this);
             Register receiver = asRegister(cc.getArgument(0));
             AMD64Address src = new AMD64Address(receiver, config.hubOffset);
             int before;
-
-            if (config.useCompressedClassPointers) {
-                Register register = r10;
-                Register heapBase = providers.getRegisters().getHeapBaseRegister();
-                AMD64HotSpotMove.decodeKlassPointer(asm, register, heapBase, src, config);
-                if (config.narrowKlassBase != 0) {
-                    // The heap base register was destroyed above, so restore it
-                    if (config.narrowOopBase == 0L) {
-                        asm.xorq(heapBase, heapBase);
-                    } else {
-                        asm.movq(heapBase, config.narrowOopBase);
+            if (config.icSpeculatedKlassOffset == Integer.MAX_VALUE) {
+                crb.recordMark(HotSpotMarkId.UNVERIFIED_ENTRY);
+                // c1_LIRAssembler_x86.cpp: const Register IC_Klass = rax;
+                Register inlineCacheKlass = rax;
+
+                if (config.useCompressedClassPointers) {
+                    Register register = r10;
+                    Register heapBase = providers.getRegisters().getHeapBaseRegister();
+                    AMD64HotSpotMove.decodeKlassPointer(asm, register, heapBase, src, config);
+                    if (config.narrowKlassBase != 0) {
+                        // The heap base register was destroyed above, so restore it
+                        if (config.narrowOopBase == 0L) {
+                            asm.xorq(heapBase, heapBase);
+                        } else {
+                            asm.movq(heapBase, config.narrowOopBase);
+                        }
                     }
+                    before = asm.cmpqAndJcc(inlineCacheKlass, register, ConditionFlag.NotEqual, null, false);
+                } else {
+                    before = asm.cmpqAndJcc(inlineCacheKlass, src, ConditionFlag.NotEqual, null, false);
                 }
-                before = asm.cmpqAndJcc(inlineCacheKlass, register, ConditionFlag.NotEqual, null, false);
+                crb.recordDirectCall(before, asm.position(), getForeignCalls().lookupForeignCall(IC_MISS_HANDLER), null);
             } else {
-                before = asm.cmpqAndJcc(inlineCacheKlass, src, ConditionFlag.NotEqual, null, false);
+                // JDK-8322630 (removed ICStubs)
+                Register data = rax;
+                Register temp = r10;
+                ForeignCallLinkage icMissHandler = getForeignCalls().lookupForeignCall(IC_MISS_HANDLER);
+
+                // Size of IC check sequence checked with a guarantee below.
+                int inlineCacheCheckSize = 14;
+                asm.align(config.codeEntryAlignment, asm.position() + inlineCacheCheckSize);
+
+                int startICCheck = asm.position();
+                crb.recordMark(HotSpotMarkId.UNVERIFIED_ENTRY);
+                AMD64Address icSpeculatedKlass = new AMD64Address(data, config.icSpeculatedKlassOffset);
+
+                AMD64BaseAssembler.OperandSize size;
+                if (config.useCompressedClassPointers) {
+                    asm.movl(temp, src);
+                    size = AMD64BaseAssembler.OperandSize.DWORD;
+                } else {
+                    asm.movptr(temp, src);
+                    size = AMD64BaseAssembler.OperandSize.QWORD;
+                }
+                before = asm.cmpAndJcc(size, temp, icSpeculatedKlass, ConditionFlag.NotEqual, null, false);
+                crb.recordDirectCall(before, asm.position(), icMissHandler, null);
+
+                int actualInlineCacheCheckSize = asm.position() - startICCheck;
+                if (actualInlineCacheCheckSize != inlineCacheCheckSize) {
+                    // Code emission pattern has changed: adjust `inlineCacheCheckSize`
+                    // initialization above accordingly.
+                    throw new GraalError("%s != %s", actualInlineCacheCheckSize, inlineCacheCheckSize);
+                }
             }
-            crb.recordDirectCall(before, asm.position(), getForeignCalls().lookupForeignCall(IC_MISS_HANDLER), null);
         }
 
         asm.align(config.codeEntryAlignment);

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotMacroAssembler.java

@@ -44,11 +44,10 @@ public AMD64HotSpotMacroAssembler(GraalHotSpotVMConfig config, TargetDescription
     }
 
     @Override
-    public void postCallNop(Infopoint infopoint) {
-        if (config.continuationsEnabled && infopoint instanceof Call) {
+    public void postCallNop(Call call) {
+        if (config.continuationsEnabled) {
             // Support for loom requires custom nops after call sites that might deopt
-            Call call = (Call) infopoint;
-            if (call.debugInfo != null && call.reason == InfopointReason.CALL) {
+            if (call.debugInfo != null) {
                 // Expected nop pattern taken from src/hotspot/cpu/x86/macroAssembler_x86.cpp in
                 // MacroAssembler::post_call_nop(). JVMCI will add a relocation during installation.
                 emitByte(0x0f);
@@ -59,7 +58,7 @@ public void postCallNop(Infopoint infopoint) {
                 return;
             }
         }
-        super.postCallNop(infopoint);
+        super.postCallNop(call);
     }
 
     /**