cpu: support darwin/arm64 CPU feature detection

Support ARM64 features detection. The CPU features which are supported by
Apple Silicon M1 are assumed as the minimal set of features for Go programs
running on darwin/arm64.

The ARM64 supporting features are referred to
https://en.wikichip.org/wiki/arm/armv8#ARMv8_Extensions_and_Processor_Features

cpu/cpu.go

@@ -100,6 +100,7 @@ var ARM64 struct {
 	HasSHA512   bool // SHA512 hardware implementation
 	HasSVE      bool // Scalable Vector Extensions
 	HasASIMDFHM bool // Advanced SIMD multiplication FP16 to FP32
+	HasFMA      bool // Fused Multiply Add
 	_           CacheLinePad
 }
 

cpu/cpu_android_arm64.go

@@ -0,0 +1,78 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build android
+
+package cpu
+
+// HWCAP/HWCAP2 bits. These are exposed by Linux.
+const (
+	hwcap_FP       = 1 << 0
+	hwcap_ASIMD    = 1 << 1
+	hwcap_EVTSTRM  = 1 << 2
+	hwcap_AES      = 1 << 3
+	hwcap_PMULL    = 1 << 4
+	hwcap_SHA1     = 1 << 5
+	hwcap_SHA2     = 1 << 6
+	hwcap_CRC32    = 1 << 7
+	hwcap_ATOMICS  = 1 << 8
+	hwcap_FPHP     = 1 << 9
+	hwcap_ASIMDHP  = 1 << 10
+	hwcap_CPUID    = 1 << 11
+	hwcap_ASIMDRDM = 1 << 12
+	hwcap_JSCVT    = 1 << 13
+	hwcap_FCMA     = 1 << 14
+	hwcap_LRCPC    = 1 << 15
+	hwcap_DCPOP    = 1 << 16
+	hwcap_SHA3     = 1 << 17
+	hwcap_SM3      = 1 << 18
+	hwcap_SM4      = 1 << 19
+	hwcap_ASIMDDP  = 1 << 20
+	hwcap_SHA512   = 1 << 21
+	hwcap_SVE      = 1 << 22
+	hwcap_ASIMDFHM = 1 << 23
+)
+
+func osInit() {
+	if err := readHWCAP(); err != nil {
+		// failed to read /proc/self/auxv, try reading registers directly
+		readARM64Registers()
+		return
+	}
+
+	// Use HWCap information since reading aarch64 system registers
+	// is not supported in user space on older linux kernels.
+	ARM64.HasFP = isSet(hwCap, hwcap_FP)
+	ARM64.HasASIMD = isSet(hwCap, hwcap_ASIMD)
+	ARM64.HasEVTSTRM = isSet(hwCap, hwcap_EVTSTRM)
+	ARM64.HasAES = isSet(hwCap, hwcap_AES)
+	ARM64.HasPMULL = isSet(hwCap, hwcap_PMULL)
+	ARM64.HasSHA1 = isSet(hwCap, hwcap_SHA1)
+	ARM64.HasSHA2 = isSet(hwCap, hwcap_SHA2)
+	ARM64.HasCRC32 = isSet(hwCap, hwcap_CRC32)
+	ARM64.HasFPHP = isSet(hwCap, hwcap_FPHP)
+	ARM64.HasASIMDHP = isSet(hwCap, hwcap_ASIMDHP)
+	ARM64.HasASIMDRDM = isSet(hwCap, hwcap_ASIMDRDM)
+	ARM64.HasJSCVT = isSet(hwCap, hwcap_JSCVT)
+	ARM64.HasFCMA = isSet(hwCap, hwcap_FCMA)
+	ARM64.HasLRCPC = isSet(hwCap, hwcap_LRCPC)
+	ARM64.HasDCPOP = isSet(hwCap, hwcap_DCPOP)
+	ARM64.HasSHA3 = isSet(hwCap, hwcap_SHA3)
+	ARM64.HasSM3 = isSet(hwCap, hwcap_SM3)
+	ARM64.HasSM4 = isSet(hwCap, hwcap_SM4)
+	ARM64.HasASIMDDP = isSet(hwCap, hwcap_ASIMDDP)
+	ARM64.HasSHA512 = isSet(hwCap, hwcap_SHA512)
+	ARM64.HasSVE = isSet(hwCap, hwcap_SVE)
+	ARM64.HasASIMDFHM = isSet(hwCap, hwcap_ASIMDFHM)
+
+	// The Samsung S9+ kernel reports support for atomics, but not all cores
+	// actually support them, resulting in SIGILL. See issue #28431.
+	// TODO(elias.naur): Only disable the optimization on bad chipsets on android.
+	ARM64.HasATOMICS = false
+}
+
+func isSet(hwc uint, value uint) bool {
+	return hwc&value != 0
+}

cpu/cpu_arm64.go

@@ -41,8 +41,8 @@ func archInit() {
 	switch runtime.GOOS {
 	case "freebsd":
 		readARM64Registers()
-	case "linux", "netbsd":
-		doinit()
+	case "linux", "netbsd", "android", "darwin":
+		osInit()
 	default:
 		// Most platforms don't seem to allow reading these registers.
 		//
@@ -52,13 +52,6 @@ func archInit() {
 	}
 }
 
-// setMinimalFeatures fakes the minimal ARM64 features expected by
-// TestARM64minimalFeatures.
-func setMinimalFeatures() {
-	ARM64.HasASIMD = true
-	ARM64.HasFP = true
-}
-
 func readARM64Registers() {
 	Initialized = true
 
@@ -170,3 +163,10 @@ func parseARM64SystemRegisters(isar0, isar1, pfr0 uint64) {
 func extractBits(data uint64, start, end uint) uint {
 	return (uint)(data>>start) & ((1 << (end - start + 1)) - 1)
 }
+
+// setMinimalFeatures fakes the minimal ARM64 features expected by
+// TestARM64minimalFeatures.
+func setMinimalFeatures() {
+	ARM64.HasASIMD = true
+	ARM64.HasFP = true
+}

cpu/cpu_darwin_arm64.go

@@ -0,0 +1,58 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build darwin
+// +build !ios
+
+package cpu
+
+const (
+	commpageHasNeonFP16      uint64 = 0x00000008         // ARM v8.2 NEON FP16 supported
+	commpageHasNeon          uint64 = 0x00000100         // Advanced SIMD is supported
+	commpageHasNeonHPFP      uint64 = 0x00000200         // Advanced SIMD half-precision
+	commpageHasVfp           uint64 = 0x00000400         // VFP is supported
+	commpageHasEvent         uint64 = 0x00001000         // WFE/SVE and period event wakeup
+	commpageHasFMA           uint64 = 0x00002000         // Fused multiply add is supported
+	commpageHasARMv82FHM     uint64 = 0x00004000         // Optional ARMv8.2 FMLAL/FMLSL instructions (required in ARMv8.4)
+	commpageHasARMv8Crypto   uint64 = 0x01000000         // Optional ARMv8 Crypto extensions
+	commpageHasARMv81Atomics uint64 = 0x02000000         // ARMv8.1 Atomic instructions supported
+	commpageHasARMv8Crc32    uint64 = 0x04000000         // Optional ARMv8 crc32 instructions (required in ARMv8.1)
+	commpageHasARMv82SHA512  uint64 = 0x80000000         // Optional ARMv8.2 SHA512 instructions
+	commpageHasARMv82SHA3    uint64 = 0x0000000100000000 // Optional ARMv8.2 SHA3 instructions
+)
+
+func osInit() {
+	ARM64.HasFP = darwinFeatureEnable(commpageHasVfp)
+	ARM64.HasASIMD = darwinFeatureEnable(commpageHasNeon)
+	ARM64.HasCRC32 = darwinFeatureEnable(commpageHasARMv8Crc32)
+	ARM64.HasATOMICS = darwinFeatureEnable(commpageHasARMv81Atomics)
+	ARM64.HasFPHP = darwinFeatureEnable(commpageHasNeonFP16)
+	ARM64.HasASIMDHP = darwinFeatureEnable(commpageHasNeonHPFP)
+	ARM64.HasSHA3 = darwinFeatureEnable(commpageHasARMv82SHA3)
+	ARM64.HasSHA512 = darwinFeatureEnable(commpageHasARMv82SHA512)
+	ARM64.HasASIMDFHM = darwinFeatureEnable(commpageHasARMv82FHM)
+	ARM64.HasSVE = darwinFeatureEnable(commpageHasEvent)
+	ARM64.HasFMA = darwinFeatureEnable(commpageHasFMA)
+	// There are no hw.optional sysctl values for the below features on Mac OS 11.0
+	// to detect their supported state dynamically. Assume the CPU features that
+	// Apple Silicon M1 supports to be available as a minimal set of features
+	// to all Go programs running on darwin/arm64.
+	ARM64.HasEVTSTRM = true
+	ARM64.HasAES = true
+	ARM64.HasPMULL = true
+	ARM64.HasSHA1 = true
+	ARM64.HasSHA2 = true
+	ARM64.HasCPUID = true
+	ARM64.HasASIMDRDM = true
+	ARM64.HasJSCVT = true
+	ARM64.HasFCMA = true
+	ARM64.HasLRCPC = true
+	ARM64.HasDCPOP = true
+	ARM64.HasSM3 = true
+	ARM64.HasSM4 = true
+	ARM64.HasASIMDDP = true
+}
+
+func darwinFeatureEnable(feature_vec uint64) bool

cpu/cpu_darwin_arm64.s

@@ -0,0 +1,25 @@
+//go:build arm64 && gc
+// +build arm64
+// +build gc
+
+#include "textflag.h"
+
+// func darwinFeatureEnable(feature_vec uint64) bool
+TEXT ·darwinFeatureEnable(SB), NOSPLIT, $0-8
+    MOVD    feature_vec+0(FP), R0
+    MOVD    $0, ret+0(FP) // default to false
+    MOVD    $1, R3        // set R3 as true boolean constan
+#ifdef GOOS_darwin   // return if not darwin
+#ifdef GOARCH_arm64  // return if not amd64
+// These values from:
+// https://github.com/apple/darwin-xnu/blob/main/osfmk/arm/cpu_capabilities.h
+#define commpage64_base_address         0x0000000fffffc000
+#define commpage64_cpu_capabilities64   (commpage64_base_address+0x010)
+    MOVD    $commpage64_cpu_capabilities64, R1
+    AND   R0, R1, R2
+    CBZ      R2, no_feature
+    MOVD    R3, ret+0(FP)
+no_feature:
+#endif
+#endif
+    RET

cpu/cpu_freebsd_arm64.go

@@ -0,0 +1,12 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build arm64
+// +build freebsd
+
+package cpu
+
+func osInit() {
+	readARM64Registers()
+}

cpu/cpu_linux_arm64.go

@@ -2,6 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+// +build arm64
+// +build linux
+// +build !android
+
 package cpu
 
 // HWCAP/HWCAP2 bits. These are exposed by Linux.
@@ -32,7 +36,7 @@ const (
 	hwcap_ASIMDFHM = 1 << 23
 )
 
-func doinit() {
+func osInit() {
 	if err := readHWCAP(); err != nil {
 		// failed to read /proc/self/auxv, try reading registers directly
 		readARM64Registers()

cpu/cpu_other_arm64.go

@@ -2,9 +2,11 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:build !linux && !netbsd && arm64
-// +build !linux,!netbsd,arm64
+//go:build !linux && !netbsd && !darwin && arm64
+// +build !linux,!netbsd,!darwin,arm64
 
 package cpu
 
-func doinit() {}
+func osInit() {
+	setMinimalFeatures()
+}

cpu/cpu_test.go

@@ -42,7 +42,7 @@ func TestAVX512HasAVX2AndAVX(t *testing.T) {
 }
 
 func TestARM64minimalFeatures(t *testing.T) {
-	if runtime.GOARCH != "arm64" || (runtime.GOOS == "darwin" || runtime.GOOS == "ios") {
+	if runtime.GOARCH != "arm64" || runtime.GOOS == "ios" {
 		return
 	}
 	if !cpu.ARM64.HasASIMD {