From 94d03db1730f4f731877d151ff1242f3792fba66 Mon Sep 17 00:00:00 2001
From: Ian Wahbe <ian@wahbe.com>
Date: Mon, 10 Oct 2022 17:57:01 -0700
Subject: [PATCH] Add integration test to check type error display (#360)

* Add integration test to check type error display

This test is unfortunately and necessarily fragile, since it tests what
the user is displayed instead of a structured result.

* Fix copyright lint

* Depend on pulumi 3.41.1

* Change name to avoid confliting with FAIL

* Assert contain instead of equal

* Get test data and add new test
---
 pkg/tests/integration_test.go            | 30 ++++++++++++++++++++++++
 pkg/tests/testdata/type-fail/Pulumi.yaml |  8 +++++++
 pkg/tests/utils.go                       | 30 ++++++++++++++++++++++--
 3 files changed, 66 insertions(+), 2 deletions(-)
 create mode 100644 pkg/tests/integration_test.go
 create mode 100644 pkg/tests/testdata/type-fail/Pulumi.yaml

diff --git a/pkg/tests/integration_test.go b/pkg/tests/integration_test.go
new file mode 100644
index 00000000..6b2444d6
--- /dev/null
+++ b/pkg/tests/integration_test.go
@@ -0,0 +1,30 @@
+// Copyright 2022, Pulumi Corporation.  All rights reserved.
+
+package tests
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func integrationDir(dir string) string {
+	return filepath.Join("./testdata", dir)
+}
+
+//nolint:paralleltest // uses parallel programtest
+func TestTypeCheckError(t *testing.T) {
+	testWrapper(t, integrationDir("type-fail"), ExpectFailure, StderrValidator{
+		f: func(t *testing.T, stderr string) {
+			assert.Contains(t, stderr,
+				`Cannot assign '{length: string, lower: number}' to 'random:index/randomString:RandomString':
+
+  length: Cannot assign type 'string' to type 'integer'
+
+  lower: Cannot assign type 'number' to type 'boolean'
+`)
+		},
+	})
+
+}
diff --git a/pkg/tests/testdata/type-fail/Pulumi.yaml b/pkg/tests/testdata/type-fail/Pulumi.yaml
new file mode 100644
index 00000000..66fb70c1
--- /dev/null
+++ b/pkg/tests/testdata/type-fail/Pulumi.yaml
@@ -0,0 +1,8 @@
+name: test-type-fail
+runtime: yaml
+resources:
+  randomString:
+    type: random:RandomString
+    properties:
+      length: "three"
+      lower: 7
diff --git a/pkg/tests/utils.go b/pkg/tests/utils.go
index 40a25535..9d488e5e 100644
--- a/pkg/tests/utils.go
+++ b/pkg/tests/utils.go
@@ -3,6 +3,7 @@
 package tests
 
 import (
+	"bytes"
 	"os"
 	"path/filepath"
 	"testing"
@@ -29,6 +30,8 @@ func prepareYamlProject(*engine.Projinfo) error {
 type testOptions struct {
 	requireLiveRun     *bool
 	programTestOptions integration.ProgramTestOptions
+	// Must be called *after* the test has been run
+	validateStderr func(t *testing.T)
 }
 
 type TestOption interface {
@@ -109,6 +112,26 @@ func (o EditDir) apply(options *testOptions) {
 	options.programTestOptions.EditDirs = append(options.programTestOptions.EditDirs, o.editDir)
 }
 
+type expectFailure struct{}
+
+var ExpectFailure = expectFailure{}
+
+func (expectFailure) apply(options *testOptions) {
+	options.programTestOptions.ExpectFailure = true
+}
+
+type StderrValidator struct {
+	f func(t *testing.T, stderr string)
+}
+
+func (o StderrValidator) apply(options *testOptions) {
+	stderr := &bytes.Buffer{}
+	options.programTestOptions.Stderr = stderr
+	options.validateStderr = func(t *testing.T) {
+		o.f(t, stderr.String())
+	}
+}
+
 type Validator struct {
 	f func(t *testing.T, stack integration.RuntimeValidationStackInfo)
 }
@@ -135,8 +158,7 @@ func testWrapper(t *testing.T, dir string, opts ...TestOption) {
 	var testOptions testOptions
 
 	testOptions.programTestOptions = integration.ProgramTestOptions{
-		Dir: filepath.Join(getCwd(t), dir),
-
+		Dir:            filepath.Join(getCwd(t), dir),
 		PrepareProject: prepareYamlProject,
 	}
 
@@ -162,6 +184,10 @@ func testWrapper(t *testing.T, dir string, opts ...TestOption) {
 	}
 
 	integration.ProgramTest(t, &testOptions.programTestOptions)
+
+	if testOptions.validateStderr != nil {
+		testOptions.validateStderr(t)
+	}
 }
 
 func getCwd(t *testing.T) string {