Add integration test to check type error display (#360)

* Add integration test to check type error display

This test is unfortunately and necessarily fragile, since it tests what
the user is displayed instead of a structured result.

* Fix copyright lint

* Depend on pulumi 3.41.1

* Change name to avoid confliting with FAIL

* Assert contain instead of equal

* Get test data and add new test

pkg/tests/integration_test.go

@@ -0,0 +1,30 @@
+// Copyright 2022, Pulumi Corporation.  All rights reserved.
+
+package tests
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func integrationDir(dir string) string {
+	return filepath.Join("./testdata", dir)
+}
+
+//nolint:paralleltest // uses parallel programtest
+func TestTypeCheckError(t *testing.T) {
+	testWrapper(t, integrationDir("type-fail"), ExpectFailure, StderrValidator{
+		f: func(t *testing.T, stderr string) {
+			assert.Contains(t, stderr,
+				`Cannot assign '{length: string, lower: number}' to 'random:index/randomString:RandomString':
+
+  length: Cannot assign type 'string' to type 'integer'
+
+  lower: Cannot assign type 'number' to type 'boolean'
+`)
+		},
+	})
+
+}

pkg/tests/testdata/type-fail/Pulumi.yaml

@@ -0,0 +1,8 @@
+name: test-type-fail
+runtime: yaml
+resources:
+  randomString:
+    type: random:RandomString
+    properties:
+      length: "three"
+      lower: 7

pkg/tests/utils.go

@@ -3,6 +3,7 @@
 package tests
 
 import (
+	"bytes"
 	"os"
 	"path/filepath"
 	"testing"
@@ -29,6 +30,8 @@ func prepareYamlProject(*engine.Projinfo) error {
 type testOptions struct {
 	requireLiveRun     *bool
 	programTestOptions integration.ProgramTestOptions
+	// Must be called *after* the test has been run
+	validateStderr func(t *testing.T)
 }
 
 type TestOption interface {
@@ -109,6 +112,26 @@ func (o EditDir) apply(options *testOptions) {
 	options.programTestOptions.EditDirs = append(options.programTestOptions.EditDirs, o.editDir)
 }
 
+type expectFailure struct{}
+
+var ExpectFailure = expectFailure{}
+
+func (expectFailure) apply(options *testOptions) {
+	options.programTestOptions.ExpectFailure = true
+}
+
+type StderrValidator struct {
+	f func(t *testing.T, stderr string)
+}
+
+func (o StderrValidator) apply(options *testOptions) {
+	stderr := &bytes.Buffer{}
+	options.programTestOptions.Stderr = stderr
+	options.validateStderr = func(t *testing.T) {
+		o.f(t, stderr.String())
+	}
+}
+
 type Validator struct {
 	f func(t *testing.T, stack integration.RuntimeValidationStackInfo)
 }
@@ -135,8 +158,7 @@ func testWrapper(t *testing.T, dir string, opts ...TestOption) {
 	var testOptions testOptions
 
 	testOptions.programTestOptions = integration.ProgramTestOptions{
-		Dir: filepath.Join(getCwd(t), dir),
-
+		Dir:            filepath.Join(getCwd(t), dir),
 		PrepareProject: prepareYamlProject,
 	}
 
@@ -162,6 +184,10 @@ func testWrapper(t *testing.T, dir string, opts ...TestOption) {
 	}
 
 	integration.ProgramTest(t, &testOptions.programTestOptions)
+
+	if testOptions.validateStderr != nil {
+		testOptions.validateStderr(t)
+	}
 }
 
 func getCwd(t *testing.T) string {