Merge #13282

13282: [sdk/nodejs,python] Add support for explicit providers for packaged components r=justinvp a=justinvp

Add support for explicit providers for packaged components in the Node.js and Python SDKs.

Go already supports it. Node.js briefly supported it with #11093, but the change was reverted in #11509 due to an issue. Python has never supported it.

The PR is broken up into multiple commits for easier reviewing.

Fixes #13074
Part of #11520

Co-authored-by: Justin Van Patten <jvp@justinvp.com>

changelog/pending/20230626--sdk-nodejs-python--support-explicit-providers-for-packaged-components.yaml

@@ -0,0 +1,4 @@
+changes:
+- type: feat
+  scope: sdk/nodejs,python
+  description: Support explicit providers for packaged components

sdk/nodejs/resource.ts

@@ -15,7 +15,7 @@
 import { ResourceError } from "./errors";
 import * as log from "./log";
 import { Input, Inputs, interpolate, Output, output } from "./output";
-import { getResource, readResource, registerResource, registerResourceOutputs } from "./runtime/resource";
+import { getResource, pkgFromType, readResource, registerResource, registerResourceOutputs } from "./runtime/resource";
 import { unknownValue } from "./runtime/rpc";
 import { getProject, getStack } from "./runtime/settings";
 import { getStackResource } from "./runtime/state";
@@ -279,11 +279,10 @@ export abstract class Resource {
 
     // getProvider fetches the provider for the given module member, if any.
     public getProvider(moduleMember: string): ProviderResource | undefined {
-        const memComponents = moduleMember.split(":");
-        if (memComponents.length !== 3) {
+        const pkg = pkgFromType(moduleMember);
+        if (pkg === undefined) {
             return undefined;
         }
-        const pkg = memComponents[0];
 
         return this.__providers[pkg];
     }
@@ -384,12 +383,8 @@ export abstract class Resource {
         // 1. opts.provider
         // 2. a matching provider in opts.providers
         // 3. a matching provider inherited from opts.parent
-        if (custom && opts.provider === undefined) {
-            let pkg = undefined;
-            const memComponents = t.split(":");
-            if (memComponents.length === 3) {
-                pkg = memComponents[0];
-            }
+        if ((custom || remote) && opts.provider === undefined) {
+            const pkg = pkgFromType(t);
             const parentProvider = parent?.getProvider(t);
 
             if (pkg && pkg in this.__providers) {
@@ -400,7 +395,7 @@ export abstract class Resource {
         }
 
         this.__protect = !!opts.protect;
-        this.__prov = custom ? opts.provider : undefined;
+        this.__prov = custom || remote ? opts.provider : undefined;
         this.__version = opts.version;
         this.__pluginDownloadURL = opts.pluginDownloadURL;
 

sdk/nodejs/runtime/resource.ts

@@ -688,11 +688,26 @@ export async function prepareResource(
         // If no parent was provided, parent to the root resource.
         const parentURN = parent ? await parent.urn.promise() : undefined;
 
-        let providerRef: string | undefined;
         let importID: ID | undefined;
         if (custom) {
             const customOpts = <CustomResourceOptions>opts;
             importID = customOpts.import;
+        }
+
+        let providerRef: string | undefined;
+        let sendProvider = custom;
+        if (remote && opts.provider) {
+            // If it's a remote component and a provider was specified, only
+            // send the provider in the request if the provider's package is
+            // the same as the component's package. Otherwise, don't send it
+            // because the user specified `provider: someProvider` as shorthand
+            // for `providers: [someProvider]`.
+            const pkg = pkgFromType(type!);
+            if (pkg && pkg === opts.provider.getPackage()) {
+                sendProvider = true;
+            }
+        }
+        if (sendProvider) {
             providerRef = await ProviderResource.register(opts.provider);
         }
 
@@ -1114,3 +1129,15 @@ function runAsyncResourceOp(label: string, callback: () => Promise<void>, serial
         }
     }
 }
+
+/**
+ * Extract the pkg from the type token of the form "pkg:module:member".
+ * @internal
+ */
+export function pkgFromType(type: string): string | undefined {
+    const parts = type.split(":");
+    if (parts.length === 3) {
+        return parts[0];
+    }
+    return undefined;
+}

sdk/nodejs/tests/runtime/langhost/cases/068.remote_component_providers/index.js

@@ -9,6 +9,12 @@ class Provider extends pulumi.ProviderResource {
     }
 }
 
+class FooProvider extends pulumi.ProviderResource {
+    constructor(name, opts) {
+        super("foo", name, {}, opts);
+    }
+}
+
 class RemoteComponent extends pulumi.ComponentResource {
     constructor(name, opts) {
         super("test:index:Component", name, {}, opts, true /*remote*/);
@@ -20,3 +26,9 @@ const myprovider = new Provider("myprovider");
 new RemoteComponent("singular", { provider: myprovider });
 new RemoteComponent("map", { providers: { test: myprovider } });
 new RemoteComponent("array", { providers: [myprovider] });
+
+const fooprovider = new FooProvider("fooprovider");
+
+new RemoteComponent("foo-singular", { provider: fooprovider });
+new RemoteComponent("foo-map", { providers: { foo: fooprovider } });
+new RemoteComponent("foo-array", { providers: [fooprovider] });

sdk/nodejs/tests/runtime/langhost/run.spec.ts

@@ -1469,7 +1469,7 @@ describe("rpc", () => {
         },
         remote_component_providers: {
             program: path.join(base, "068.remote_component_providers"),
-            expectResourceCount: 4,
+            expectResourceCount: 8,
             registerResource: (
                 ctx: any,
                 dryrun: boolean,
@@ -1489,10 +1489,14 @@ describe("rpc", () => {
                 providers?: any,
             ) => {
                 if (name === "singular" || name === "map" || name === "array") {
-                    assert.strictEqual(provider, "");
+                    assert.strictEqual(provider, "pulumi:providers:test::myprovider::1");
                     assert.deepStrictEqual(Object.keys(providers), ["test"]);
                 }
-                return { urn: makeUrn(t, name), id: undefined, props: undefined };
+                if (name === "foo-singular" || name === "foo-map" || name === "foo-array") {
+                    assert.strictEqual(provider, "");
+                    assert.deepStrictEqual(Object.keys(providers), ["foo"]);
+                }
+                return { urn: makeUrn(t, name), id: name === "myprovider" ? "1" : undefined, props: undefined };
             },
         },
         ambiguous_entrypoints: {

sdk/python/lib/pulumi/resource.py

@@ -34,6 +34,7 @@
 from .metadata import get_project, get_stack
 from .runtime import known_types
 from .runtime.resource import (
+    _pkg_from_type,
     get_resource,
     register_resource,
     register_resource_outputs,
@@ -843,15 +844,11 @@ def __init__(
             # Infer providers and provider maps from parent, if one was provided.
             self._providers = opts.parent._providers
 
-        type_components = t.split(":")
-        pkg = None
-        if len(type_components) == 3:
-            [pkg, _, _] = type_components
-
+        pkg = _pkg_from_type(t)
         opts.provider, opts.providers = self._get_providers(t, pkg, opts)
 
         self._protect = bool(opts.protect)
-        self._provider = opts.provider if custom else None
+        self._provider = opts.provider if (custom or remote) else None
         if self._provider and self._provider.package != pkg:
             action = (
                 "get"
@@ -994,11 +991,10 @@ def get_provider(self, module_member: str) -> Optional["ProviderResource"]:
         :return: The :class:`ProviderResource` associated with the given module member, or None if one does not exist.
         :rtype: Optional[ProviderResource]
         """
-        components = module_member.split(":")
-        if len(components) != 3:
+        pkg = _pkg_from_type(module_member)
+        if pkg is None:
             return None
 
-        [pkg, _, _] = components
         return self._providers.get(pkg)
 
 

sdk/python/lib/pulumi/runtime/resource.py

@@ -193,7 +193,15 @@ async def prepare_resource(
 
     # Construct the provider reference, if we were given a provider to use.
     provider_ref = None
-    if custom and opts is not None and opts.provider is not None:
+    send_provider = custom
+    if remote and opts is not None and opts.provider is not None:
+        # If it's a remote component and a provider was specified, only
+        # send the provider in the request if the provider's package is
+        # the same as the component's package.
+        pkg = _pkg_from_type(ty)
+        if pkg is not None and pkg == opts.provider.package:
+            send_provider = True
+    if send_provider and opts is not None and opts.provider is not None:
         provider = opts.provider
 
         # If we were given a provider, wait for it to resolve and construct a provider reference from it.
@@ -1106,3 +1114,13 @@ async def _resolve_depends_on_urns(
             all_deps.add(direct_dep)
 
     return await rpc._expand_dependencies(all_deps, from_resource)
+
+
+def _pkg_from_type(ty: str) -> Optional[str]:
+    """
+    Extract the pkg from the type token of the form "pkg:module:member".
+    """
+    parts = ty.split(":")
+    if len(parts) != 3:
+        return None
+    return parts[0]

sdk/python/lib/test/langhost/aliases/test_aliases.py

@@ -21,7 +21,7 @@ def test_component_dependencies(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, _protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         return {
             "urn": f"urn:pulumi:stack::project::{ty}::{name}",
             "id": "myID",

sdk/python/lib/test/langhost/asset/test_asset.py

@@ -24,7 +24,7 @@ def test_asset(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         self.assertEqual(ty, "test:index:MyResource")
         if name == "file":
             self.assertIsInstance(_resource["asset"], FileAsset)

sdk/python/lib/test/langhost/chained_failure/test_chained_failure.py

@@ -32,7 +32,7 @@ def test_chained_failure(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         if ty == "test:index:ResourceA":
             self.assertEqual(name, "resourceA")
             self.assertDictEqual(_resource, {"inprop": 777})

sdk/python/lib/test/langhost/component_dependencies/test_component_dependencies.py

@@ -24,7 +24,7 @@ def test_component_dependencies(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
 
         if name == "resD":
             self.assertListEqual(_dependencies, ["resA"], msg=f"{name}._dependencies")

sdk/python/lib/test/langhost/component_provider_resolution/test_component_provider_resolution.py

@@ -24,7 +24,7 @@ def test_component_provider_resolution(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         if name == "combined-mine":
             self.assertTrue(protect)
             self.assertEqual(_provider, "")

sdk/python/lib/test/langhost/component_resource_list_of_providers/test_component_resource_list_of_providers.py

@@ -31,7 +31,7 @@ def test_component_resource_list_of_providers(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         if _custom and not ty.startswith("pulumi:providers:"):
             expect_protect = False
             expect_provider_name = ""

sdk/python/lib/test/langhost/component_resource_single_provider/test_component_resource_single_provider.py

@@ -31,7 +31,7 @@ def test_component_resource_single_provider(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         if _custom and not ty.startswith("pulumi:providers:"):
             expect_protect = False
             expect_provider_name = ""

sdk/python/lib/test/langhost/config/test_config.py

@@ -29,7 +29,7 @@ def test_config(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         self.assertEqual("test:index:MyResource", ty)
         self.assertEqual("myname", name)
         return {

sdk/python/lib/test/langhost/delete_before_replace/test_delete_before_replace.py

@@ -26,7 +26,7 @@ def test_delete_before_replace(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         self.assertEqual("foo", name)
         self.assertTrue(_delete_before_replace)
         return {

sdk/python/lib/test/langhost/first_class_provider/test_first_class_provider.py

@@ -30,7 +30,7 @@ def test_first_class_provider(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         if name == "testprov":
             # Provider resource.
             self.assertEqual("pulumi:providers:test", ty)

sdk/python/lib/test/langhost/first_class_provider_invoke/test_first_class_provider_invoke.py

@@ -53,7 +53,7 @@ def invoke(self, _ctx, token, args, provider, _version):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         if name == "testprov":
             self.assertEqual("pulumi:providers:test", ty)
             self.prov_urn = self.make_urn(ty, name)

sdk/python/lib/test/langhost/first_class_provider_unknown/test_first_class_provider_unknown.py

@@ -32,7 +32,7 @@ def test_first_class_provider_unknown(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         if name == "testprov":
             self.assertEqual("pulumi:providers:test", ty)
             # Only provide an ID when doing an update. When doing a preview the ID will be unknown

sdk/python/lib/test/langhost/future_failure/test_future_failure.py

@@ -28,7 +28,7 @@ def invoke(self, _ctx, token, args, provider, _version):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         self.assertEqual("test:index:MyResource", ty)
         return {
             "urn": self.make_urn(ty, name),

sdk/python/lib/test/langhost/future_input/test_future_input.py

@@ -27,7 +27,7 @@ def test_future_input(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         self.assertEqual(ty, "test:index:FileResource")
         self.assertEqual(name, "file")
         self.assertDictEqual(_resource, {

sdk/python/lib/test/langhost/ignore_changes/test_ignore_changes.py

@@ -26,7 +26,7 @@ def test_ignore_changes(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
 
         # Note that here we expect to receive `ignoredProperty`, even though the user provided `ignored_property`.
         self.assertListEqual(_ignore_changes, ["ignoredProperty", "ignored_property_other"])

sdk/python/lib/test/langhost/inherit_defaults/test_inherit_defaults.py

@@ -31,7 +31,7 @@ def test_inherit_defaults(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         if _custom and not ty.startswith("pulumi:providers:"):
             expect_protect = False
             expect_provider_name = ""

sdk/python/lib/test/langhost/inheritance_translation/test_inheritance_translation.py

@@ -23,7 +23,7 @@ def test_inheritance_translation(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         self.assertEqual("test:index:MyResource", ty)
         return {
             "urn": self.make_urn(ty, name),

sdk/python/lib/test/langhost/inheritance_types/test_inheritance_types.py

@@ -23,7 +23,7 @@ def test_inheritance_types(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         self.assertEqual("test:index:MyResource", ty)
         return {
             "urn": self.make_urn(ty, name),

sdk/python/lib/test/langhost/input_type_mismatch/test_input_type_mismatch.py

@@ -25,7 +25,7 @@ def test_input_type_mismatch(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         self.assertEqual("test:index:MyResource", ty)
 
         policy = _resource["policy"]

sdk/python/lib/test/langhost/input_values_for_outputs/test_input_values_for_outputs.py

@@ -26,7 +26,7 @@ def test_input_values_for_outputs(self):
 
     def register_resource(self, _ctx, _dry_run, ty, name, _resource, _dependencies, _parent, _custom, protect,
                           _provider, _property_deps, _delete_before_replace, _ignore_changes, _version, _import,
-                          _replace_on_changes):
+                          _replace_on_changes, _providers):
         return {
             "urn": self.make_urn(ty, name),
             "id": name,