Why podInformer does not work in CI (Unit Tests)?

[jswxstw]

Related PR: #12723

var httpwf = `apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  name: hello-world
  namespace: default
spec:
  entrypoint: http
  templates:
    - name: http
      http:
        url: https://www.google.com/
`

func TestHTTPTemplate(t *testing.T) {
	wf := wfv1.MustUnmarshalWorkflow(httpwf)
	cancel, controller := newController(wf, defaultServiceAccount)
	defer cancel()

	t.Run("ExecuteHTTPTemplate", func(t *testing.T) {
		ctx := context.Background()
		woc := newWorkflowOperationCtx(wf, controller)
		woc.operate(ctx)
		pod, err := controller.kubeclientset.CoreV1().Pods(woc.wf.Namespace).Get(ctx, woc.getAgentPodName(), metav1.GetOptions{})
		assert.NoError(t, err)
		assert.NotNil(t, pod)
		ts, err := controller.wfclientset.ArgoprojV1alpha1().WorkflowTaskSets(wf.Namespace).Get(ctx, "hello-world", metav1.GetOptions{})
		assert.NoError(t, err)
		assert.NotNil(t, ts)
		assert.Len(t, ts.Spec.Tasks, 1)

		// simulate agent pod failure scenario
		pod.Status.Phase = v1.PodFailed
		pod.Status.Message = "manual termination"
		pod, err = controller.kubeclientset.CoreV1().Pods(woc.wf.Namespace).UpdateStatus(ctx, pod, metav1.UpdateOptions{})
		assert.Nil(t, err)
		assert.Equal(t, v1.PodFailed, pod.Status.Phase)
		woc.operate(ctx)
		assert.Equal(t, wfv1.WorkflowError, woc.wf.Status.Phase)
		assert.Equal(t, `agent pod failed with reason:"manual termination"`, woc.wf.Status.Message)
		assert.Len(t, woc.wf.Status.Nodes, 1)
		assert.Equal(t, wfv1.NodeError, woc.wf.Status.Nodes["hello-world"].Phase)
		assert.Equal(t, `agent pod failed with reason:"manual termination"`, woc.wf.Status.Nodes["hello-world"].Message)
		ts, err = controller.wfclientset.ArgoprojV1alpha1().WorkflowTaskSets(wf.Namespace).Get(ctx, "hello-world", metav1.GetOptions{})
		assert.NoError(t, err)
		assert.NotNil(t, ts)
		assert.Empty(t, ts.Spec.Tasks)
		assert.Empty(t, ts.Status.Nodes)
	})
}

TestHTTPTemplate/ExecuteHTTPTemplate:
Local: PASS
CI: FAIL

Local logs:

# first operate
time="2024-03-04T20:01:47+08:00" level=info msg="Processing workflow" Phase= ResourceVersion= namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Task-result reconciliation" namespace=default numObjs=0 workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Updated phase  -> Running" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=warning msg="Node was nil, will be initialized as type Skipped" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="was unable to obtain node for , letting display name to be nodeName" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="HTTP node hello-world initialized Pending" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="TaskSet Reconciliation" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Creating TaskSet" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg=reconcileAgentPod namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Created Agent pod" namespace=default podName=hello-world-1340600742-agent workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion= workflow=hello-world

# second operate
time="2024-03-04T20:01:47+08:00" level=info msg="Processing workflow" Phase=Running ResourceVersion= namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Task-result reconciliation" namespace=default numObjs=0 workflow=hello-world
# `podReconciliation` finds the agent pod error and marks the node to `ERROR`.
time="2024-03-04T20:01:47+08:00" level=info msg=updateAgentPodStatus namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg=assessAgentPodStatus namespace=default podName=hello-world-1340600742-agent
time="2024-03-04T20:01:47+08:00" level=error msg="Mark error node" error="agent pod failed with reason:\"manual termination\"" namespace=default nodeName=hello-world workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="node hello-world phase Pending -> Error" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="node hello-world message: agent pod failed with reason:\"manual termination\"" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="node hello-world finished: 2024-03-04 12:01:47.091854 +0000 UTC" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=error msg="was unable to obtain node for hello-world-2166136261" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="TaskSet Reconciliation" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Creating TaskSet" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg=reconcileAgentPod namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg=updateAgentPodStatus namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg=assessAgentPodStatus namespace=default podName=hello-world-1340600742-agent
# Workflow is marked to `Error` because node phase was marked to `ERROR` in `podReconciliation`.
time="2024-03-04T20:01:47+08:00" level=info msg="Updated phase Running -> Error" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Updated message  -> agent pod failed with reason:\"manual termination\"" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Marking workflow completed" namespace=default workflow=hello-world
time="2024-03-04T20:01:47+08:00" level=info msg="Workflow update successful" namespace=default phase=Error resourceVersion= workflow=hello-world

CI logs:

# first operate
time="2024-03-04T10:10:45Z" level=info msg="Processing workflow" Phase= ResourceVersion= namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="Task-result reconciliation" namespace=default numObjs=0 workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="Updated phase  -> Running" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=warning msg="Node was nil, will be initialized as type Skipped" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="was unable to obtain node for , letting display name to be nodeName" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="HTTP node hello-world initialized Pending" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="TaskSet Reconciliation" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="Creating TaskSet" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg=reconcileAgentPod namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="Created Agent pod" namespace=default podName=hello-world-1340600742-agent workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion= workflow=hello-world

# second operate
time="2024-03-04T10:10:45Z" level=info msg="Processing workflow" Phase=Running ResourceVersion= namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="Task-result reconciliation" namespace=default numObjs=0 workflow=hello-world
# `podReconciliation` is skipped because there is no agent pod in `podInformer`
time="2024-03-04T10:10:45Z" level=info msg="TaskSet Reconciliation" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="Creating TaskSet" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg=reconcileAgentPod namespace=default workflow=hello-world
# can not find agent pod in `podInformer`, so create it again, but it is acturally exists and then return an `AlreadyExists` error.
time="2024-03-04T10:10:45Z" level=info msg="Failed to create Agent pod" error="pods \"hello-world-1340600742-agent\" already exists" namespace=default podName=hello-world-1340600742-agent workflow=hello-world
# `reconcileAgentPod` in `taskSetReconciliation` finds the agent pod error and marks the node to `ERROR`.
time="2024-03-04T10:10:45Z" level=info msg=updateAgentPodStatus namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg=assessAgentPodStatus namespace=default podName=hello-world-1340600742-agent
time="2024-03-04T10:10:45Z" level=error msg="Mark error node" error="agent pod failed with reason:\"manual termination\"" namespace=default nodeName=hello-world workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="node hello-world phase Pending -> Error" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="node hello-world message: agent pod failed with reason:\"manual termination\"" namespace=default workflow=hello-world
time="2024-03-04T10:10:45Z" level=info msg="node hello-world finished: 2024-03-04 10:10:45.035762838 +0000 UTC" namespace=default workflow=hello-world
# Workflow is still `Running` because node phase was not changed in `podReconciliation`.
time="2024-03-04T10:10:45Z" level=info msg="Workflow update successful" namespace=default phase=Running resourceVersion= workflow=hello-world

[jswxstw]

Problem solved: sleep 1 second to wait for informer getting pod info

[agilgur5]

That makes sense. There are some race conditions, and depending on the performance of the machine they could be worse.
Typically GitHub Actions run slower and may even starve CPU and memory as the runners are small. The performance of runners was recently doubled, but that may still be less CPU than local