ServiceMonitor is not discovered

[mirza-ali-ctct]

This seems to be a very trivial issue but I have been struggling to discover the ServiceMonitor. This is how the whole config looks:

apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  creationTimestamp: "2023-08-31T15:54:09Z"
  generation: 6
  labels:
    app: kube-prometheus-stack-prometheus
    app.kubernetes.io/instance: default-ctct-gp-qa-useast1-blue-prometheus
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/part-of: kube-prometheus-stack
    app.kubernetes.io/version: 41.5.1
    argocd.argoproj.io/instance: default-ctct-gp-qa-useast1-blue-prometheus
    chart: kube-prometheus-stack-41.5.1
    heritage: Helm
    release: default-ctct-gp-qa-useast1-blue-prometheus
  name: default-ctct-gp-qa-useast1-prometheus
  namespace: kube-prometheus
  resourceVersion: "220319691"
  uid: 374766dc-c5f4-42ed-8c2c-8240b385649c
spec:
  alerting:
    alertmanagers:
    - apiVersion: v2
      name: default-ctct-gp-qa-useast1-alertmanager
      namespace: kube-prometheus
      pathPrefix: /
      port: http-web
  enableAdminAPI: false
  evaluationInterval: 30s
  externalLabels:
    cluster: default_ctct-gp-qa-useast1-blue-control-plane
  externalUrl: http://default-ctct-gp-qa-useast1-prometheus.kube-prometheus:9090
  image: quay.io/prometheus/prometheus:v2.39.1
  listenLocal: false
  logFormat: logfmt
  logLevel: info
  paused: false
  podMonitorNamespaceSelector: {}
  podMonitorSelector:
    matchLabels:
      release: default-ctct-gp-qa-useast1-blue-prometheus
  portName: http-web
  probeNamespaceSelector: {}
  probeSelector:
    matchLabels:
      release: default-ctct-gp-qa-useast1-blue-prometheus
  prometheusExternalLabelName: ""
  remoteWrite:
  - name: mimir
    url: http://qa.mimir.ctct.io/api/v1/push
    writeRelabelConfigs:
    - action: labeldrop
      regex: (major|minor)
    - action: labeldrop
      regex: (app_framework|app_kubernetes_io_managed_by|install_operator_istio_io_owning_resource)
    - action: labeldrop
      regex: (istio_io_rev|sidecar_istio_io_inject|source_version|version|helm_sh_chart)
    - action: labeldrop
      regex: (install_operator_istio_io_owning_resource|spring-boot|sidecar_istio_io_inject)
    - action: labeldrop
      regex: (security_istio_io_tlsMode|endpoint|connection_security_policy|destination_version)
    - action: labeldrop
      regex: (destination_canonical_revision|Ie|service_istio_io_canonical_revision|source_canonical_revision)
    - action: drop
      regex: apiserver_.*
    - action: drop
      regex: alertmanager_.*
    - action: drop
      regex: grafana_.*
    - action: drop
      regex: jaeger_.*
  replicaExternalLabelName: ""
  replicas: 1
  resources:
    limits:
      cpu: 4
      memory: 12G
    requests:
      cpu: 4
      memory: 12G
  retention: 14d
  routePrefix: /
  ruleNamespaceSelector: {}
  ruleSelector:
    matchLabels:
      release: default-ctct-gp-qa-useast1-blue-prometheus
  scrapeInterval: 1m
  securityContext:
    fsGroup: 2000
    runAsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceAccountName: default-ctct-gp-qa-useast1-prometheus
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}
  shards: 1
  storage:
    volumeClaimTemplate:
      spec:
        resources:
          requests:
            storage: 50Gi
        storageClassName: gp2
  version: v2.39.1
  walCompression: true
status:
  availableReplicas: 1
  conditions:
  - lastTransitionTime: "2023-09-13T15:00:58Z"
    observedGeneration: 6
    status: "True"
    type: Available
  - lastTransitionTime: "2023-09-13T15:00:23Z"
    observedGeneration: 6
    status: "True"
    type: Reconciled
  paused: false
  replicas: 1
  shardStatuses:
  - availableReplicas: 1
    replicas: 1
    shardID: "0"
    unavailableReplicas: 0
    updatedReplicas: 1
  unavailableReplicas: 0
  updatedReplicas: 1

This is the ServiceMonitor:

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    release: default-ctct-gp-qa-useast1-blue-prometheus
  name: test-argocd-controller-metrics
  namespace: kube-prometheus
spec:
  endpoints:
  - port: "metrics"
  namespaceSelector:
    any: true
  selector:
    matchLabels:
      app.kubernetes.io/instance: argo-cd
      app.kubernetes.io/name: argocd-application-controller     

And this is the Service I am trying to scrape:

apiVersion: v1
kind: Service
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"Service","metadata":{"annotations":{},"labels":{"app.kubernetes.io/component":"application-controller","app.kubernetes.io/instance":"argo-cd","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/name":"argocd-metrics","app.kubernetes.io/part-of":"argocd","argocd.argoproj.io/instance":"default-ctct-gp-qa-useast1-blue-argo-cd","helm.sh/chart":"argo-cd-5.24.0"},"name":"argo-cd-argocd-application-controller-metrics","namespace":"argocd"},"spec":{"ports":[{"name":"http-metrics","port":8082,"protocol":"TCP","targetPort":"metrics"}],"selector":{"app.kubernetes.io/instance":"argo-cd","app.kubernetes.io/name":"argocd-application-controller"}}}
  creationTimestamp: "2023-05-02T23:33:42Z"
  labels:
    app.kubernetes.io/component: application-controller
    app.kubernetes.io/instance: argo-cd
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: argocd-metrics
    app.kubernetes.io/part-of: argocd
    argocd.argoproj.io/instance: default-ctct-gp-qa-useast1-blue-argo-cd
    helm.sh/chart: argo-cd-5.24.0
  name: argo-cd-argocd-application-controller-metrics
  namespace: argocd
  resourceVersion: "13683"
  uid: 7a9d495a-a457-4442-a818-1d4291e5d6d4
spec:
  clusterIP: 172.20.88.140
  clusterIPs:
  - 172.20.88.140
  internalTrafficPolicy: Cluster
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - name: http-metrics
    port: 8082
    protocol: TCP
    targetPort: metrics
  selector:
    app.kubernetes.io/instance: argo-cd
    app.kubernetes.io/name: argocd-application-controller
  sessionAffinity: None
  type: ClusterIP
status:
  loadBalancer: {}