Rosetta doesn't start during helm deployment

[insider89]

Description

I am using hedera-mirror-node chart to setup mirror node. When I enable rosetta, it doesn't start cause ReadinnesProbe failed. In the logs I see following message:

│ time="2024-05-09T13:08:39Z" level=info msg="::1 POST /network/status (500) in 1.009906ms"                                                                                                                        
│ time="2024-05-09T13:08:39Z" level=error msg="Readiness check, /network/status failed: error {Code:110 Message:Node is starting Description:<nil> Retriable:true Details:map[]}"

My values file is very simple:

            enabled: false,
          },
          global: {
            useReleaseForNameLabel: true,
          },
          graphql: {
            nodeSelector: {
              'kubernetes.io/hostname': args.node.host,
            },
            enabled: true,
            replicas: 2,
            ingress: {
              enabled: false,
            },
          },
          grpc: {
            nodeSelector: {
              'kubernetes.io/hostname': args.node.host,
            },
            replicas: 2,
            config: {
              hedera: {
                mirror: {
                  grpc: {
                    listener: {
                      type: 'NOTIFY',
                    },
                  },
                },
              },
              management: {
                endpoint: {
                  health: {
                    group: {
                      readiness: {
                        exclude: 'redis',
                      },
                    },
                  },
                },
              },
            },
            hpa: {
              enabled: false,
            },
            ingress: {
              enabled: false,
            },
            rbac: {
              enabled: true,
            },
            service: {
              type: 'ClusterIP',
            },
            serviceAccount: {
              create: true,
            },
          },
          importer: {
            nodeSelector: {
              'kubernetes.io/hostname': args.node.host,
            },
            config: {
              hedera: {
                mirror: {
                  importer: {
                    network: args.network,
                    downloader: {
                      cloudProvider: 's3',
                      accessKey: 'ACCESS_KEY',
                      secretKey: 'SECRET_ACCESS_KEY',
                    },
                    parser: {
                      record: {
                        entity: {
                          notify: {
                            enabled: true,
                          },
                          redis: {
                            enabled: true,
                          },
                        },
                      },
                    },
                  },
                },
              },
              management: {
                endpoint: {
                  health: {
                    group: {
                      readiness: {
                        exclude: 'redis',
                      },
                    },
                  },
                },
              },
            },
            rbac: {
              enabled: true,
            },
            replicas: 1,
            resources: {
              requests: {
                cpu: '200m',
                memory: '512Mi',
              },
            },
            serviceAccount: {
              create: true,
            },
          },
          monitor: {
            enabled: false,
          },
          postgresql: {
            postgresql: {
              replicaCount: 2,
              nodeSelector: {
                'kubernetes.io/hostname': args.node.host,
              },
            },
            enabled: true,
            pgpool: {
              replicaCount: 2,
              nodeSelector: {
                'kubernetes.io/hostname': args.node.host,
              },
            },
            serviceAccount: {
              create: false,
            },
          },
          redis: {
            enabled: true,
            replica: {
              replicaCount: 3,
              nodeSelector: {
                'kubernetes.io/hostname': args.node.host,
              },
            },
          },
          rest: {
            nodeSelector: {
              'kubernetes.io/hostname': args.node.host,
            },
            replicaCount: 2,
            config: {
              hedera: {
                mirror: {
                  rest: {
                    metrics: {
                      config: {
                        enabled: false,
                      },
                    },
                  },
                },
              },
            },
            hpa: {
              enabled: false,
            },
            ingress: {
              enabled: false,
            },
            monitor: {
              enabled: false,
            },
            rbac: {
              enabled: true,
            },
            serviceAccount: {
              create: true,
            },
            service: {
              type: 'ClusterIP',
            },
          },
          rosetta: {
            nodeSelector: {
              'kubernetes.io/hostname': args.node.host,
            },
            replicas: 2,
            enabled: true,
            env: {
              CONFIG_PATH: "/usr/etc/hedera/",
              NETWORK: args.network,
            },
            ingress: {
              enabled: false,
            },
          },
          web3: {
            nodeSelector: {
              'kubernetes.io/hostname': args.node.host,
            },
            replicas: 2,
            enabled: true,
            ingress: {
              enabled: false,
            },
          },

Steps to reproduce

1. Install latest helm chart with values provided above
2. Check log of rosetta pod

Additional context

No response

Hedera network

mainnet, testnet

Version

v0.103.0

Operating system

Linux

[xin-hedera]

I believe the mirrornode is not configured to ingest data from open access back in 09/2019 for mainnet, or from genesis in 02/2024 for the current testnet.

This is expected behavior, not really a bug. Code:110 Message:Node is starting generally means rosetta server needs the importer to either ingest or generate the first account balance snapshot to work.

[insider89]

@xin-hedera what is the right configuration for importer to ingest data? I didn't find proper values in the documentation.

[steven-sheehy]

You need to set HEDERA_MIRROR_IMPORTER_STARTDATE to some date in the past that had an address book like 2023-09-19T14:00:00.0Z. But why are you trying to run the Rosetta module? It's an application intended to be solely ran by Coinbase and we don't provide any support for it outside their deployment.

[insider89]

I thought rosetta was mandatory for production running. If rosetta is not mandatory, I keep it disabled. If I don't specify HEDERA_MIRROR_IMPORTER_STARTDATE date in the configuration, from which date it start to sync? If I want to sync from first available date, what date I should use? And how much postgres storage I should use to sync from first available date? Maybe you have some snapshots to restore from it?

[insider89]

If I specify 1970-01-01T00:00:00Z, will it sync all history data?

[insider89]

Got an error with hedera.mirror.importer.startDate=1970-01-01T00:00:00Z :

Caused by: org.postgresql.util.PSQLException: ERROR: null value in column "balance" of relation "account_balance_p2019_09" violates not-null constraint                                                          │
│   Detail: Failing row contains (1568416468456068000, null, 2).                                                                                                                                                   │
│     at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2713)                                                                                                                │
│     at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2401)                                                                                                                      │
│     at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:368)                                                                                                                              │
│     at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:498)                                                                                                                                     │
│     at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:415)                                                                                                                                             │
│     at org.postgresql.jdbc.PgPreparedStatement.executeWithFlags(PgPreparedStatement.java:190)                                                                                                                    │
│     at org.postgresql.jdbc.PgPreparedStatement.executeUpdate(PgPreparedStatement.java:152)                                                                                                                       │
│     at com.zaxxer.hikari.pool.ProxyPreparedStatement.executeUpdate(ProxyPreparedStatement.java:61)                                                                                                               │
│     at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.executeUpdate(HikariProxyPreparedStatement.java)                                                                                                      │
│     at org.hibernate.sql.exec.internal.StandardJdbcMutationExecutor.execute(StandardJdbcMutationExecutor.java:87)                                                                                                │
│     ... 42 common frames omitted