io.grpc.StatusRuntimeException: INVALID_ARGUMENT: etcdserver: revision of auth store is old - client doesn't retry.

[vivekpatani]

Versions

• etcd: 3.5.9
• jetcd: main
• java: openjdk 20.0.2 2023-07-18

Describe the bug
When spinning up etcd with --auth-token=jwt,pub-key=jwt_RS256.pub,priv-key=jwt_RS256,sign-method=RS256 --auth-token-ttl=1, run into this io.grpc.StatusRuntimeException: INVALID_ARGUMENT: etcdserver: revision of auth store is old and the client fails to retry.

To Reproduce

• Create a tmp directory
• cd tmp
• Create a Dockerfile

FROM gcr.io/etcd-development/etcd:v3.5.9

COPY jwt_RS256 /usr/local/bin/jwt_RS256
COPY jwt_RS256 /var/etcd/jwt_RS256
COPY jwt_RS256 /var/lib/etcd/jwt_RS256
COPY jwt_RS256 /jwt_RS256

COPY jwt_RS256.pub /usr/local/bin/jwt_RS256.pub
COPY jwt_RS256.pub /var/etcd/jwt_RS256.pub
COPY jwt_RS256.pub /var/lib/etcd/jwt_RS256.pub
COPY jwt_RS256.pub /jwt_RS256.pub

ENV ETCD_UNSUPPORTED_ARCH=arm64

EXPOSE 2379 2380

# Define default command.
CMD ["/usr/local/bin/etcd"]

• Create random keys

openssl genrsa -out jwt_RS256 4096
openssl rsa -in jwt_RS256 -pubout > jwt_RS256.pub

• docker build -t etcd-custom .
• Open jetcd project, and create AuthClientTTLTest.java

package io.etcd.jetcd.impl;

import io.etcd.jetcd.Auth;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.Client;
import io.etcd.jetcd.KV;
import io.etcd.jetcd.auth.AuthRoleListResponse;
import io.etcd.jetcd.auth.Permission;
import io.etcd.jetcd.kv.DeleteResponse;
import io.etcd.jetcd.kv.GetResponse;
import io.etcd.jetcd.test.EtcdClusterExtension;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.extension.RegisterExtension;

import java.util.List;
import java.util.concurrent.TimeUnit;

import static io.etcd.jetcd.impl.TestUtil.bytesOf;
import static org.assertj.core.api.Assertions.assertThat;

@Timeout(value = 30, unit = TimeUnit.SECONDS)
public class AuthClientTTLTest {
    @RegisterExtension
    public static final EtcdClusterExtension cluster = EtcdClusterExtension.builder()
            .withNodes(1)
            .withAdditionalArgs(List.of("--auth-token=jwt,pub-key=jwt_RS256.pub,priv-key=jwt_RS256,sign-method=RS256", "--auth-token-ttl=1"))
            .withImage("etcd-custom")
            .build();

    private static final String rootString = "root";
    private static final ByteSequence rootPass = bytesOf("123");
    private static final String rootRoleString = "root";
    private static final String userString = "user";
    private static final String userRoleString = "userRole";
    private static Auth authDisabledAuthClient;
    private static KV authDisabledKVClient;
    private final ByteSequence rootRoleKey = bytesOf("root");
    private final ByteSequence rootRoleValue = bytesOf("b");
    private final ByteSequence rootRoleKeyRangeBegin = bytesOf("root");
    private final ByteSequence rootRoleKeyRangeEnd = bytesOf("root1");
    private final ByteSequence userRoleKey = bytesOf("foo");
    private final ByteSequence userRoleValue = bytesOf("bar");
    private final ByteSequence userRoleKeyRangeBegin = bytesOf("foo");
    private final ByteSequence userRoleKeyRangeEnd = bytesOf("foo1");
    private final ByteSequence root = bytesOf(rootString);
    private final ByteSequence rootRole = bytesOf(rootRoleString);
    private final ByteSequence user = bytesOf(userString);
    private final ByteSequence userPass = bytesOf("userPass");
    private final ByteSequence userNewPass = bytesOf("newUserPass");
    private final ByteSequence userRole = bytesOf(userRoleString);
    private final ByteSequence testKey = bytesOf("test_key");
    private final ByteSequence testValue = bytesOf("test_value");

    /**
     * Build etcd client to create role, permission.
     */
    @BeforeAll
    public static void setupEnv() {
        Client client = TestUtil.client(cluster).build();
        authDisabledAuthClient = client.getAuthClient();
        authDisabledKVClient = client.getKVClient();
    }

    @Test
    public void testAuth() throws Exception {
        // add root auth role, list auth role, and verify
        authDisabledAuthClient.roleAdd(rootRole).get();
        final AuthRoleListResponse response = authDisabledAuthClient.roleList().get();
        assertThat(response.getRoles()).containsOnly(rootRoleString);

        // role grant permission
        authDisabledAuthClient
                .roleGrantPermission(rootRole, rootRoleKeyRangeBegin, rootRoleKeyRangeEnd, Permission.Type.READWRITE).get();


        // add root user, list users and verify
        authDisabledAuthClient.userAdd(root, rootPass).get();
        List<String> users = authDisabledAuthClient.userList().get().getUsers();
        assertThat(users).containsOnly(rootString);

        // role granted to user and verify
        authDisabledAuthClient.userGrantRole(root, rootRole).get();
        assertThat(authDisabledAuthClient.userGet(root).get().getRoles()).containsOnly(rootRoleString);

        // enable auth
        authDisabledAuthClient.authEnable().get();

        // create an auth enabled root client
        final Client authEnabledRootClient = TestUtil.client(cluster).user(root).password(rootPass).build();
        final Auth authEnabledAuthClient = authEnabledRootClient.getAuthClient();
        final KV authEnabledKVClient = authEnabledRootClient.getKVClient();

        authEnabledKVClient.put(testKey, testValue).get();
        final GetResponse getResponse = authEnabledKVClient.get(testKey).get();

        authEnabledAuthClient.roleAdd(userRole).get();
        authEnabledAuthClient.roleGrantPermission(userRole, userRoleKeyRangeBegin, userRoleKeyRangeEnd, Permission.Type.READWRITE).get();
        authEnabledAuthClient.userAdd(user, userPass).get();
        authEnabledAuthClient.userGrantRole(user, userRole);

        Thread.sleep(5000);

        DeleteResponse deleteResponse = authEnabledKVClient.delete(testKey).get();
    }
}

Expected behavior
Client should retry.

Additional context
I see there was a fix for this: #1103, but maybe I'm missing context here.

If it's easier to recreate this issue, I've made a standalone app so that you can hit the etcd-server

/*
 * This Java source file was generated by the Gradle 'init' task.
 */
package jetcd.sample;

import io.etcd.jetcd.Auth;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.Client;
import io.etcd.jetcd.KV;
import io.etcd.jetcd.auth.Permission;
import io.etcd.jetcd.kv.GetResponse;

import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;

public class App {
    public String getGreeting() {
        return "Hello World!";
    }

    public static void main(String[] args) throws ExecutionException, InterruptedException {
        System.out.println(new App().getGreeting());
        final ByteSequence user = ByteSequence.from("root".getBytes());
        final ByteSequence pass = ByteSequence.from("root".getBytes());

        // create client using endpoints
        final  Client client1 = Client.builder()
                .user(user)
                .password(user)
                .endpoints("http://localhost:2379")
                .build();

        // everything works fine
        final KV kvClient = client1.getKVClient();
        final ByteSequence key = ByteSequence.from("test_key".getBytes());
        final ByteSequence value = ByteSequence.from("test_value".getBytes());

        // put the key-value
        kvClient.put(key, value).get();

        // get the CompletableFuture
        CompletableFuture<GetResponse> getFuture = kvClient.get(key);

        // get the value from CompletableFuture
        GetResponse response = getFuture.get();

        System.out.println(response.toString());

        // create client using endpoints
        final  Client client2 = Client.builder()
                .user(user)
                .password(user)
                .endpoints("http://localhost:2379")
                .build();

        final Auth authClient = client2.getAuthClient();
        final ByteSequence role0 = ByteSequence.from("role0".getBytes());
        final ByteSequence user0 = ByteSequence.from("user0".getBytes());
        authClient.roleAdd(role0);
        authClient.roleGrantPermission(role0, key, key, Permission.Type.READWRITE);
        authClient.userAdd(user0, pass);
        authClient.userGrantRole(user0, role0);

        Thread.sleep(11000);

        // should fail
        kvClient.delete(key).get();

        // close
        client1.close();

        System.out.println(new App().getGreeting());
    }
}

[lburgazzoli]

can you submit a PR with the failing test ? I know there is some code here but a PR would eventually speed up the implementation of a fix

[vivekpatani]

@lburgazzoli i can submit a PR but the test needs a custom image, which is why i had to submit it like this.

Currently the test setup doesn't allow for certificates, it takes the etcd image from upstream and uses that, but in order to reproduce the problem you need to pass a custom etcd with certificates already present which is why I've jotted the steps down. Not sure if it makes sense. Thanks.

[lburgazzoli]

we have some tests that add the certificates and some that use jwt too, like

jetcd/jetcd-core/src/test/java/io/etcd/jetcd/impl/WatchTokenExpireTest.java

Lines 52 to 59 in b95fc68

	public static final EtcdClusterExtension cluster = EtcdClusterExtension.builder()
	.withNodes(1)
	.withSsl(true)
	.withAdditionalArgs(
	List.of(
	"--auth-token",
	"jwt,pub-key=/etc/ssl/etcd/server.pem,priv-key=/etc/ssl/etcd/server-key.pem,sign-method=RS256,ttl=1s"))
	.build();

[vivekpatani]

That makes more sense, let me fix the test and submit a PR in sometime. Thanks.