vault serviceAccount authentication failing on vault-crd-chart 1.6.1

[pthornton]

values.yaml:
# Specifies the used authentication method the following values are allowed: token | serviceAccount
vaultAuth: serviceAccount
# Token with access to the resources that Vault-CRD shares from Vault to Kubernetes. Required if vaultAuth = token
vaultToken: ""
# Path to authentication backend in HashiCorp Vault. Only used if vaultAuth = serviceAccount
vaultAuthPath: auth/dev-kubernetes
vaultRole: "dev-role"

I have vault sidecars installed in the cluster and working with service accounts with these same credentials.

2021-02-05 16:27:31.758 INFO 1 --- [TaskScheduler-1] d.k.v.k.scheduler.ScheduledRefresh : Refresh of secret assets-cred in namespace dnext failed with exception

de.koudingspawn.vault.vault.communication.SecretNotAccessibleException: Couldn't load secret from vault path dev-kv/gitlab/credentials/registry/high-five
at de.koudingspawn.vault.vault.VaultCommunication.getVersionedSecret(VaultCommunication.java:136) ~[classes!/:0.0.1-SNAPSHOT]
at de.koudingspawn.vault.vault.VaultCommunication.getDockerCfg(VaultCommunication.java:67) ~[classes!/:0.0.1-SNAPSHOT]
at de.koudingspawn.vault.vault.impl.DockerCfgGenerator.getHash(DockerCfgGenerator.java:35) ~[classes!/:0.0.1-SNAPSHOT]
at de.koudingspawn.vault.kubernetes.scheduler.impl.DockerCfgRefresh.dockerCfgHashHasChanged(DockerCfgRefresh.java:36) ~[classes!/:0.0.1-SNAPSHOT]
at de.koudingspawn.vault.kubernetes.scheduler.impl.DockerCfgRefresh.refreshIsNeeded(DockerCfgRefresh.java:30) ~[classes!/:0.0.1-SNAPSHOT]
at de.koudingspawn.vault.kubernetes.scheduler.ScheduledRefresh.refreshCertificates(ScheduledRefresh.java:43) ~[classes!/:0.0.1-SNAPSHOT]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_275]
at org.springframework.scheduling.support.ScheduledMethodRunnable.run(ScheduledMethodRunnable.java:84) [spring-context-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) [spring-context-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_275]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_275]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_275]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_275]
Caused by: org.springframework.vault.authentication.VaultLoginException: Cannot login using Kubernetes: missing client token; nested exception is org.springframework.web.client.HttpClientErrorException$BadRequest: 400 Bad Request: [{"errors":["missing client token"]}
]
at org.springframework.vault.authentication.VaultLoginException.create(VaultLoginException.java:64) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.vault.authentication.KubernetesAuthentication.login(KubernetesAuthentication.java:107) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.vault.authentication.LifecycleAwareSessionManager.doGetSessionToken(LifecycleAwareSessionManager.java:291) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.vault.authentication.LifecycleAwareSessionManager.getSessionToken(LifecycleAwareSessionManager.java:277) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.vault.core.VaultTemplate.lambda$getSessionInterceptor$1(VaultTemplate.java:276) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:93) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.vault.client.VaultClients.lambda$createRestTemplate$0(VaultClients.java:128) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:93) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.http.client.InterceptingClientHttpRequest.executeInternal(InterceptingClientHttpRequest.java:77) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:742) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:677) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:586) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.vault.core.VaultVersionedKeyValueTemplate.lambda$doRead$0(VaultVersionedKeyValueTemplate.java:103) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.vault.core.VaultTemplate.doWithSession(VaultTemplate.java:466) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.vault.core.VaultVersionedKeyValueTemplate.doRead(VaultVersionedKeyValueTemplate.java:100) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.vault.core.VaultVersionedKeyValueTemplate.get(VaultVersionedKeyValueTemplate.java:89) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at org.springframework.vault.core.VaultVersionedKeyValueOperations.get(VaultVersionedKeyValueOperations.java:72) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
at de.koudingspawn.vault.vault.VaultCommunication.getVersionedSecret(VaultCommunication.java:125) ~[classes!/:0.0.1-SNAPSHOT]
... 18 common frames omitted
Caused by: org.springframework.web.client.HttpClientErrorException$BadRequest: 400 Bad Request: [{"errors":["missing client token"]}
]
at org.springframework.web.client.HttpClientErrorException.create(HttpClientErrorException.java:101) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.DefaultResponseErrorHandler.handleError(DefaultResponseErrorHandler.java:170) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.DefaultResponseErrorHandler.handleError(DefaultResponseErrorHandler.java:112) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.ResponseErrorHandler.handleError(ResponseErrorHandler.java:63) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.RestTemplate.handleResponse(RestTemplate.java:785) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:743) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:677) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.web.client.RestTemplate.postForObject(RestTemplate.java:421) ~[spring-web-5.2.3.RELEASE.jar!/:5.2.3.RELEASE]
at org.springframework.vault.authentication.KubernetesAuthentication.login(KubernetesAuthentication.java:96) ~[spring-vault-core-2.2.1.RELEASE.jar!/:2.2.1.RELEASE]
... 36 common frames omitted

[pthornton]

Also, the vault-crd-dev-vault-token secret seems to contain a valid token and this token has permission to read the secret "dev-kv/gitlab/credentials/registry/high-five". Validated using vault command line.

[felipem1210]

Hi @pthornton

Hi faced the same issue and it was because I was passing wrong the name of the vaultAuthPath.
It must be the path that you used when you created the kubernetes secret engine in Vault.

In the tool's documentation is this part:
vault auth enable kubernetes
It by default creates a path named kubernetes. Check that and correct variable and maybe that makes the trick