Consul sidecar does not work with vault in EKS

Consul
1

I have set up vault and consul with helm in EKS. I can set up consul as the storage backend for vault with no issues when connecting directly to consul clients or consul service in EKS. For example:

 storage "consul" {
   address = "https://consul-staging-server.consul-staging.svc:8501"
   path    = "vault/"
   tls_ca_file = "/vault/userconfig/consul-tls-ca/ca.crt"
 }

But when I enable consul connect inject, vault can no longer connect to consul. Connect-inject can successfully add a consul-dataplane but Vault gets stuck on

2025-03-13T22:04:32.060Z [WARN]  storage migration check error: error="Get \"https://172.16.41.238:8500/v1/kv/vault/core/migration\": read tcp 172.16.41.126:57130->172.16.41.238:8500: read: connection reset by peer"
2025-03-13T22:04:34.061Z [WARN]  storage migration check error: error="Get \"https://172.16.41.238:8500/v1/kv/vault/core/migration\": read tcp 172.16.41.126:57148->172.16.41.238:8500: read: connection reset by peer"
2025-03-13T22:04:36.063Z [WARN]  storage migration check error: error="Get \"https://172.16.41.238:8500/v1/kv/vault/core/migration\": read tcp 172.16.41.126:57168->172.16.41.238:8500: read: connection reset by peer"
2025-03-13T22:04:38.064Z [WARN]  storage migration check error: error="Get \"https://172.16.41.238:8500/v1/kv/vault/core/migration\": read tcp 172.16.41.126:45268->172.16.41.238:8500: read: connection reset by peer"
2025-03-13T22:04:40.066Z [WARN]  storage migration check error: error="Get \"https://172.16.41.238:8500/v1/kv/vault/core/migration\": read tcp 172.16.41.126:45292->172.16.41.238:8500: read: connection reset by peer"
2025-03-13T22:04:42.067Z [WARN]  storage migration check error: error="Get \"https://172.16.41.238:8500/v1/kv/vault/core/migration\": read tcp 172.16.41.126:45310->172.16.41.238:8500: read: connection reset by peer"

vault config

global:
  enabled: true
  tlsDisable: false
server:
  enabled: true
  annotations: |
    "consul.hashicorp.com/connect-inject": "true"
    "consul.hashicorp.com/connect-service": "vault-staging"

  readinessProbe:
    enabled: true
    path: "/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200"
  livenessProbe:
    enabled: true
    path: "/v1/sys/health?standbyok=true&sealedcode=200&uninitcode=200"

  extraEnvironmentVars:
    VAULT_CACERT: /vault/userconfig/tls-ca/ca.crt
    VAULT_SEAL_TYPE: awskms
    VAULT_AWS_REGION: region
    VAULT_AWSKMS_SEAL_KEY_ID: kmskey

  extraSecretEnvironmentVars:
    - envName: CONSUL_HTTP_TOKEN
      secretName: consul-token
      secretKey: token


  ingress:
    enabled: true
    annotations:
      ...

    ingressClassName: alb
    hosts:
      - host: host
    paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: servicename
            port:
              number: 8200

  ha:
    enabled: true
    replicas: 3
    raft:
      enabled: false
      config: |
          ui = true

          listener "tcp" {
            tls_disable = "false"
            address = "[::]:8200"
            cluster_address = "[::]:8201"
            tls_cert_file = "/vault/userconfig/tls-server/tls.crt"
            tls_key_file = "/vault/userconfig/tls-server/tls.key"
            tls_client_ca_file = "/vault/userconfig/tls-ca/ca.crt"
            tls_disable_client_certs = "true"
            tls_require_and_verify_client_cert = "false"
            tls_min_version = "tls10"
          }

          service_registration "kubernetes" {}

          storage "consul" {
            address = "HOST_IP:8500"
            path    = "vault/"
            service = "vault-staging"
            scheme  = "https"
            tls_ca_file = "/vault/userconfig/consul-tls-ca/ca.crt"
            tls_cert_file = "/vault/userconfig/tls-server/tls.crt"
            tls_key_file = "/vault/userconfig/tls-server/tls.key"
            tls_skip_verify = "true"
          }

Consul Config

global:
  enabled: true
  name: consul
  image: hashicorp/consul:1.20.2
  datacenter: dc1
  tls:
    enabled: true
    verify: false
    httpsOnly: true
    caCert:
      secretName: ca-cert
      secretKey: tls.crt
    caKey:
      secretName: ca-key
      secretKey: tls.key
  acls:
    manageSystemACLs: true

  secretsBackend:
    vault:
      enabled: false

server:
  enabled: true
  replicas: 3
  serverCert:
    secretName: server-cert-and-key

client:
  enabled: true

connectInject:
  enabled: true
  default: false
  failurePolicy: "Ignore"
  namespaceSelector: |
    matchExpressions:
      - key: "kubernetes.io/metadata.name"
        operator: "In"
        values: ["vault-staging"]