How do I get the envoy sidecar to connect to Consul over TLS?

[Material-Scientist]

For anyone still wondering how to do this, it’s only mentioned as a warning in the Nomad docs’ Consul block config:

You have to disable grpc verify_incoming in Consul’s config bc it doesn’t support incoming TLS verification!

Working consul.hcl snippet example:

ports {
  https = 8501
  #http = 8500
  http = -1 # disable http
  grpc_tls = 8502 # must be its own port
  grpc = 8503
}

connect {
  enabled = true
}

addresses = {
  https = "0.0.0.0"
#  http  = "127.0.0.1"
#  grpc  = "127.0.0.1"
}

tls {
  defaults {
    ca_file = "/etc/ssl/certs/consul-agent-ca.pem"
    cert_file = "/etc/ssl/certs/dc1-server-consul-0.pem"
    key_file = "/etc/ssl/certs/dc1-server-consul-0-key.pem"
    verify_incoming = true
    verify_outgoing = true
    verify_server_hostname = true
  }
  grpc {
    # this needs to be disabled!
    verify_incoming = false
  }
}

# disable, so Consul doesn't use connect-ca!
#auto_encrypt {
#  allow_tls = true
#}

For nomad.hcl, you need:

consul {
  #address = "192.168.178.35:8500"
  address = "192.168.178.35:8501" # https is on port 8501
  token = "<your-token>"
  ssl = true # forces https
  verify_ssl = true
  grpc_ca_file = "<your-ca-cert>"
  ca_file = "<your-ca-cert>"
  cert_file = "<your-client-cert>"
  key_file = "<your-client-cert>"

  auto_advertise = true
  server_auto_join = true
  client_auto_join = true
  server_service_name = "nomad"
  client_service_name = "nomad-client"
  #grpc_address = "192.168.178.35:8503"
  grpc_address = "192.168.178.35:8502" # tls

  # required, so Consul doesn't use anon token for KV:
  service_identity {
    aud = ["consul.io"]
    ttl = "1h"
  }

  task_identity {
    aud = ["consul.io"]
    ttl = "1h"
  }
  # be sure to also follow the docs to create policies for nomad agents & tasks, as well as a role for nomad-tasks.
}

Also, you will need to add proxy defaults to Consul:

Kind      = "proxy-defaults"
Name      = "global"
Config {
  local_connect_timeout_ms = 1000
  handshake_timeout_ms     = 10000
}

consul config write proxy-defaults.hcl

You may also need to give Consul’s anon token more rights by attaching it to a policy that has this:

service_prefix "" { policy = "read" }
node_prefix    "" { policy = "read" }

Using these settings, I’m able to run the countdash example job with ACL & mTLS enabled:

This was not easy to figure out and the docs should really be updated with an example with all the required steps.