How to configure an EDS Cluster for Envoy?

Consul Version: 1.7.1
Envoy Version: 1.13.0

Here’s my HCL file with Envoy config. I’m trying to add a test EDS cluster and envoy proxy boots up fine and then I get this error for EDS
[2020-03-09 19:02:45.257][6][warning][config] [bazel-out/k8-opt/bin/source/common/config/_virtual_includes/grpc_stream_lib/common/config/grpc_stream.h:91] gRPC config stream closed: 12, unknown service envoy.api.v2.EndpointDiscoveryService

services {
  name = "node"
  port = 9100
  connect {
    sidecar_service {
      proxy {
        config {
      envoy_bootstrap_json_tpl = <<EOF
{
  "admin": {
    "access_log_path": "/dev/null",
    "address": {
      "socket_address": {
        "address": "{{ .AdminBindAddress }}",
        "port_value": {{ .AdminBindPort }}
      }
    }
  },
  "cluster_manager": {
    "local_cluster_name": "test"
  },
  "node": {
    "cluster": "{{ .ProxyCluster }}",
    "id": "{{ .ProxyID }}"
  },
  "static_resources": {
    "clusters": [
      {
        "name": "{{ .LocalAgentClusterName }}",
        "connect_timeout": "1s",
        "type": "STATIC",
        {{- if .AgentTLS -}}
        "tls_context": {
          "common_tls_context": {
            "validation_context": {
              "trusted_ca": {
                "filename": "{{ .AgentCAFile }}"
              }
            }
          }
        },
        {{- end }}
        "http2_protocol_options": {},
        "hosts": [
          {
            "socket_address": {
              "address": "{{ .AgentAddress }}",
              "port_value": {{ .AgentPort }}
            }
          }
        ]
      },
      {
        "name": "test",
        "type": "EDS",
        "connect_timeout": "5s",
        "eds_cluster_config": {
          "eds_config": {
            "api_config_source": {
              "api_type": "GRPC",
              "grpc_services": [
                {
                  "initial_metadata": [
                    {
                      "key": "x-consul-token",
                      "value": "{{ .Token }}"
                    }
                  ],
                  "envoy_grpc": {
                    "cluster_name": "local_agent"
                  }
                }
              ]
            }
          }
        },
        "load_assignment": {
         "cluster_name": "test",
         "endpoints": [
          {
           "locality": {
            "region": "us-east-1",
            "zone": "us-east-1d"
           },
           "lb_endpoints": [
            {
             "endpoint": {
              "address": {
               "socket_address": {
                "address": "127.0.0.1",
                "port_value": 9091
               }
              }
             }
            }
           ]
          }
         ]
        }
      }
      {{- if .StaticClustersJSON -}}
      ,
      {{ .StaticClustersJSON }}
      {{- end }}
    ]{{- if .StaticListenersJSON -}}
    ,
    "listeners": [
      {{ .StaticListenersJSON }}
    ]
    {{- end }}
  },
  {{- if .StatsSinksJSON }}
  "stats_sinks": {{ .StatsSinksJSON }},
  {{- end }}
  {{- if .StatsConfigJSON }}
  "stats_config": {{ .StatsConfigJSON }},
  {{- end }}
  {{- if .StatsFlushInterval }}
  "stats_flush_interval": "{{ .StatsFlushInterval }}",
  {{- end }}
  {{- if .TracingConfigJSON }}
  "tracing": {{ .TracingConfigJSON }},
  {{- end }}
  "dynamic_resources": {
    "lds_config": { "ads": {} },
    "cds_config": { "ads": {} },
    "ads_config": {
      "api_type": "GRPC",
      "grpc_services": {
        "initial_metadata": [
          {
            "key": "x-consul-token",
            "value": "{{ .Token }}"
          }
        ],
        "envoy_grpc": {
          "cluster_name": "{{ .LocalAgentClusterName }}"
        }
      }
    }
  }
}
EOF
        }
      }
    }
  }
}

Consul Agent config:

{
  "data_dir": "/opt/consul",
  "log_level": "INFO",
  "server": true,
  "client_addr": "0.0.0.0",
  "bootstrap_expect": 5,
  "ui": true,
  "encrypt": "REDACTED",
  "primary_datacenter": "us-east-1",
  "enable_script_checks": true,
  "enable_central_service_config": true,
  "discovery_max_stale": "0.1s",
  "performance": {
     "leave_drain_time": "10s",
     "rpc_hold_timeout": "10s",
     "raft_multiplier": 1
  },
  "telemetry": {
     "prometheus_retention_time": "30s"
  },
  "connect": {
    "enabled": true
  },
  "ports": {
    "grpc": 8502
  },
  "retry_join": ["provider=aws tag_key=aws:autoscaling:groupName tag_value=infra-consul-prod-servers"],
  "acl": {
    "enabled": true,
    "default_policy": "allow",
    "enable_token_persistence":  true
  }
}

Full log:

[2020-03-09 19:22:10.454][6][info][main] [source/server/server.cc:251] initializing epoch 0 (hot restart version=disabled)
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:253] statically linked extensions:
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.thrift_proxy.protocols: auto, binary, binary/non-strict, compact, twitter
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.filters.listener: envoy.listener.http_inspector, envoy.listener.original_dst, envoy.listener.original_src, envoy.listener.proxy_protocol, envoy.listener.tls_inspector
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.dubbo_proxy.route_matchers: default
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.filters.http: envoy.buffer, envoy.cors, envoy.csrf, envoy.ext_authz, envoy.fault, envoy.filters.http.adaptive_concurrency, envoy.filters.http.dynamic_forward_proxy, envoy.filters.http.grpc_http1_reverse_bridge, envoy.filters.http.grpc_stats, envoy.filters.http.header_to_metadata, envoy.filters.http.jwt_authn, envoy.filters.http.on_demand, envoy.filters.http.original_src, envoy.filters.http.rbac, envoy.filters.http.tap, envoy.grpc_http1_bridge, envoy.grpc_json_transcoder, envoy.grpc_web, envoy.gzip, envoy.health_check, envoy.http_dynamo_filter, envoy.ip_tagging, envoy.lua, envoy.rate_limit, envoy.router, envoy.squash
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.filters.network: envoy.client_ssl_auth, envoy.echo, envoy.ext_authz, envoy.filters.network.dubbo_proxy, envoy.filters.network.kafka_broker, envoy.filters.network.local_ratelimit, envoy.filters.network.mysql_proxy, envoy.filters.network.rbac, envoy.filters.network.sni_cluster, envoy.filters.network.thrift_proxy, envoy.filters.network.zookeeper_proxy, envoy.http_connection_manager, envoy.mongo_proxy, envoy.ratelimit, envoy.redis_proxy, envoy.tcp_proxy
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.thrift_proxy.transports: auto, framed, header, unframed
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.resource_monitors: envoy.resource_monitors.fixed_heap, envoy.resource_monitors.injected_resource
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.stats_sinks: envoy.dog_statsd, envoy.metrics_service, envoy.stat_sinks.hystrix, envoy.statsd
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.thrift_proxy.filters: envoy.filters.thrift.rate_limit, envoy.filters.thrift.router
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.tracers: envoy.dynamic.ot, envoy.lightstep, envoy.tracers.datadog, envoy.tracers.opencensus, envoy.tracers.xray, envoy.zipkin
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.retry_host_predicates: envoy.retry_host_predicates.omit_canary_hosts, envoy.retry_host_predicates.previous_hosts
[2020-03-09 19:22:10.455][6][info][main] [source/server/server.cc:255]   envoy.retry_priorities: envoy.retry_priorities.previous_priorities
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.transport_sockets.upstream: envoy.transport_sockets.alts, envoy.transport_sockets.raw_buffer, envoy.transport_sockets.tap, envoy.transport_sockets.tls, raw_buffer, tls
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.clusters: envoy.cluster.eds, envoy.cluster.logical_dns, envoy.cluster.original_dst, envoy.cluster.static, envoy.cluster.strict_dns, envoy.clusters.aggregate, envoy.clusters.dynamic_forward_proxy, envoy.clusters.redis
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.access_loggers: envoy.file_access_log, envoy.http_grpc_access_log, envoy.tcp_grpc_access_log
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.filters.udp_listener: envoy.filters.udp_listener.udp_proxy
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.udp_listeners: raw_udp_listener
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.dubbo_proxy.serializers: dubbo.hessian2
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.resolvers: envoy.ip
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.dubbo_proxy.filters: envoy.filters.dubbo.router
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.transport_sockets.downstream: envoy.transport_sockets.alts, envoy.transport_sockets.raw_buffer, envoy.transport_sockets.tap, envoy.transport_sockets.tls, raw_buffer, tls
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.dubbo_proxy.protocols: dubbo
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.grpc_credentials: envoy.grpc_credentials.aws_iam, envoy.grpc_credentials.default, envoy.grpc_credentials.file_based_metadata
[2020-03-09 19:22:10.456][6][info][main] [source/server/server.cc:255]   envoy.health_checkers: envoy.health_checkers.redis
[2020-03-09 19:22:10.464][6][warning][misc] [source/common/protobuf/utility.cc:441] Using deprecated option 'envoy.api.v2.listener.Filter.config' from file listener_components.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/intro/deprecated for details.
[2020-03-09 19:22:10.465][6][warning][misc] [source/common/protobuf/utility.cc:441] Using deprecated option 'envoy.api.v2.Cluster.hosts' from file cluster.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/intro/deprecated for details.
[2020-03-09 19:22:10.465][6][warning][misc] [source/common/protobuf/utility.cc:441] Using deprecated option 'envoy.api.v2.Cluster.hosts' from file cluster.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/intro/deprecated for details.
[2020-03-09 19:22:10.465][6][info][main] [source/server/server.cc:336] admin address: 127.0.0.1:19000
[2020-03-09 19:22:10.466][6][info][main] [source/server/server.cc:455] runtime: layers:
  - name: base
    static_layer:
      {}
  - name: admin
    admin_layer:
      {}
[2020-03-09 19:22:10.467][6][info][config] [source/server/configuration_impl.cc:62] loading 0 static secret(s)
[2020-03-09 19:22:10.467][6][info][config] [source/server/configuration_impl.cc:68] loading 3 cluster(s)
[2020-03-09 19:22:10.500][6][info][upstream] [source/common/upstream/cluster_manager_impl.cc:145] cm init: initializing secondary clusters
[2020-03-09 19:22:10.505][6][info][config] [source/server/configuration_impl.cc:72] loading 1 listener(s)
[2020-03-09 19:22:10.515][6][info][config] [source/server/configuration_impl.cc:97] loading tracing configuration
[2020-03-09 19:22:10.515][6][info][config] [source/server/configuration_impl.cc:116] loading stats sink configuration
[2020-03-09 19:22:10.515][6][info][main] [source/server/server.cc:550] starting main dispatch loop
[2020-03-09 19:22:10.528][6][warning][config] [bazel-out/k8-opt/bin/source/common/config/_virtual_includes/grpc_stream_lib/common/config/grpc_stream.h:91] gRPC config stream closed: 12, unknown service envoy.api.v2.EndpointDiscoveryService

@vilva42 Do your logs from the Consul agent running the xDS/gRPC server indicate any failures?

@mkeeler thanks for looking into this. I’ve set the log_level to TRACE and do not see any failures. Here are the only envoy/proxy logs I see and there’s one irrelevant error.

2020-03-10T17:37:16.732Z [DEBUG] agent.envoy: generating cluster for: cluster=node.default.us-east-1.internal.087fea5f-7f28-6248-ec6e-0d70f9d13f57.consul
2020-03-10T17:37:16.780Z [DEBUG] agent.envoy: generating cluster for: cluster=node.default.us-east-1.internal.087fea5f-7f28-6248-ec6e-0d70f9d13f57.consul
2020-03-10T17:37:17.074Z [DEBUG] agent.envoy: generating cluster for: cluster=node.default.us-east-1.internal.087fea5f-7f28-6248-ec6e-0d70f9d13f57.consul
2020-03-10T17:37:21.249Z [DEBUG] agent.envoy: generating cluster for: cluster=node.default.us-east-1.internal.087fea5f-7f28-6248-ec6e-0d70f9d13f57.consul 
**2020-03-10T17:37:16.439Z [ERROR] agent.proxycfg: watch error: id=service-http-checks: error="invalid type for service checks response: cache.FetchResult, want: []structs.CheckType"**
2020-03-10T17:37:16.568Z [TRACE] agent.proxycfg: resetting watches for discovery chain: id=node
2020-03-10T17:37:16.569Z [TRACE] agent.proxycfg: initializing watch of target: upstream=node chain=node target=node.default.us-east-1 mesh-gateway-mode=

Another possibly irrelevant error I notice is

2020-03-10T17:37:16.742Z [DEBUG] agent: systemd notify failed: error="No socket"

Is it possible to use EDS using Connect? We’re wondering whether to use Fabio or Connect to implement Zone aware routing

@mkeeler Did you get a chance to look into this? I would like to know if anyone has a working config for EDS.