#### Overview of the Issue
We are trying to set up a simple Envoy (1.13.0) + …Consul Connect (1.7.2) integration, and have been unable to make any successful curl requests through Envoy to the local service.
We've configured Envoy to listen on `HTTP:8080` which should route requests to a local `hello-world` service via `http://127.0.0.1:80`. We are not yet trying to implement any upstream service functionality. Everything is performed over HTTP for the time being.
So far, we've been able to start Envoy using `consul connect envoy`. Its alive checks in Consul are passing. We are able to hit the local service directly via `HTTP:80`.
However, when attempting to connect via Envoy:
```
$ curl xx.xx.xx.xx:8080
curl: (56) Recv failure: Connection reset by peer
```
Versus directly against the application:
```
$ curl xx.xx.xx.xx:80
Hello World!
```
With Envoy debug logs, we see these TLS handshake errors come through when making our test requests through the proxy:
```
[2020-05-01 18:26:11.475][31][debug][connection]
[source/extensions/transport_sockets/tls/ssl_socket.cc:198] [C45] handshake error: 1
[2020-05-01 18:26:11.475][31][debug][connection]
[source/extensions/transport_sockets/tls/ssl_socket.cc:226] [C45] TLS error: 268435612:SSL routines:OPENSSL_internal:HTTP_REQUEST
```
This is confusing as there is no mention of TLS in our service configuration, yet a `tls_context` filter exists in our http filter's dynamic config:
<details><summary>Envoy admin config dump</summary>
<p>
```json
{
"configs": [
{
"@type": "type.googleapis.com/envoy.admin.v3.BootstrapConfigDump",
"bootstrap": {
"node": {
"id": "hello-world-sidecar-proxy",
"cluster": "hello-world",
"metadata": {
"namespace": "default",
"envoy_version": "1.13.0"
},
"hidden_envoy_deprecated_build_version": "bb7ceff4c3c5bd4555dff28b6e56d27f2f8be0a7/1.13.0/Clean/RELEASE/BoringSSL",
"user_agent_name": "envoy",
"user_agent_build_version": {
"version": {
"major_number": 1,
"minor_number": 13
},
"metadata": {
"build.type": "RELEASE",
"ssl.version": "BoringSSL",
"revision.status": "Clean",
"revision.sha": "bb7ceff4c3c5bd4555dff28b6e56d27f2f8be0a7"
}
},
"extensions": [
{
"name": "envoy.transport_sockets.alts",
"category": "envoy.transport_sockets.upstream"
},
{
"name": "envoy.transport_sockets.raw_buffer",
"category": "envoy.transport_sockets.upstream"
},
{
"name": "envoy.transport_sockets.tap",
"category": "envoy.transport_sockets.upstream"
},
{
"name": "envoy.transport_sockets.tls",
"category": "envoy.transport_sockets.upstream"
},
{
"name": "raw_buffer",
"category": "envoy.transport_sockets.upstream"
},
{
"name": "tls",
"category": "envoy.transport_sockets.upstream"
},
{
"name": "envoy.file_access_log",
"category": "envoy.access_loggers"
},
{
"name": "envoy.http_grpc_access_log",
"category": "envoy.access_loggers"
},
{
"name": "envoy.tcp_grpc_access_log",
"category": "envoy.access_loggers"
},
{
"name": "auto",
"category": "envoy.thrift_proxy.transports"
},
{
"name": "framed",
"category": "envoy.thrift_proxy.transports"
},
{
"name": "header",
"category": "envoy.thrift_proxy.transports"
},
{
"name": "unframed",
"category": "envoy.thrift_proxy.transports"
},
{
"name": "envoy.resource_monitors.fixed_heap",
"category": "envoy.resource_monitors"
},
{
"name": "envoy.resource_monitors.injected_resource",
"category": "envoy.resource_monitors"
},
{
"name": "envoy.transport_sockets.alts",
"category": "envoy.transport_sockets.downstream"
},
{
"name": "envoy.transport_sockets.raw_buffer",
"category": "envoy.transport_sockets.downstream"
},
{
"name": "envoy.transport_sockets.tap",
"category": "envoy.transport_sockets.downstream"
},
{
"name": "envoy.transport_sockets.tls",
"category": "envoy.transport_sockets.downstream"
},
{
"name": "raw_buffer",
"category": "envoy.transport_sockets.downstream"
},
{
"name": "tls",
"category": "envoy.transport_sockets.downstream"
},
{
"name": "dubbo",
"category": "envoy.dubbo_proxy.protocols"
},
{
"name": "envoy.cluster.eds",
"category": "envoy.clusters"
},
{
"name": "envoy.cluster.logical_dns",
"category": "envoy.clusters"
},
{
"name": "envoy.cluster.original_dst",
"category": "envoy.clusters"
},
{
"name": "envoy.cluster.static",
"category": "envoy.clusters"
},
{
"name": "envoy.cluster.strict_dns",
"category": "envoy.clusters"
},
{
"name": "envoy.clusters.aggregate",
"category": "envoy.clusters"
},
{
"name": "envoy.clusters.dynamic_forward_proxy",
"category": "envoy.clusters"
},
{
"name": "envoy.clusters.redis",
"category": "envoy.clusters"
},
{
"name": "auto",
"category": "envoy.thrift_proxy.protocols"
},
{
"name": "binary",
"category": "envoy.thrift_proxy.protocols"
},
{
"name": "binary/non-strict",
"category": "envoy.thrift_proxy.protocols"
},
{
"name": "compact",
"category": "envoy.thrift_proxy.protocols"
},
{
"name": "twitter",
"category": "envoy.thrift_proxy.protocols"
},
{
"name": "envoy.filters.thrift.rate_limit",
"category": "envoy.thrift_proxy.filters"
},
{
"name": "envoy.filters.thrift.router",
"category": "envoy.thrift_proxy.filters"
},
{
"name": "envoy.health_checkers.redis",
"category": "envoy.health_checkers"
},
{
"name": "envoy.retry_priorities.previous_priorities",
"category": "envoy.retry_priorities"
},
{
"name": "envoy.filters.dubbo.router",
"category": "envoy.dubbo_proxy.filters"
},
{
"name": "default",
"category": "envoy.dubbo_proxy.route_matchers"
},
{
"name": "envoy.dog_statsd",
"category": "envoy.stats_sinks"
},
{
"name": "envoy.metrics_service",
"category": "envoy.stats_sinks"
},
{
"name": "envoy.stat_sinks.hystrix",
"category": "envoy.stats_sinks"
},
{
"name": "envoy.statsd",
"category": "envoy.stats_sinks"
},
{
"name": "envoy.retry_host_predicates.omit_canary_hosts",
"category": "envoy.retry_host_predicates"
},
{
"name": "envoy.retry_host_predicates.previous_hosts",
"category": "envoy.retry_host_predicates"
},
{
"name": "envoy.grpc_credentials.aws_iam",
"category": "envoy.grpc_credentials"
},
{
"name": "envoy.grpc_credentials.default",
"category": "envoy.grpc_credentials"
},
{
"name": "envoy.grpc_credentials.file_based_metadata",
"category": "envoy.grpc_credentials"
},
{
"name": "envoy.dynamic.ot",
"category": "envoy.tracers"
},
{
"name": "envoy.lightstep",
"category": "envoy.tracers"
},
{
"name": "envoy.tracers.datadog",
"category": "envoy.tracers"
},
{
"name": "envoy.tracers.opencensus",
"category": "envoy.tracers"
},
{
"name": "envoy.tracers.xray",
"category": "envoy.tracers"
},
{
"name": "envoy.zipkin",
"category": "envoy.tracers"
},
{
"name": "raw_udp_listener",
"category": "envoy.udp_listeners"
},
{
"name": "envoy.filters.udp_listener.udp_proxy",
"category": "envoy.filters.udp_listener"
},
{
"name": "dubbo.hessian2",
"category": "envoy.dubbo_proxy.serializers"
},
{
"name": "envoy.buffer",
"category": "envoy.filters.http"
},
{
"name": "envoy.cors",
"category": "envoy.filters.http"
},
{
"name": "envoy.csrf",
"category": "envoy.filters.http"
},
{
"name": "envoy.ext_authz",
"category": "envoy.filters.http"
},
{
"name": "envoy.fault",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.adaptive_concurrency",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.dynamic_forward_proxy",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.grpc_http1_reverse_bridge",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.grpc_stats",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.header_to_metadata",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.jwt_authn",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.on_demand",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.original_src",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.rbac",
"category": "envoy.filters.http"
},
{
"name": "envoy.filters.http.tap",
"category": "envoy.filters.http"
},
{
"name": "envoy.grpc_http1_bridge",
"category": "envoy.filters.http"
},
{
"name": "envoy.grpc_json_transcoder",
"category": "envoy.filters.http"
},
{
"name": "envoy.grpc_web",
"category": "envoy.filters.http"
},
{
"name": "envoy.gzip",
"category": "envoy.filters.http"
},
{
"name": "envoy.health_check",
"category": "envoy.filters.http"
},
{
"name": "envoy.http_dynamo_filter",
"category": "envoy.filters.http"
},
{
"name": "envoy.ip_tagging",
"category": "envoy.filters.http"
},
{
"name": "envoy.lua",
"category": "envoy.filters.http"
},
{
"name": "envoy.rate_limit",
"category": "envoy.filters.http"
},
{
"name": "envoy.router",
"category": "envoy.filters.http"
},
{
"name": "envoy.squash",
"category": "envoy.filters.http"
},
{
"name": "envoy.listener.http_inspector",
"category": "envoy.filters.listener"
},
{
"name": "envoy.listener.original_dst",
"category": "envoy.filters.listener"
},
{
"name": "envoy.listener.original_src",
"category": "envoy.filters.listener"
},
{
"name": "envoy.listener.proxy_protocol",
"category": "envoy.filters.listener"
},
{
"name": "envoy.listener.tls_inspector",
"category": "envoy.filters.listener"
},
{
"name": "envoy.client_ssl_auth",
"category": "envoy.filters.network"
},
{
"name": "envoy.echo",
"category": "envoy.filters.network"
},
{
"name": "envoy.ext_authz",
"category": "envoy.filters.network"
},
{
"name": "envoy.filters.network.dubbo_proxy",
"category": "envoy.filters.network"
},
{
"name": "envoy.filters.network.kafka_broker",
"category": "envoy.filters.network"
},
{
"name": "envoy.filters.network.local_ratelimit",
"category": "envoy.filters.network"
},
{
"name": "envoy.filters.network.mysql_proxy",
"category": "envoy.filters.network"
},
{
"name": "envoy.filters.network.rbac",
"category": "envoy.filters.network"
},
{
"name": "envoy.filters.network.sni_cluster",
"category": "envoy.filters.network"
},
{
"name": "envoy.filters.network.thrift_proxy",
"category": "envoy.filters.network"
},
{
"name": "envoy.filters.network.zookeeper_proxy",
"category": "envoy.filters.network"
},
{
"name": "envoy.http_connection_manager",
"category": "envoy.filters.network"
},
{
"name": "envoy.mongo_proxy",
"category": "envoy.filters.network"
},
{
"name": "envoy.ratelimit",
"category": "envoy.filters.network"
},
{
"name": "envoy.redis_proxy",
"category": "envoy.filters.network"
},
{
"name": "envoy.tcp_proxy",
"category": "envoy.filters.network"
},
{
"name": "envoy.ip",
"category": "envoy.resolvers"
}
]
},
"static_resources": {
"clusters": [
{
"name": "local_agent",
"type": "STATIC",
"connect_timeout": "1s",
"hidden_envoy_deprecated_hosts": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 8502
}
}
],
"http2_protocol_options": {}
}
]
},
"dynamic_resources": {
"lds_config": {
"ads": {}
},
"cds_config": {
"ads": {}
},
"ads_config": {
"api_type": "GRPC",
"grpc_services": [
{
"envoy_grpc": {
"cluster_name": "local_agent"
},
"initial_metadata": [
{
"key": "x-consul-token",
"value": "xxx"
}
]
}
]
}
},
"admin": {
"access_log_path": "/dev/null",
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 19000
}
}
},
"stats_config": {
"stats_tags": [
{
"tag_name": "consul.custom_hash",
"regex": "^cluster\\.((?:([^.]+)~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)"
},
{
"tag_name": "consul.service_subset",
"regex": "^cluster\\.((?:[^.]+~)?(?:([^.]+)\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)"
},
{
"tag_name": "consul.service",
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)"
},
{
"tag_name": "consul.namespace",
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.[^.]+\\.consul\\.)"
},
{
"tag_name": "consul.datacenter",
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)"
},
{
"tag_name": "consul.routing_type",
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.[^.]+\\.consul\\.)"
},
{
"tag_name": "consul.trust_domain",
"regex": "^cluster\\.((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.([^.]+)\\.consul\\.)"
},
{
"tag_name": "consul.target",
"regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+)\\.[^.]+\\.[^.]+\\.consul\\.)"
},
{
"tag_name": "consul.full_target",
"regex": "^cluster\\.(((?:[^.]+~)?(?:[^.]+\\.)?[^.]+\\.[^.]+\\.[^.]+\\.[^.]+\\.[^.]+)\\.consul\\.)"
},
{
"tag_name": "local_cluster",
"fixed_value": "hello-world"
}
],
"use_all_default_tags": true
},
"layered_runtime": {
"layers": [
{
"name": "static_layer",
"static_layer": {
"envoy.deprecated_features:envoy.config.trace.v2.ZipkinConfig.HTTP_JSON_V1": true,
"envoy.deprecated_features:envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager.Tracing.operation_name": true,
"envoy.deprecated_features:envoy.api.v2.Cluster.tls_context": true
}
}
]
}
},
"last_updated": "2020-05-01T17:51:23.455Z"
},
{
"@type": "type.googleapis.com/envoy.admin.v3.ClustersConfigDump",
"version_info": "00000001",
"static_clusters": [
{
"cluster": {
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "local_agent",
"type": "STATIC",
"connect_timeout": "1s",
"hosts": [
{
"socket_address": {
"address": "127.0.0.1",
"port_value": 8502
}
}
],
"http2_protocol_options": {}
},
"last_updated": "2020-05-01T17:51:23.481Z"
}
],
"dynamic_active_clusters": [
{
"version_info": "00000001",
"cluster": {
"@type": "type.googleapis.com/envoy.api.v2.Cluster",
"name": "local_app",
"type": "STATIC",
"connect_timeout": "5s",
"load_assignment": {
"cluster_name": "local_app",
"endpoints": [
{
"lb_endpoints": [
{
"endpoint": {
"address": {
"socket_address": {
"address": "127.0.0.1",
"port_value": 80
}
}
}
}
]
}
]
}
},
"last_updated": "2020-05-01T17:51:23.559Z"
}
]
},
{
"@type": "type.googleapis.com/envoy.admin.v3.ListenersConfigDump",
"version_info": "00000001",
"dynamic_listeners": [
{
"name": "public_listener:0.0.0.0:8080",
"active_state": {
"version_info": "00000001",
"listener": {
"@type": "type.googleapis.com/envoy.api.v2.Listener",
"name": "public_listener:0.0.0.0:8080",
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 8080
}
},
"filter_chains": [
{
"tls_context": {
"common_tls_context": {
"tls_params": {},
"tls_certificates": [
{
"certificate_chain": {
"inline_string": "-----BEGIN CERTIFICATE-----xxx"
},
"private_key": {
"inline_string": "[redacted]"
}
}
],
"validation_context": {
"trusted_ca": {
"inline_string": "-----BEGIN CERTIFICATE-----xxx"
}
}
},
"require_client_certificate": true
},
"filters": [
{
"name": "envoy.ext_authz",
"config": {
"grpc_service": {
"envoy_grpc": {
"cluster_name": "local_agent"
},
"initial_metadata": [
{
"key": "x-consul-token",
"value": "xxx"
}
]
},
"stat_prefix": "connect_authz"
}
},
{
"name": "envoy.http_connection_manager",
"config": {
"tracing": {
"random_sampling": {}
},
"route_config": {
"name": "public_listener",
"virtual_hosts": [
{
"routes": [
{
"route": {
"cluster": "local_app"
},
"match": {
"prefix": "/"
}
}
],
"domains": [
"*"
],
"name": "public_listener"
}
]
},
"http_filters": [
{
"name": "envoy.router"
}
],
"stat_prefix": "public_listener_http"
}
}
]
}
]
},
"last_updated": "2020-05-01T17:51:23.575Z"
}
}
]
},
{
"@type": "type.googleapis.com/envoy.admin.v3.ScopedRoutesConfigDump"
},
{
"@type": "type.googleapis.com/envoy.admin.v3.RoutesConfigDump",
"static_route_configs": [
{
"route_config": {
"@type": "type.googleapis.com/envoy.api.v2.RouteConfiguration",
"name": "public_listener",
"virtual_hosts": [
{
"name": "public_listener",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "local_app"
}
}
]
}
]
},
"last_updated": "2020-05-01T17:51:23.574Z"
}
]
},
{
"@type": "type.googleapis.com/envoy.admin.v3.SecretsConfigDump"
}
]
}
```
</p>
</details>
#### Reproduction Steps
Service registration config:
```
{
"name": "hello-world",
"port": 80,
"connect": {
"sidecar_service": {
"port": 8080
}
},
"checks": [
{
"id": "api-service",
"name": "HTTP API check on port 80",
"http": "http://127.0.0.1:80/health",
"interval": "10s",
"timeout": "1s"
}
]
}
```
Config entries:
```
kind = "proxy-defaults"
name = "global"
config {
protocol = "http"
local_connect_timeout_ms = 2000
}
```
```
kind = "service-defaults"
name = "hello-world"
protocol = "http"
```
### Operating system and Environment details
ECS Fargate