Consul global proxy configuration not displaying service-service logs, envoy proxies configured using consul connect:
I’ve a service mesh on ECS with EC2 working as the control plane where my consul server is installed and configured correctly. All my services on ECS work with a consul agent that redirects the requests within the mesh and each service as Envoy working as proxy to communicate with the consul agent (Envoy is installed as a sidecar container for each service). The proxies are configured using the consul connect command: consul connect envoy
My issue is when I apply a global configuration to enable access logging on my Envoy proxies, as in: Service Mesh Observability - Access Logs | Consul | HashiCorp Developer the proxies do not display service-service communication logs, only readiness and admin access logs.
Versions and details:
Consul: 1.18.2
Envoy: 1.28.7
Services: ECS
Consul Leader: EC2 instance with 1 leader and 2 followers
Output of consul config read -kind proxy-defaults -name global:
{
"Kind": "proxy-defaults",
"Name": "global",
"TransparentProxy": {},
"Config": {
"LocalRequestTimeoutMs": 15000,
"Protocol": "http"
},
"MeshGateway": {},
"Expose": {},
"AccessLogs": {
"Enabled": true,
"JSONFormat": "{\n \"timestamp\": \"%START_TIME%\",\n \"protocol\": \"%PROTOCOL%\",\n \"duration\": \"%DURATION%\",\n \"upstream_host\": \"%UPSTREAM_HOST%\",\n \"upstream_cluster\": \"%UPSTREAM_CLUSTER%\",\n \"response_code\": \"%RESPONSE_CODE%\",\n \"response_flags\": \"%RESPONSE_FLAGS%\",\n \"bytes_received\": \"%BYTES_RECEIVED%\",\n \"bytes_sent\": \"%BYTES_SENT%\",\n \"request_method\": \"%REQ(:METHOD)%\",\n \"request_path\": \"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%\",\n \"request_id\": \"%REQ(X-REQUEST-ID)%\",\n \"request_host\": \"%REQ(HOST)%\",\n \"user_agent\": \"%REQ(USER-AGENT)%\",\n \"client_ip\": \"%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%\",\n \"connection_termination_details\": \"%CONNECTION_TERMINATION_DETAILS%\",\n \"upstream_transport_failure_reason\": \"%UPSTREAM_TRANSPORT_FAILURE_REASON%\"\n}\n"
},
"CreateIndex": 3666562,
"ModifyIndex": 17581039
}
Log Example on the proxies
{"user_agent":"curl/7.68.0","response_flags":"-","bytes_sent":5,"request_id":null,"request_host":"localhost:19000","upstream_host":null,"request_method":"GET","request_path":"/ready","bytes_received":0,"connection_termination_details":null,"upstream_cluster":null,"duration":0,"client_ip":"127.0.0.1","protocol":"HTTP/1.1","response_code":200,"upstream_transport_failure_reason":null,"timestamp":"2024-12-09T16:00:09.341Z"} {"upstream_cluster":null,"user_agent":"Datadog Agent/7.52.1","timestamp":"2024-12-09T16:00:13.451Z","connection_termination_details":null,"upstream_host":null,"upstream_transport_failure_reason":null,"response_code":200,"request_method":"GET","request_id":"41240e1e-dd71-406c-a98b-4009a24d84bf","client_ip":"127.0.0.1","request_path":"/stats","protocol":"HTTP/1.1","bytes_sent":303877,"request_host":"ip-10-29-13-237.us-west-2.compute.internal:19001","bytes_received":0,"duration":2,"response_flags":"-"} {"duration":0,"protocol":"HTTP/1.1","connection_termination_details":null,"request_id":null,"client_ip":"127.0.0.1","request_method":"GET","upstream_cluster":null,"upstream_host":null,"request_path":"/ready","response_code":200,"user_agent":"curl/7.68.0","bytes_received":0,"request_host":"localhost:19000","timestamp":"2024-12-09T16:00:14.412Z","upstream_transport_failure_reason":null,"response_flags":"-","bytes_sent":5} {"upstream_transport_failure_reason":null,"request_id":null,"request_method":"GET","bytes_received":0,"request_path":"/ready","response_code":200,"upstream_cluster":null,"duration":0,"upstream_host":null,"client_ip":"127.0.0.1","protocol":"HTTP/1.1","response_flags":"-","user_agent":"curl/7.68.0","connection_termination_details":null,"bytes_sent":5,"timestamp":"2024-12-09T16:00:19.500Z","request_host":"localhost:19000"} {"bytes_sent":5,"bytes_received":0,"upstream_host":null,"request_id":null,"protocol":"HTTP/1.1","user_agent":"curl/7.68.0","request_host":"localhost:19000","connection_termination_details":null,"response_code":200,"upstream_transport_failure_reason":null,"request_method":"GET","timestamp":"2024-12-09T16:00:24.581Z","duration":0,"response_flags":"-","upstream_cluster":null,"request_path":"/ready","client_ip":"127.0.0.1"} {"request_method":"GET","client_ip":"127.0.0.1","connection_termination_details":null,"bytes_sent":303875,"upstream_cluster":null,"request_host":"ip-10-29-13-237.us-west-2.compute.internal:19001","upstream_host":null,"user_agent":"Datadog Agent/7.52.1","response_code":200,"upstream_transport_failure_reason":null,"duration":2,"request_path":"/stats","timestamp":"2024-12-09T16:00:28.452Z","bytes_received":0,"protocol":"HTTP/1.1","response_flags":"-","request_id":"987eb0b8-3c84-4016-bfae-9e4a42d8acbf"} {"connection_termination_details":null,"bytes_received":0,"response_code":200,"request_path":"/ready","protocol":"HTTP/1.1","request_method":"GET","client_ip":"127.0.0.1","upstream_transport_failure_reason":null,"request_host":"localhost:19000","bytes_sent":5,"timestamp":"2024-12-09T16:00:29.680Z","upstream_cluster":null,"user_agent":"curl/7.68.0","response_flags":"-","duration":0,"request_id":null,"upstream_host":null} {"user_agent":"curl/7.68.0","request_id":null,"response_code":200,"response_flags":"-","connection_termination_details":null,"request_host":"localhost:19000","bytes_received":0,"protocol":"HTTP/1.1","upstream_transport_failure_reason":null,"timestamp":"2024-12-09T16:00:34.790Z","bytes_sent":5,"upstream_cluster":null,"request_path":"/ready","upstream_host":null,"client_ip":"127.0.0.1","duration":0,"request_method":"GET"} {"timestamp":"2024-12-09T16:00:39.890Z","request_id":null,"upstream_cluster":null,"upstream_transport_failure_reason":null,"response_code":200,"request_path":"/ready","request_method":"GET","request_host":"localhost:19000","client_ip":"127.0.0.1","bytes_received":0,"connection_termination_details":null,"duration":0,"protocol":"HTTP/1.1","bytes_sent":5,"upstream_host":null,"response_flags":"-","user_agent":"curl/7.68.0"} {"connection_termination_details":null,"bytes_sent":303874,"upstream_transport_failure_reason":null,"duration":2,"response_flags":"-","request_host":"ip-10-29-13-237.us-west-2.compute.internal:19001","upstream_host":null,"user_agent":"Datadog Agent/7.52.1","request_id":"7ecbc0b6-cc32-4d4b-83be-4ba75026e61c","client_ip":"127.0.0.1","request_method":"GET","bytes_received":0,"request_path":"/stats","timestamp":"2024-12-09T16:00:43.451Z","protocol":"HTTP/1.1","upstream_cluster":null,"response_code":200} {"upstream_cluster":null,"duration":0,"upstream_host":null,"upstream_transport_failure_reason":null,"protocol":"HTTP/1.1","bytes_sent":5,"bytes_received":0,"timestamp":"2024-12-09T16:00:44.982Z","client_ip":"127.0.0.1","response_code":200,"request_id":null,"user_agent":"curl/7.68.0","connection_termination_details":null,"response_flags":"-","request_path":"/ready","request_method":"GET","request_host":"localhost:19000"} {"timestamp":"2024-12-09T16:00:50.071Z","user_agent":"curl/7.68.0","request_method":"GET","protocol":"HTTP/1.1","response_code":200,"duration":0,"request_path":"/ready","bytes_sent":5,"upstream_transport_failure_reason":null,"bytes_received":0,"client_ip":"127.0.0.1","request_host":"localhost:19000","response_flags":"-","upstream_cluster":null,"upstream_host":null,"request_id":null,"connection_termination_details":null} {"timestamp":"2024-12-09T16:00:55.160Z","request_path":"/ready","client_ip":"127.0.0.1","upstream_transport_failure_reason":null,"response_code":200,"bytes_received":0,"protocol":"HTTP/1.1","upstream_host":null,"response_flags":"-","duration":0,"request_method":"GET","request_id":null,"request_host":"localhost:19000","upstream_cluster":null,"user_agent":"curl/7.68.0","connection_termination_details":null,"bytes_sent":5} {"response_code":200,"upstream_transport_failure_reason":null,"bytes_received":0,"timestamp":"2024-12-09T16:00:58.452Z","upstream_cluster":null,"request_method":"GET","duration":2,"protocol":"HTTP/1.1","request_host":"ip-10-29-13-237.us-west-2.compute.internal:19001","response_flags":"-","user_agent":"Datadog Agent/7.52.1","upstream_host":null,"request_path":"/stats","bytes_sent":303860,"connection_termination_details":null,"client_ip":"127.0.0.1","request_id":"c6748d59-5ad2-4e54-b1f3-155ea30aae94"} {"response_flags":"-","duration":0,"protocol":"HTTP/1.1","upstream_cluster":null,"bytes_sent":5,"request_id":null,"connection_termination_details":null,"request_path":"/ready","client_ip":"127.0.0.1","request_host":"localhost:19000","request_method":"GET","bytes_received":0,"user_agent":"curl/7.68.0","timestamp":"2024-12-09T16:01:00.262Z","upstream_transport_failure_reason":null,"upstream_host":null,"response_code":200} {"upstream_cluster":null,"protocol":"HTTP/1.1","request_host":"localhost:19000","client_ip":"127.0.0.1","response_flags":"-","request_id":null,"request_method":"GET","upstream_transport_failure_reason":null,"duration":0,"user_agent":"curl/7.68.0","request_path":"/ready","upstream_host":null,"connection_termination_details":null,"response_code":200,"timestamp":"2024-12-09T16:01:05.341Z","bytes_sent":5,"bytes_received":0}
Additional information:
After writing the global configuration I checked that all the consul server instances have it correctly applied, I also restarted the services to bootstrap them from scratch. On the Envoy proxies, looking at curl localhost:19000/config_dump the access logging is not enabled for the dynamic discovered services only to the admin interface.
Output sample of the Envoy config_dump:
"dynamic_listeners": [
{
"name": "public_listener:[REDACTED_IP]:[REDACTED_PORT]",
"active_state": {
"version_info": "[REDACTED]",
"listener": {
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "public_listener:[REDACTED_IP]:[REDACTED_PORT]",
"address": {
"socket_address": {
"address": "[REDACTED_IP]",
"port_value": [REDACTED_PORT]
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.filters.network.rbac",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC",
"rules": {
"policies": {
"consul-intentions-layer4": {
"permissions": [
{
"any": true
}
],
"principals": [
{
"authenticated": {
"principal_name": {
"safe_regex": {
"regex": "^spiffe://[REDACTED]"
}
}
}
}
]
}
}
},
"stat_prefix": "connect_authz"
}
},
{
"name": "envoy.filters.network.tcp_proxy",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy",
"stat_prefix": "public_listener",
"cluster": "local_app"
}
}
],
"transport_socket": {
"name": "tls",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext",
"common_tls_context": {
"tls_params": {},
"tls_certificates": [
{
"certificate_chain": {
"inline_string": "[REDACTED_CERTIFICATE]"
},
"private_key": {
"inline_string": "[REDACTED]"
}
}
],
"validation_context": {
"trusted_ca": {
"inline_string": "[REDACTED_CA_CERTIFICATE]"
}
}
},
"require_client_certificate": true
}
}
}
],
"traffic_direction": "INBOUND"
},
"last_updated": "[REDACTED_TIMESTAMP]"
}
},
{
"name": "mesh-sb-test-appliB:[REDACTED_IP]:[REDACTED_PORT]",
"active_state": {
"version_info": "[REDACTED]",
"listener": {
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "mesh-sb-test-appliB:[REDACTED_IP]:[REDACTED_PORT]",
"address": {
"socket_address": {
"address": "[REDACTED_IP]",
"port_value": [REDACTED_PORT]
}
},
"filter_chains": [
{
"filters": [
{
"name": "envoy.filters.network.http_connection_manager",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "[REDACTED]",
"route_config": {
"name": "[REDACTED]",
"virtual_hosts": [
{
"name": "[REDACTED]",
"domains": [
"*"
],
"routes": [
{
"match": {
"prefix": "/"
},
"route": {
"cluster": "[REDACTED]"
}
}
]
}
]
},
"http_filters": [
{
"name": "envoy.filters.http.router",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}
],
"tracing": {
"random_sampling": {}
},
"upgrade_configs": [
{
"upgrade_type": "websocket"
}
]
}
}
]
}
],
"traffic_direction": "OUTBOUND"
},
"last_updated": "[REDACTED_TIMESTAMP]"
}
}
]
Let me know if you need more information.
Kind Regards,