Consul Connect Envoy Public Listener keeps getting drained

Hi Everyone,

I’m using a 3 node consul cluster (1.8.5) with Vault as the connect CA and connect enabled with Envoy Proxy. Below is my consul configuration for one of the test service I have in my lab.

services {
  name    = "frontend-service"
  id      = "frontend-service"
  address = "10.249.57.121"
  port    = 8001
  token   = "9b879TOKENe06c928ef3"

  tags = [
    "service",
    "envoy-stats",
    "v1",
  ]

  meta = {
    envoy_admin_listener_port = "19000"
    envoy_prometheus_metrics_port = "19100"
    envoy_public_listener_port = "8001"
  }

  connect {
    sidecar_service {
      port = 8001
      tags = [
        "service-proxy",
        "v1",
      ]

      proxy {
        local_service_address = "127.0.0.1"

        config {
          envoy_prometheus_bind_addr = "0.0.0.0:19100"
          protocol                   = "http"

          envoy_public_listener_json = <<EOL
{
  "name": "public_listener:10.249.57.121:8001",
  "address": {
    "socket_address": {
      "address": "10.249.57.121",
      "port_value": 8001
    }
  },
  "filter_chains": [
    {
      "filters": [
        {
          "name": "envoy.http_connection_manager",
          "config": {
            "http_filters": [
              {
                "name": "envoy.router"
              }
            ],
            "stat_prefix": "public_listener_http",
            "access_log": {
              "name": "envoy.file_access_log",
              "config": {
                "path": "/var/log/envoy/access_public_listener_10.249.57.121_8001.log",
                "json_format": {
                  "method": "%REQ(:METHOD)%",
                  "path": "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%",
                  "protocol": "%PROTOCOL%",
                  "duration": "%DURATION%",
                  "start_time": "%START_TIME%",
                  "bytes_received": "%BYTES_RECEIVED%",
                  "response_code": "%RESPONSE_CODE%",
                  "response_duration": "%RESPONSE_DURATION%",
                  "response_flags": "%RESPONSE_FLAGS%",
                  "upstream_host": "%UPSTREAM_HOST%",
                  "upstream_service_time": "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%",
                  "downstream_remote_address": "%DOWNSTREAM_REMOTE_ADDRESS%",
                  "forwarded_for": "%REQ(X-FORWARDED-FOR)%",
                  "user_agent": "%REQ(USER-AGENT)%",
                  "request_id": "%REQ(X-REQUEST-ID)%",
                  "authority": "%REQ(:AUTHORITY)%",
                  "downstream_local_uri_san": "%DOWNSTREAM_LOCAL_URI_SAN%",
                  "downstream_peer_uri_san": "%DOWNSTREAM_PEER_URI_SAN%"
                }
              }
            },
            "tracing": {
              "random_sampling": {}
            },
            "route_config": {
              "virtual_hosts": [
                {
                  "domains": [
                    "*"
                  ],
                  "name": "public_listener",
                  "routes": [
                    {
                      "match": {
                        "prefix": "/"
                      },
                      "route": {
                        "cluster": "local_app"
                      }
                    }
                  ]
                }
              ],
              "name": "public_listener"
            }
          }
        }
      ]
    }
  ]
}
EOL

        }

        upstreams {
          destination_name = "backend-service"
          destination_type = "service"
          datacenter = "aws"
          local_bind_address  = "127.0.0.1"
          local_bind_port     = 9001
          config {
          }

        }
      }

      checks {
        name = "Connect Sidecar (frontend-service) /listeners"
        http = "http://127.0.0.1:19000/listeners"
        interval = "30s"
        // Assume passing for this check, as it's mostly informational.
        status = "passing"
      }
    }
  }
}

In this setup, my frontend service public Listener (10.249.57.121:8001) is keeps getting recreated due to Listener updates, due to this my upstreams are getting disconnected and getting recreated for every 10 mins (Drain time is 10 mins in my setup).
I want to keep this public listener static. How can I achieve that?

These are my trace level (stripped) logs for the frontend service.


[2020-11-23 17:36:56.382][13183][debug][config] [external/envoy/source/server/listener_impl.cc:80] Create listen socket for listener public_listener:10.249.57.121:8001 on address 10.249.57.121:8001
[2020-11-23 17:36:56.382][13183][debug][config] [external/envoy/source/server/listener_impl.cc:70] Set listener public_listener:10.249.57.121:8001 socket factory local address to 10.249.57.121:8001
[2020-11-23 17:36:56.382][13183][debug][config] [external/envoy/source/server/listener_impl.cc:395] add active listener: name=public_listener:10.249.57.121:8001, hash=9172118973310572056, address=10.249.57.121:8001
[2020-11-23 17:36:56.382][13183][info][upstream] [external/envoy/source/server/lds_api.cc:73] lds: add/update listener 'public_listener:10.249.57.121:8001'
[2020-11-23 17:36:56.382][13183][warning][misc] [external/envoy/source/common/protobuf/utility.cc:441] Using deprecated option 'envoy.api.v2.listener.Filter.config' from file listener_components.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/intro/deprecated for details.
[2020-11-23 17:36:56.382][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:377] begin add/update listener: name=backend-service?dc=aws:127.0.0.1:9001 hash=13871615550858586218
[2020-11-23 17:36:56.382][13183][warning][config] [external/envoy/source/server/listener_impl.cc:196] Listener connection limit runtime key envoy.resource_limits.listener.backend-service?dc=aws:127.0.0.1:9001.connection_limit is empty. There are currently no limitations on the number of accepted connections for listener backend-service?dc=aws:127.0.0.1:9001.
[2020-11-23 17:36:56.382][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:92]   filter #0:
[2020-11-23 17:36:56.382][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:93]     name: envoy.tcp_proxy
[2020-11-23 17:36:56.382][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:100]   config: {
 "cluster": "backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul",
 "stat_prefix": "upstream_backend-service?dc=aws_tcp"
}

[2020-11-23 17:36:56.385][13183][debug][config] [external/envoy/source/server/listener_impl.cc:80] Create listen socket for listener backend-service?dc=aws:127.0.0.1:9001 on address 127.0.0.1:9001
[2020-11-23 17:36:56.385][13183][debug][config] [external/envoy/source/server/listener_impl.cc:70] Set listener backend-service?dc=aws:127.0.0.1:9001 socket factory local address to 127.0.0.1:9001
[2020-11-23 17:36:56.385][13183][debug][config] [external/envoy/source/server/listener_impl.cc:395] add active listener: name=backend-service?dc=aws:127.0.0.1:9001, hash=13871615550858586218, address=127.0.0.1:9001
[2020-11-23 17:36:56.385][13183][info][upstream] [external/envoy/source/server/lds_api.cc:73] lds: add/update listener 'backend-service?dc=aws:127.0.0.1:9001'
[2020-11-23 17:36:56.385][13183][debug][init] [external/envoy/source/common/init/watcher_impl.cc:14] target LDS initialized, notifying init manager Server
[2020-11-23 17:36:56.385][13183][debug][init] [external/envoy/source/common/init/watcher_impl.cc:14] init manager Server initialized, notifying RunHelper
[2020-11-23 17:36:56.385][13183][info][config] [external/envoy/source/server/listener_manager_impl.cc:707] all dependencies initialized. starting workers
[2020-11-23 17:36:56.385][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:718] starting worker 0
[2020-11-23 17:36:56.385][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:113] Resuming discovery requests for type.googleapis.com/envoy.api.v2.RouteConfiguration
[2020-11-23 17:36:56.385][13183][debug][config] [external/envoy/source/common/config/grpc_subscription_impl.cc:67] gRPC config for type.googleapis.com/envoy.api.v2.Listener accepted with 2 resources with version 00000001

[2020-11-23 17:36:56.385][13183][trace][router] [external/envoy/source/common/router/router.cc:1580] [C0][S511235682213879845] proxying 5789 bytes
[2020-11-23 17:36:56.385][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:463] [C0] dispatched 8567 bytes
[2020-11-23 17:36:56.385][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:704] [C0] about to send frame type=6, flags=1
[2020-11-23 17:36:56.385][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:752] [C0] send data: bytes=17
[2020-11-23 17:36:56.385][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:409] [C0] writing 17 bytes, end_stream false
[2020-11-23 17:36:56.385][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:643] [C0] sent frame type=6
[2020-11-23 17:36:56.385][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:409] [C0] writing 5798 bytes, end_stream false
[2020-11-23 17:36:56.385][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:643] [C0] sent frame type=0
[2020-11-23 17:36:56.385][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:473] [C0] socket event: 2
[2020-11-23 17:36:56.385][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:561] [C0] write ready
[2020-11-23 17:36:56.385][13183][trace][connection] [external/envoy/source/extensions/transport_sockets/tls/ssl_socket.cc:259] [C0] ssl write returns: 5815
[2020-11-23 17:36:56.385][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:473] [C0] socket event: 3
[2020-11-23 17:36:56.386][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:561] [C0] write ready
[2020-11-23 17:36:56.386][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:511] [C0] read ready
[2020-11-23 17:36:56.386][13183][trace][connection] [external/envoy/source/extensions/transport_sockets/tls/ssl_socket.cc:80] [C0] ssl read returns: 43
[2020-11-23 17:36:56.386][13183][trace][connection] [external/envoy/source/extensions/transport_sockets/tls/ssl_socket.cc:80] [C0] ssl read returns: -1
[2020-11-23 17:36:56.386][13183][trace][connection] [external/envoy/source/extensions/transport_sockets/tls/ssl_socket.cc:155] [C0] ssl read 43 bytes into 1 slices
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:444] [C0] dispatching 43 bytes
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:507] [C0] about to recv frame type=8, flags=0
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:522] [C0] recv frame type=8
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:507] [C0] about to recv frame type=6, flags=0
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:522] [C0] recv frame type=6
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:507] [C0] about to recv frame type=8, flags=0
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:522] [C0] recv frame type=8
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:463] [C0] dispatched 43 bytes
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:704] [C0] about to send frame type=6, flags=1
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:752] [C0] send data: bytes=17
[2020-11-23 17:36:56.386][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:409] [C0] writing 17 bytes, end_stream false
[2020-11-23 17:36:56.386][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:643] [C0] sent frame type=6
[2020-11-23 17:36:56.386][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:473] [C0] socket event: 2
[2020-11-23 17:36:56.386][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:561] [C0] write ready
[2020-11-23 17:36:56.386][13183][trace][connection] [external/envoy/source/extensions/transport_sockets/tls/ssl_socket.cc:259] [C0] ssl write returns: 17
[2020-11-23 17:36:56.386][13213][debug][main] [external/envoy/source/server/worker_impl.cc:103] worker entering dispatch loop
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:925] adding TLS initial cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:925] adding TLS initial cluster local_agent
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:925] adding TLS initial cluster local_app
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:925] adding TLS initial cluster self_admin
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:1084] membership update for TLS cluster local_agent added 1 removed 0
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:1084] membership update for TLS cluster self_admin added 1 removed 0
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:606] updating TLS cluster local_app
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:1084] membership update for TLS cluster local_app added 1 removed 0
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:606] updating TLS cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul
[2020-11-23 17:36:56.386][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:1084] membership update for TLS cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul added 2 removed 0
[2020-11-23 17:36:56.387][13214][debug][grpc] [external/envoy/source/common/grpc/google_async_client_impl.cc:45] completionThread running
[2020-11-23 17:36:58.402][13183][debug][conn_handler] [external/envoy/source/server/connection_handler_impl.cc:361] [C1] new connection
[2020-11-23 17:36:58.402][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:473] [C1] socket event: 2
[2020-11-23 17:36:58.402][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:561] [C1] write ready
[2020-11-23 17:36:58.402][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:473] [C1] socket event: 3
[2020-11-23 17:36:58.402][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:561] [C1] write ready
[2020-11-23 17:36:58.402][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:511] [C1] read ready
[2020-11-23 17:36:58.402][13183][trace][connection] [external/envoy/source/common/network/raw_buffer_socket.cc:24] [C1] read returns: 158
[2020-11-23 17:36:58.402][13183][trace][connection] [external/envoy/source/common/network/raw_buffer_socket.cc:38] [C1] read error: Resource temporarily unavailable
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:522] [C1] parsing 158 bytes
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:713] [C1] message begin
[2020-11-23 17:36:58.402][13183][debug][http] [external/envoy/source/common/http/conn_manager_impl.cc:263] [C1] new stream
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:459] [C1] completed header: key=Host value=127.0.0.1:19000
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:459] [C1] completed header: key=User-Agent value=Consul Health Check
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:459] [C1] completed header: key=Accept value=text/plain, text/*, */*
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:459] [C1] completed header: key=Accept-Encoding value=gzip
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:625] [C1] onHeadersCompleteBase complete for headers
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:459] [C1] completed header: key=Connection value=close
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:813] [C1] Server: onHeadersComplete size=5
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:692] [C1] message complete
[2020-11-23 17:36:58.402][13183][debug][http] [external/envoy/source/common/http/conn_manager_impl.cc:732] [C1][S3807601570355532749] request headers complete (end_stream=true):
':authority', '127.0.0.1:19000'
':path', '/listeners'
':method', 'GET'
'user-agent', 'Consul Health Check'
'accept', 'text/plain, text/*, */*'
'accept-encoding', 'gzip'
'connection', 'close'

[2020-11-23 17:36:58.402][13183][debug][http] [external/envoy/source/common/http/conn_manager_impl.cc:1280] [C1][S3807601570355532749] request end stream
[2020-11-23 17:36:58.402][13183][debug][admin] [external/envoy/source/server/http/admin.cc:1336] [C1][S3807601570355532749] request complete: path: /listeners
[2020-11-23 17:36:58.402][13183][debug][http] [external/envoy/source/common/http/conn_manager_impl.cc:1557] [C1][S3807601570355532749] closing connection due to connection close header
[2020-11-23 17:36:58.402][13183][debug][http] [external/envoy/source/common/http/conn_manager_impl.cc:1619] [C1][S3807601570355532749] encoding headers via codec (end_stream=false):
':status', '200'
'content-type', 'text/plain; charset=UTF-8'
'cache-control', 'no-cache, max-age=0'
'x-content-type-options', 'nosniff'
'date', 'Mon, 23 Nov 2020 17:36:58 GMT'
'server', 'envoy'
'connection', 'close'

[2020-11-23 17:36:58.402][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:409] [C1] writing 228 bytes, end_stream false
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/conn_manager_impl.cc:1767] [C1][S3807601570355532749] encoding data via codec (size=158 end_stream=true)
[2020-11-23 17:36:58.402][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:409] [C1] writing 169 bytes, end_stream false
[2020-11-23 17:36:58.402][13183][trace][main] [external/envoy/source/common/event/dispatcher_impl.cc:162] item added to deferred deletion list (size=1)
[2020-11-23 17:36:58.402][13183][debug][connection] [external/envoy/source/common/network/connection_impl.cc:101] [C1] closing data_to_write=397 type=2
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/conn_manager_impl.cc:989] [C1][S3807601570355532749] decode headers called: filter=0x560c8e3f8480 status=1
[2020-11-23 17:36:58.402][13183][trace][http] [external/envoy/source/common/http/http1/codec_impl.cc:543] [C1] parsed 158 bytes
[2020-11-23 17:36:58.402][13183][debug][connection] [external/envoy/source/common/network/connection_impl.cc:101] [C1] closing data_to_write=397 type=2
[2020-11-23 17:36:58.402][13183][trace][main] [external/envoy/source/common/event/dispatcher_impl.cc:79] clearing deferred deletion list (size=1)
[2020-11-23 17:36:58.403][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:473] [C1] socket event: 2
[2020-11-23 17:36:58.403][13183][trace][connection] [external/envoy/source/common/network/connection_impl.cc:561] [C1] write ready
[2020-11-23 17:36:58.403][13183][trace][connection] [external/envoy/source/common/network/raw_buffer_socket.cc:67] [C1] write returns: 397
[2020-11-23 17:36:58.403][13183][debug][connection] [external/envoy/source/common/network/connection_impl.cc:600] [C1] write flush complete
[2020-11-23 17:36:58.403][13183][debug][connection] [external/envoy/source/common/network/connection_impl.cc:192] [C1] closing socket: 1
[2020-11-23 17:36:58.403][13183][debug][conn_handler] [external/envoy/source/server/connection_handler_impl.cc:86] [C1] adding to cleanup list
[2020-11-23 17:36:58.403][13183][trace][main] [external/envoy/source/common/event/dispatcher_impl.cc:162] item added to deferred deletion list (size=1)
[2020-11-23 17:36:58.403][13183][trace][main] [external/envoy/source/common/event/dispatcher_impl.cc:162] item added to deferred deletion list (size=2)
[2020-11-23 17:36:58.403][13183][trace][main] [external/envoy/source/common/event/dispatcher_impl.cc:79] clearing deferred deletion list (size=2)

[2020-11-23 17:37:05.411][13183][trace][http] [external/envoy/source/common/http/async_client_impl.cc:111] async http request response data (length=7004 end_stream=false)
[2020-11-23 17:37:05.411][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:136] Received gRPC message for type.googleapis.com/envoy.api.v2.Cluster at version 00000002
[2020-11-23 17:37:05.412][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:105] Pausing discovery requests for type.googleapis.com/envoy.api.v2.ClusterLoadAssignment
[2020-11-23 17:37:05.412][13183][info][upstream] [external/envoy/source/common/upstream/cds_api_impl.cc:74] cds: add 2 cluster(s), remove 2 cluster(s)
[2020-11-23 17:37:05.412][13183][debug][upstream] [external/envoy/source/common/upstream/cds_api_impl.cc:92] cds: add/update cluster 'local_app' skipped
[2020-11-23 17:37:05.412][13183][warning][misc] [external/envoy/source/common/protobuf/utility.cc:441] Using deprecated option 'envoy.api.v2.Cluster.tls_context' from file cluster.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/intro/deprecated for details.
[2020-11-23 17:37:05.432][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:105] Pausing discovery requests for type.googleapis.com/envoy.api.v2.Cluster
[2020-11-23 17:37:05.432][13183][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:576] add/update cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul starting warming
[2020-11-23 17:37:05.432][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:82] gRPC mux subscribe for type.googleapis.com/envoy.api.v2.ClusterLoadAssignment
[2020-11-23 17:37:05.432][13183][trace][config] [external/envoy/source/common/config/grpc_mux_impl.cc:45] API type.googleapis.com/envoy.api.v2.ClusterLoadAssignment paused during sendDiscoveryRequest(), setting pending.
[2020-11-23 17:37:05.432][13183][info][upstream] [external/envoy/source/common/upstream/cds_api_impl.cc:90] cds: add/update cluster 'backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul'
[2020-11-23 17:37:05.432][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:113] Resuming discovery requests for type.googleapis.com/envoy.api.v2.ClusterLoadAssignment

resource_names: "backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul"
type_url: "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"
response_nonce: "00000002"

[2020-11-23 17:37:05.432][13183][trace][router] [external/envoy/source/common/router/router.cc:1580] [C0][S511235682213879845] proxying 5884 bytes
[2020-11-23 17:37:05.432][13183][debug][init] [external/envoy/source/common/init/watcher_impl.cc:27] ClusterImplBase destroyed
[2020-11-23 17:37:05.432][13183][debug][init] [external/envoy/source/common/init/watcher_impl.cc:27] init manager Cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul destroyed
[2020-11-23 17:37:05.433][13183][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:588] warming cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul complete
[2020-11-23 17:37:05.433][13183][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:606] updating TLS cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul
[2020-11-23 17:37:05.433][13183][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:1084] membership update for TLS cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul added 2 removed 0
[2020-11-23 17:37:05.433][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:113] Resuming discovery requests for type.googleapis.com/envoy.api.v2.Cluster
[2020-11-23 17:37:05.433][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:606] updating TLS cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul
[2020-11-23 17:37:05.433][13213][debug][upstream] [external/envoy/source/common/upstream/cluster_manager_impl.cc:1084] membership update for TLS cluster backend-service.default.aws.internal.f5d1ffd7-c7dc-ea0f-bafb-96e96ceaa66f.consul added 2 removed 0

[2020-11-23 17:37:05.433][13183][trace][router] [external/envoy/source/common/router/router.cc:1580] [C0][S511235682213879845] proxying 5884 bytes
[2020-11-23 17:37:05.433][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:507] [C0] about to recv frame type=0, flags=0
[2020-11-23 17:37:05.433][13183][trace][http2] [external/envoy/source/common/http/http2/codec_impl.cc:522] [C0] recv frame type=0
[2020-11-23 17:37:05.433][13183][trace][http] [external/envoy/source/common/http/async_client_impl.cc:111] async http request response data (length=8528 end_stream=false)
[2020-11-23 17:37:05.433][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:136] Received gRPC message for type.googleapis.com/envoy.api.v2.Listener at version 00000002
[2020-11-23 17:37:05.434][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:105] Pausing discovery requests for type.googleapis.com/envoy.api.v2.RouteConfiguration
[2020-11-23 17:37:05.434][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:669] begin remove listener: name=envoy_prometheus_metrics_listener
[2020-11-23 17:37:05.434][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:677] unknown/locked listener 'envoy_prometheus_metrics_listener'. no remove
[2020-11-23 17:37:05.434][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:377] begin add/update listener: name=public_listener:10.249.57.121:8001 hash=1888268925375460456

[2020-11-23 17:37:05.440][13183][debug][config] [external/envoy/source/server/listener_impl.cc:395] add warming listener: name=public_listener:10.249.57.121:8001, hash=1888268925375460456, address=10.249.57.121:8001
[2020-11-23 17:37:05.440][13183][debug][init] [external/envoy/source/common/init/manager_impl.cc:45] init manager Listener public_listener:10.249.57.121:8001 contains no targets
[2020-11-23 17:37:05.440][13183][debug][init] [external/envoy/source/common/init/watcher_impl.cc:14] init manager Listener public_listener:10.249.57.121:8001 initialized, notifying ListenerImpl
[2020-11-23 17:37:05.440][13183][debug][config] [external/envoy/source/server/listener_impl.cc:395] warm complete. updating active listener: name=public_listener:10.249.57.121:8001, hash=1888268925375460456, address=10.249.57.121:8001
[2020-11-23 17:37:05.440][13183][debug][config] [external/envoy/source/server/listener_impl.cc:395] draining listener: name=public_listener:10.249.57.121:8001, hash=9172118973310572056, address=10.249.57.121:8001
[2020-11-23 17:37:05.440][13183][trace][main] [external/envoy/source/server/drain_manager_impl.cc:44] drain tick #0
[2020-11-23 17:37:05.440][13183][info][upstream] [external/envoy/source/server/lds_api.cc:73] lds: add/update listener 'public_listener:10.249.57.121:8001'
[2020-11-23 17:37:05.440][13183][warning][misc] [external/envoy/source/common/protobuf/utility.cc:441] Using deprecated option 'envoy.api.v2.listener.Filter.config' from file listener_components.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/intro/deprecated for details.
[2020-11-23 17:37:05.440][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:377] begin add/update listener: name=backend-service?dc=aws:127.0.0.1:9001 hash=13871615550858586218
[2020-11-23 17:37:05.440][13183][debug][config] [external/envoy/source/server/listener_manager_impl.cc:388] duplicate/locked listener 'backend-service?dc=aws:127.0.0.1:9001'. no add/update
[2020-11-23 17:37:05.440][13183][debug][upstream] [external/envoy/source/server/lds_api.cc:76] lds: add/update listener 'backend-service?dc=aws:127.0.0.1:9001' skipped
[2020-11-23 17:37:05.440][13183][debug][config] [external/envoy/source/common/config/grpc_mux_impl.cc:113] Resuming discovery requests for type.googleapis.com/envoy.api.v2.RouteConfiguration
[2020-11-23 17:37:05.440][13183][debug][config] [external/envoy/source/common/config/grpc_subscription_impl.cc:67] gRPC config for type.googleapis.com/envoy.api.v2.Listener accepted with 2 resources with version 00000002

Thanks in Advance.

After upgrading my server and Client to 1.9.0 and Envoy to 1.16.0, I still see the issue with filter chain updates. And envoy is performing the “in place filter chain updates”

I guess is this filter chain updates are part of the consul dynamic configuration??

In that case, I’m not sure how we can establish a long lived TCP connection between two services using Consul Connect and Envoy (if the filter chains are keep getting updated and it replaces the listeners)

[external/envoy/source/server/filter_chain_manager_impl.cc:215] new fc_contexts has 1 filter chains, including 1 newly built
[external/envoy/source/server/listener_impl.cc:653] add warming listener: name=public_listener:10.243.72.34:21002, hash=7325325565812112725, address=10.243.72.34:21002
[external/envoy/source/server/listener_impl.cc:662] Initialize listener public_listener:10.243.72.34:21002 local-init-manager.
[external/envoy/source/server/listener_impl.cc:653] execute in place filter chain update: name=public_listener:10.243.72.34:21002, hash=7325325565812112725, address=10.243.72.34:21002
[external/envoy/source/server/listener_manager_impl.cc:690] replacing existing listener 2
[external/envoy/source/server/connection_handler_impl.cc:199] replacing listener 
[external/envoy/source/server/listener_impl.cc:653] draining 1 filter chains in listener public_listener:10.243.72.34:21002: name=public_listener:10.243.72.34:21002, hash=7499858273634101412, address=10.243.72.34:21002