TLS Handshake Error with Boundary Deployed in Nomad Cluster

Boundary
1

I tried deploying Boundary using the docker image in our Consul Nomad Cluster. I have been at it now for a while and I am stuck. I need help with the validation of the configuration and what I am doing wrong.

job "boundary-job" {
  datacenters = ["us-west1-a", "us-west1-b", "us-west1-c"]

  group "boundary-controller-group" {
    count = 1

    update {
      max_parallel     = 1
      min_healthy_time = "30s"
      healthy_deadline = "5m"
      auto_revert = true
    }

    network {
      mode ="bridge"

      port "api" {
        static = 9200
      }

      port "cluster" {
        static = 9201
      }

      port "ops" {
        static = 9202
      }
    }

    service {
      name = "boundary-controller-service-api"
      port = "api"

      check {
        type     = "http"
        interval = "10s"
        port     = "api"
        timeout  = "2s"
        path     = "/"
      }

      connect {
        sidecar_service {
          proxy {
            upstreams {
              local_bind_port = 5432
              destination_name = "postgres-db-service"
            }
          }
        }
      }
    }

    service{
      name = "boundary-controller-service-cluster"
      port = "cluster"

      connect {
        sidecar_service { }
      }
    }

    task "server" {
      driver = "docker"

      config {
        image        = "hashicorp/boundary"
        ports        = ["api", "cluster", "ops"]
        command      = "/bin/sh"
        args         = ["-c", "boundary server -config local/boundary.hcl"]
      }

      template {
        data = <<EOF
{{ with secret "partsimony/credentials" }}
{{ .Data.service_account }}
{{ end }}
EOF
        destination   = "local/service_account.json"
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }

      template {
        data = <<EOF
# Disable memory lock: https://www.man7.org/linux/man-pages/man2/mlock.2.html
disable_mlock = true

# Controller configuration block
controller {
# This name attr must be unique across all controller instances if running in HA mode
name = "{{env "NOMAD_ALLOC_ID"}}"
public_cluster_addr = "{{env "NOMAD_HOST_ADDR_cluster"}}"

{{ with secret "partsimony/credentials" }}
database {
  url = "postgresql://{{ .Data.database_username }}:{{ .Data.database_password }}@{{ env "NOMAD_UPSTREAM_ADDR_postgres-db-service"}}/boundary-db"
}
{{ end }}
}

# API listener configuration block
listener "tcp" {
# Should be the address of the NIC that the controller server will be reached on
address = "0.0.0.0:9200"
# The purpose of this listener block
purpose = "api"

tls_disable = true

# Uncomment to enable CORS for the Admin UI. Be sure to set the allowed origin(s)
# to appropriate values.
cors_enabled = true
cors_allowed_origins = ["*"]
}

# Data-plane listener configuration block (used for worker coordination)
listener "tcp" {
# Should be the IP of the NIC that the worker will connect on
address = "0.0.0.0:9201"
# The purpose of this listener
purpose = "cluster"

tls_disable = true
tls_min_version = "tls13"
}

listener "tcp" {
purpose = "ops"
tls_disable = true
address = "0.0.0.0:9202"
}

# Root KMS configuration block: this is the root key for Boundary
# Use a production KMS such as AWS KMS in production installs
kms "gcpckms" {
purpose     = "root"
credentials = "/local/service_account.json"
project     = "partsimony-infrastructure-dev"
region      = "global"
key_ring    = "pb-root"
crypto_key  = "pb-root-key"
}

# Worker authorization KMS
# Use a production KMS such as AWS KMS for production installs
# This key is the same key used in the worker configuration

kms "gcpckms" {
purpose     = "worker-auth"
credentials = "/local/service_account.json"
project     = "partsimony-infrastructure-dev"
region      = "global"
key_ring    = "pb-worker-auth"
crypto_key  = "pb-worker-auth-key"
}

# Recovery KMS block: configures the recovery key for Boundary
# Use a production KMS such as AWS KMS for production installs

kms "gcpckms" {
purpose     = "recovery"
credentials = "/local/service_account.json"
project     = "partsimony-infrastructure-dev"
region      = "global"
key_ring    = "partsimony-boundary-keyring"
crypto_key  = "partsimony-boundary-key"
}
EOF

        destination = "local/boundary.hcl"
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }
    }
  }

  group "boundary-worker-group" {
    count = 1

    update {
      max_parallel     = 1
      min_healthy_time = "30s"
      healthy_deadline = "5m"
      auto_revert = true
    }

    network {
      mode ="bridge"

      port "proxy" {
        static = 9206
      }
    }

    service {
      name = "boundary-worker-service"
      port = "proxy"

      connect {
        sidecar_service {
          proxy {
            upstreams {
              local_bind_port = 5432
              destination_name = "postgres-db-service"
            }
            upstreams {
              local_bind_port = 9201
              destination_name = "boundary-controller-service-cluster"
            }
          }
        }
      }
    }

    task "server" {
      driver = "docker"

      config {
        image        = "hashicorp/boundary"
        ports        = ["proxy"]
        command      = "/bin/sh"
        args         = ["-c", "boundary server -config local/boundary.hcl"]
      }

      template {
        data = <<EOF
{{ with secret "partsimony/credentials" }}
{{ .Data.service_account }}
{{ end }}
EOF
        destination   = "local/service_account.json"
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }

      template {
        data = <<EOF
disable_mlock = true

worker {
# Name attr must be unique across workers
name = "{{env "NOMAD_ALLOC_ID"}}-worker"
description = "{{env "NOMAD_ALLOC_ID"}}-worker"

# Workers must be able to reach controllers on :9201
controllers = [
"{{env "NOMAD_UPSTREAM_ADDR_boundary_controller_service_cluster"}}",
# {{ range service "boundary-controller-service-cluster" }}
#         "{{ .Address }}:9201",
# {{ end }}
]
address = "{{env "NOMAD_HOST_ADDR_proxy"}}"
public_addr = "{{env "NOMAD_HOST_ADDR_proxy"}}"
}

listener "tcp" {
purpose = "proxy"
tls_disable = true
address = "0.0.0.0:9206"
tls_min_version = "tls13"
}

kms "gcpckms" {
purpose     = "worker-auth"
credentials = "/local/service_account.json"
project     = "partsimony-infrastructure-dev"
region      = "global"
key_ring    = "pb-worker-auth"
crypto_key  = "pb-worker-auth-key"
}
EOF

        destination = "local/boundary.hcl"
        change_mode   = "signal"
        change_signal = "SIGHUP"
      }
    }
  }

  group "postgres-db-service-gateway-group" {
    count = 1
    
    network {
      mode = "bridge"
    }

    service {
      name = "postgres-db-service-gateway"

      connect {
        gateway {
          terminating {
            service {
              name = "postgres-db-service"
            }
          }
        }
      }
    }
  }

  group "boundary-controller-service-cluster-gateway-group" {
    count = 1
    
    network {
      mode = "bridge"
    }

    service {
      name = "boundary-controller-service-cluster-gateway"

      connect {
        gateway {
          terminating {
            service {
              name = "boundary-controller-service-cluster"
            }
          }
        }
      }
    }
  }
}
2

Generally TLS handshake errors are seen if you are running some kind of TLS-terminating proxy in front of the cluster or worker ports. If you are running a proxy or load balancer in front of those they need to be configured to be TCP-only.

3

Thanks @jeff but to be honest, I am not running the application behind any load balancer or proxy. I am using connect in Nomad. Is this a problem?

4

Here is the last log I got

{"id":"o7zAh1STTn","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"gcpckms","msg":"configuring client automatic mTLS","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.293152056Z"}
{"id":"qPDWgGlYAL","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"gcpckms","args":["/tmp/199767370/boundary-plugin-kms-gcpckms.gz"],"msg":"starting plugin","path":"/tmp/199767370/boundary-plugin-kms-gcpckms.gz","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.293534966Z"}
{"id":"9uCcjsM3a6","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"gcpckms","msg":"plugin started","path":"/tmp/199767370/boundary-plugin-kms-gcpckms.gz","pid":20,"purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.293690286Z"}
{"id":"1ooF8a565j","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"gcpckms","msg":"waiting for RPC address","path":"/tmp/199767370/boundary-plugin-kms-gcpckms.gz","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.294487349Z"}
{"id":"UlOx039qHo","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"gcpckms.boundary-plugin-kms-gcpckms.gz","msg":"configuring server automatic mTLS","purpose":"worker-auth","timestamp":"2022-05-09T15:49:30.035Z"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.294844829Z"}
{"id":"jm52IKMddl","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"gcpckms","msg":"using plugin","purpose":"worker-auth","version":1}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.295221728Z"}
{"id":"yOpRPeUNmE","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"gcpckms.boundary-plugin-kms-gcpckms.gz","address":"/tmp/plugin4205550915","msg":"plugin address","network":"unix","purpose":"worker-auth","timestamp":"2022-05-09T15:49:30.155Z"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.295456732Z"}
{"id":"IFeqIDpr3T","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"gcpckms.stdio","msg":"waiting for stdio data","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.295716357Z"}
{"id":"cXhGIchn0C","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"worker.(Worker).createClientConn","data":{"address":":9201","msg":"connected to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.297290038Z"}
{"id":"RLqKOMjXHr","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_WYe5cpu87g","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:30.333336593Z"}
{"id":"uEnlS9Eeqt","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"worker.listeners: http: TLS handshake error from 127.0.0.1:40736: EOF","error_fields":{},"id":"e_tSWRzUFRZm","version":"v0.1","op":"log.(*Logger).Output"},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:31.991372227Z"}
{"id":"fjGDpbC5QK","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_2bLFMcy0V1","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:32.20295748Z"}
{"id":"iQ9H4LAOoE","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_UY8jGIJmKr","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:34.65594515Z"}
{"id":"o2tXbwiPCf","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_xMFrkdw44I","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:36.463513122Z"}
{"id":"6AbPyvOMGz","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_6rh4P3C0Ou","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:38.058586989Z"}
{"id":"LFAvL4QyWH","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_hlH4RfU0js","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:39.978654197Z"}
{"id":"B2ReS9TMXR","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"worker.listeners: http: TLS handshake error from 127.0.0.1:40792: EOF","error_fields":{},"id":"e_qOWB8HFTsF","version":"v0.1","op":"log.(*Logger).Output"},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:41.992427517Z"}
{"id":"5NkkF6Of1x","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_s0CIFOttrk","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:42.138676002Z"}
{"id":"KUZLjP4aQC","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_LnTuciorHa","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:44.075892815Z"}
{"id":"vVuQbhLtW1","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_pfDBwu5svG","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:45.580498224Z"}
{"id":"oQ5u033h7e","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"status error grace period has expired, canceling all sessions on worker","error_fields":{},"id":"e_Azg4xPSLh2","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"grace_period":15000000000,"last_status_time":"2022-05-09 15:49:30.29789385 +0000 UTC m=+0.594789505"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:45.58069693Z"}
{"id":"OgGB9Eleyr","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_3oDIXWurYh","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:47.990763846Z"}
{"id":"WBm9m0MPHv","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"status error grace period has expired, canceling all sessions on worker","error_fields":{},"id":"e_ldav8gKDoe","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"grace_period":15000000000,"last_status_time":"2022-05-09 15:49:30.29789385 +0000 UTC m=+0.594789505"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:47.991056366Z"}
{"id":"VffgYgipgk","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_agBCvOEPHG","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:50.364828723Z"}
{"id":"CNgn1uV4ZG","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"status error grace period has expired, canceling all sessions on worker","error_fields":{},"id":"e_8o6Nifz88H","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"grace_period":15000000000,"last_status_time":"2022-05-09 15:49:30.29789385 +0000 UTC m=+0.594789505"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:50.365066002Z"}
{"id":"Z27d2mNNAb","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_Fro1BgT1tQ","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:51.993291748Z"}
{"id":"P8UPlHhA2d","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"status error grace period has expired, canceling all sessions on worker","error_fields":{},"id":"e_QAAiv1IO6w","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"grace_period":15000000000,"last_status_time":"2022-05-09 15:49:30.29789385 +0000 UTC m=+0.594789505"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:51.993508661Z"}
{"id":"3mdDJZkqVZ","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"worker.listeners: http: TLS handshake error from 127.0.0.1:40846: EOF","error_fields":{},"id":"e_c2pPCTo9No","version":"v0.1","op":"log.(*Logger).Output"},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:51.99368503Z"}
{"id":"wviUzPvddq","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_ulXh263r6c","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:54.4470925Z"}
{"id":"nROgRwAafV","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"status error grace period has expired, canceling all sessions on worker","error_fields":{},"id":"e_2X4yXLwp2c","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"grace_period":15000000000,"last_status_time":"2022-05-09 15:49:30.29789385 +0000 UTC m=+0.594789505"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:54.447375702Z"}
{"id":"PFgJcaOjqU","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_uJkbWqQG6l","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:56.06503591Z"}
{"id":"qVD0pbJUbE","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"status error grace period has expired, canceling all sessions on worker","error_fields":{},"id":"e_z440Ut3pXd","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"grace_period":15000000000,"last_status_time":"2022-05-09 15:49:30.29789385 +0000 UTC m=+0.594789505"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:56.065869604Z"}
{"id":"AHuncLFhF6","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"rpc error: code = Unavailable desc = last connection error: connection error: desc = \"transport: Error while dialing unable to write connection nonce: EOF\"","error_fields":{},"id":"e_kbln5vU5ht","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"msg":"error making status request to controller"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:57.581543539Z"}
{"id":"8kGwUM3YAo","source":"https://hashicorp.com/boundary/750e645d-9536-c344-d0f6-45af1e6a5a22-worker","specversion":"1.0","type":"error","data":{"error":"status error grace period has expired, canceling all sessions on worker","error_fields":{},"id":"e_6x2B8h3L2E","version":"v0.1","op":"worker.(Worker).sendWorkerStatus","info":{"grace_period":15000000000,"last_status_time":"2022-05-09 15:49:30.29789385 +0000 UTC m=+0.594789505"}},"datacontentype":"application/cloudevents","time":"2022-05-09T15:49:57.581743791Z"}