chown: /boundary/..2023_02_21_22_40_48.2030105668/boundary.hcl: Read-only file system chown: /boundary/..2023_02_21_22_40_48.2030105668: Read-only file system chown: /boundary/..2023_02_21_22_40_48.2030105668: Read-only file system chown: /boundary/..data: Read-only file system chown: /boundary/boundary.hcl: Read-only file system chown: /boundary: Read-only file system chown: /boundary: Read-only file system Could not chown /boundary (may not have appropriate permissions) Couldn't start Boundary with IPC_LOCK. Disabling IPC_LOCK, please use --privileged or --cap-add IPC_LOCK {"id":"FpYSHssmZy","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"configuring client automatic mTLS","purpose":"root"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.56614119Z"} {"id":"1KHtjFvFQS","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"worker.(Worker).startAuthRotationTicking","data":{"msg":"using kms worker authentication; pki auth rotation ticking not running"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.568578692Z"} {"id":"oW3qvmJVJr","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","args":["/tmp/1476209212/boundary-plugin-kms-awskms-FioMt"],"msg":"starting plugin","path":"/tmp/1476209212/boundary-plugin-kms-awskms-FioMt","purpose":"root"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.568689869Z"} {"id":"AJ97TNRWkV","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"plugin started","path":"/tmp/1476209212/boundary-plugin-kms-awskms-FioMt","pid":25,"purpose":"root"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.568761577Z"} {"id":"0RScLF8rFc","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"waiting for RPC address","path":"/tmp/1476209212/boundary-plugin-kms-awskms-FioMt","purpose":"root"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.568817972Z"} {"id":"LNOFIleTNw","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms.boundary-plugin-kms-awskms-FioMt","msg":"configuring server automatic mTLS","purpose":"root","timestamp":"2023-02-21T22:43:27.317Z"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.568873038Z"} {"id":"9XdJcyvort","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"using plugin","purpose":"root","version":1}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.568922063Z"} {"id":"JXl8KthSGF","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms.boundary-plugin-kms-awskms-FioMt","address":"/tmp/plugin51419437","msg":"plugin address","network":"unix","purpose":"root","timestamp":"2023-02-21T22:43:27.342Z"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.568968252Z"} {"id":"bh6luUN9zp","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms.stdio","msg":"waiting for stdio data","purpose":"root"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.569031914Z"} {"id":"gAgQhY0zsr","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"configuring client automatic mTLS","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.569104802Z"} {"id":"s4rrziKKQm","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","args":["/tmp/929640545/boundary-plugin-kms-awskms-HfcT7"],"msg":"starting plugin","path":"/tmp/929640545/boundary-plugin-kms-awskms-HfcT7","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.569169579Z"} {"id":"UeeT5LHNvY","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"plugin started","path":"/tmp/929640545/boundary-plugin-kms-awskms-HfcT7","pid":33,"purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.569215164Z"} {"id":"VmgXoqCSsF","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"waiting for RPC address","path":"/tmp/929640545/boundary-plugin-kms-awskms-HfcT7","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.569254227Z"} {"id":"rHzQJmq87i","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms.boundary-plugin-kms-awskms-HfcT7","msg":"configuring server automatic mTLS","purpose":"worker-auth","timestamp":"2023-02-21T22:43:28.047Z"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.569285204Z"} {"id":"vEYm9kuJf4","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"using plugin","purpose":"worker-auth","version":1}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.569314394Z"} {"id":"DkkaMed9YX","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms.boundary-plugin-kms-awskms-HfcT7","address":"/tmp/plugin1878380687","msg":"plugin address","network":"unix","purpose":"worker-auth","timestamp":"2023-02-21T22:43:28.072Z"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.569352285Z"} {"id":"JDAZiLJqyr","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms.stdio","msg":"waiting for stdio data","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:28.569395539Z"} ==> Boundary server configuration: AWS KMS KeyID: AWS KMS KeyID: AWS KMS Region: ap-northeast-1 AWS KMS Region: ap-northeast-1 Cgo: disabled Listener 1: tcp (addr: "0.0.0.0:9202", max_request_duration: "1m30s", purpose: "proxy") Log Level: info Mlock: supported: true, enabled: false Version: Boundary v0.12.0 Version Sha: 034f27ec8528251ce23b5afbf3851c48b0c54bec Worker Public Proxy Addr: worker.boundary.jpdev.rescale.com:9202 ==> Boundary server started! Log data will stream in below: {"id":"SQkwc2SOnM","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"worker.(Worker).upstreamDialerFunc","data":{"msg":"worker has successfully authenticated"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:29.379768057Z"} {"id":"Rtc4PhQ39Y","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"worker.(Worker).updateAddrs","data":{"msg":"Upstreams after first status set to: [controller.boundary.jpdev.rescale.com:9201]"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:29.689609272Z"} ==> Boundary server graceful shutdown triggered, interrupt again to enter shutdown {"id":"4pP0bviZ7D","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"worker.(Worker).GracefulShutdown","data":{"msg":"worker entering graceful shutdown"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:48.605585057Z"} {"id":"i5tjZKQcKQ","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","error":"signal: terminated","msg":"plugin process exited","path":"/tmp/929640545/boundary-plugin-kms-awskms-HfcT7","pid":33,"purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:48.607327948Z"} {"id":"5zezqmtH1N","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms.stdio","err":{},"msg":"received EOF, stopping recv loop","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:48.607522512Z"} {"id":"L4LazdegXF","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms.stdio","err":{},"msg":"received EOF, stopping recv loop","purpose":"root"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:48.607768378Z"} {"id":"EiYq4QE3WU","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","error":"signal: terminated","msg":"plugin process exited","path":"/tmp/1476209212/boundary-plugin-kms-awskms-FioMt","pid":25,"purpose":"root"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:48.608006096Z"} {"id":"IPwrGoZUT4","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"worker.(Worker).GracefulShutdown","data":{"msg":"worker connections have drained"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:52.112791436Z"} {"id":"t33li3LDvp","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"worker.(Worker).Shutdown","data":{"msg":"worker shutting down"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:52.112902723Z"} {"id":"5SqVUKH2C4","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"worker.(Worker).startStatusTicking","data":{"msg":"status ticking shutting down"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:54.11497238Z"} {"id":"AcI58qXgpG","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"worker.(Worker).Shutdown","data":{"msg":"worker finished shutting down"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:54.115092387Z"} {"id":"qyB0NDRJRP","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"plugin exited","purpose":"root"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:54.123575035Z"} {"id":"haVzho35Hg","source":"https://hashicorp.com/boundary/deployment-boundary-77884b6469-n6zxt/worker","specversion":"1.0","type":"system","data":{"version":"v0.1","op":"github.com/hashicorp/boundary/internal/observability/event.(*HclogLoggerAdapter).writeEvent","data":{"@original-log-level":"none","@original-log-name":"awskms","msg":"plugin exited","purpose":"worker-auth"}},"datacontentype":"application/cloudevents","time":"2023-02-21T22:43:54.126455188Z"} Error running shutdown tasks: 2 errors occurred: * Error finalizing kms of type awskms and purpose root: rpc error: code = Unavailable desc = connection error: desc = "transport: error while dialing: dial unix /tmp/plugin51419437: connect: connection refused" * Error finalizing kms of type awskms and purpose worker-auth: rpc error: code = Unavailable desc = connection error: desc = "transport: error while dialing: dial unix /tmp/plugin1878380687: connect: connection refused"