--> Formatting HCL ==> Removing old development build... ==> Building pkg/linux_arm64/nomad with tags ui ... ==> Running Nomad test suites: gotestsum -- \ \ -cover \ -timeout=100m \ -tags "ui " \ "./..." ✓ acl (cached) (coverage: 84.1% of statements) ✓ . (cached) (coverage: 1.7% of statements) ✓ client/allocdir (60ms) (coverage: 61.5% of statements) ✓ client/allochealth (cached) (coverage: 62.8% of statements) ✓ client/allocrunner (cached) (coverage: 65.7% of statements) ✖ client/allocrunner/taskrunner (1m48.063s) (coverage: 71.0% of statements) ✓ client/allocrunner/taskrunner/getter (cached) (coverage: 92.9% of statements) ✓ client/allocrunner/taskrunner/restarts (cached) (coverage: 78.5% of statements) ✓ client/allocrunner/taskrunner/template (cached) (coverage: 85.8% of statements) ✓ client/allocwatcher (cached) (coverage: 42.7% of statements) ✓ client/config (cached) (coverage: 7.0% of statements) ✓ client/consul (cached) (coverage: 9.5% of statements) ✓ client/devicemanager (cached) (coverage: 69.5% of statements) ✓ client/dynamicplugins (cached) (coverage: 75.8% of statements) ✖ client/fingerprint (2.58s) (coverage: 79.2% of statements) ✖ client/lib/cgutil (46ms) (coverage: 51.0% of statements) ✓ client/lib/fifo (cached) (coverage: 83.3% of statements) ✓ client/lib/streamframer (cached) (coverage: 89.7% of statements) ✓ client/logmon (cached) (coverage: 62.7% of statements) ✓ client/logmon/logging (cached) (coverage: 75.6% of statements) ✓ client/pluginmanager (cached) (coverage: 89.1% of statements) ✓ client/pluginmanager/csimanager (cached) (coverage: 82.4% of statements) ✓ client/pluginmanager/drivermanager (cached) (coverage: 55.4% of statements) ✓ client/servers (cached) (coverage: 80.4% of statements) ✓ client/state (cached) (coverage: 71.7% of statements) ✖ client/stats (2.071s) (coverage: 81.0% of statements) ✓ client/structs (cached) (coverage: 35.0% of statements) ✓ client/taskenv (cached) (coverage: 93.1% of statements) ✓ client/vaultclient (cached) (coverage: 54.1% of statements) ✖ client (2m48.894s) (coverage: 73.7% of statements) ∅ client/allocdir/input ∅ client/allocrunner/interfaces ∅ client/allocrunner/state ∅ client/allocrunner/taskrunner/interfaces ∅ client/allocrunner/taskrunner/state ∅ client/devicemanager/state ∅ client/interfaces ∅ client/lib/nsutil ∅ client/logmon/proto ∅ client/pluginmanager/drivermanager/state ∅ client/testutil ✖ command (1m40.625s) (coverage: 43.6% of statements) ✓ command/agent/consul (cached) (coverage: 76.6% of statements) ✓ command/agent/host (cached) (coverage: 90.0% of statements) ✓ command/agent/monitor (cached) (coverage: 81.4% of statements) ✓ command/agent/pprof (cached) (coverage: 86.1% of statements) ✓ devices/gpu/nvidia (cached) (coverage: 75.7% of statements) ✓ devices/gpu/nvidia/nvml (cached) (coverage: 50.0% of statements) ✓ command/agent (1m17.159s) (coverage: 67.4% of statements) ∅ command/agent/event ∅ command/raft_tools ∅ demo/digitalocean/app ∅ devices/gpu/nvidia/cmd ✖ drivers/docker/docklog (10.966s) (coverage: 36.1% of statements) ✖ drivers/docker (5m5.806s) ∅ drivers/docker/cmd ∅ drivers/docker/docklog/proto ∅ drivers/docker/util ✓ drivers/java (cached) (coverage: 56.2% of statements) ✓ drivers/mock (cached) (coverage: 1.1% of statements) ✓ drivers/qemu (cached) (coverage: 3.8% of statements) ✓ drivers/rawexec (cached) (coverage: 68.9% of statements) ✓ drivers/shared/capabilities (cached) (coverage: 96.4% of statements) ✓ drivers/shared/eventer (cached) (coverage: 70.7% of statements) ✖ drivers/exec (8m7.574s) (coverage: 66.5% of statements) ✓ drivers/shared/hostnames (cached) (coverage: 95.8% of statements) ✓ drivers/shared/resolvconf (cached) (coverage: 27.0% of statements) ✓ e2e (cached) ✓ e2e/connect (cached) (coverage: 1.7% of statements) ✓ e2e/vaultcompat (cached) ✓ helper (cached) (coverage: 35.1% of statements) ✓ helper/args (cached) (coverage: 87.5% of statements) ✓ helper/boltdd (cached) (coverage: 78.5% of statements) ✓ helper/constraints/semver (cached) (coverage: 97.2% of statements) ✓ helper/envoy (cached) (coverage: 66.7% of statements) ✓ helper/escapingio (cached) (coverage: 100.0% of statements) ✓ helper/fields (cached) (coverage: 62.7% of statements) ✓ helper/flags (cached) (coverage: 66.7% of statements) ✓ helper/flatmap (cached) (coverage: 78.0% of statements) ✓ helper/freeport (cached) (coverage: 81.7% of statements) ✓ helper/gated-writer (cached) (coverage: 100.0% of statements) ✓ helper/pluginutils/hclspecutils (cached) (coverage: 79.6% of statements) ✓ helper/pluginutils/hclutils (cached) (coverage: 78.8% of statements) ✓ helper/pluginutils/loader (cached) (coverage: 75.9% of statements) ✓ helper/pluginutils/singleton (cached) (coverage: 92.9% of statements) ✓ helper/pool (cached) (coverage: 29.4% of statements) ✓ helper/raftutil (cached) (coverage: 9.9% of statements) ✓ helper/snapshot (cached) (coverage: 76.4% of statements) ✓ helper/tlsutil (cached) (coverage: 81.2% of statements) ✓ helper/useragent (cached) (coverage: 50.0% of statements) ✓ helper/uuid (cached) (coverage: 60.0% of statements) ✓ internal/testing/apitests (cached) ✓ jobspec (100ms) (coverage: 76.7% of statements) ✓ jobspec2 (139ms) (coverage: 75.9% of statements) ✓ lib/circbufwriter (cached) (coverage: 91.7% of statements) ✓ lib/cpuset (cached) (coverage: 94.8% of statements) ✓ lib/delayheap (cached) (coverage: 67.9% of statements) ✓ lib/kheap (cached) (coverage: 70.8% of statements) ✓ nomad (cached) (coverage: 76.6% of statements) ✓ nomad/deploymentwatcher (cached) (coverage: 83.1% of statements) ✓ nomad/drainer (cached) (coverage: 59.4% of statements) ✓ nomad/state (cached) (coverage: 72.2% of statements) ✓ nomad/stream (cached) (coverage: 84.1% of statements) ✓ nomad/structs (cached) (coverage: 66.5% of statements) ✓ nomad/structs/config (cached) (coverage: 73.2% of statements) ✓ nomad/volumewatcher (cached) (coverage: 87.6% of statements) ✓ plugins/base (cached) (coverage: 64.5% of statements) ✓ plugins/csi (cached) (coverage: 65.9% of statements) ✓ plugins/device (cached) (coverage: 59.7% of statements) ✓ plugins/drivers (cached) (coverage: 14.1% of statements) ✓ plugins/drivers/testutils (cached) (coverage: 7.5% of statements) ✓ plugins/shared/structs (cached) (coverage: 48.2% of statements) ✓ scheduler (cached) (coverage: 90.1% of statements) ✓ testutil (cached) (coverage: 3.9% of statements) ✖ drivers/shared/executor (28.887s) ∅ drivers/shared/executor/proto ∅ e2e/affinities ∅ e2e/cli ∅ e2e/cli/command ∅ e2e/clientstate ∅ e2e/consul ∅ e2e/consulacls ∅ e2e/consultemplate ∅ e2e/csi ∅ e2e/deployment ∅ e2e/e2eutil ∅ e2e/events ∅ e2e/example ∅ e2e/execagent ∅ e2e/framework ∅ e2e/isolation ∅ e2e/lifecycle ∅ e2e/metrics ∅ e2e/namespaces ∅ e2e/networking ∅ e2e/nodedrain ∅ e2e/nomad09upgrade ∅ e2e/nomadexec ∅ e2e/oversubscription ∅ e2e/parameterized ∅ e2e/periodic ∅ e2e/podman ∅ e2e/quotas ∅ e2e/remotetasks ∅ e2e/rescheduling ∅ e2e/scaling ∅ e2e/scalingpolicies ∅ e2e/scheduler_sysbatch ∅ e2e/scheduler_system ∅ e2e/spread ∅ e2e/taskevents ∅ e2e/vaultsecrets ∅ e2e/volumes ∅ helper/codec ∅ helper/discover ∅ helper/exptime ∅ helper/grpc-middleware/logging ∅ helper/logging ∅ helper/mount ∅ helper/noxssrw ∅ helper/pluginutils/catalog ∅ helper/pluginutils/grpcutils ∅ helper/stats ∅ helper/testlog ∅ helper/testtask ∅ helper/winsvc ∅ jobspec2/addrs ∅ jobspec2/hclutil ∅ nomad/mock ∅ plugins ∅ plugins/base/proto ∅ plugins/base/structs ∅ plugins/csi/fake ∅ plugins/csi/testing ∅ plugins/device/cmd/example ∅ plugins/device/cmd/example/cmd ∅ plugins/device/proto ∅ plugins/drivers/proto ∅ plugins/drivers/utils ∅ plugins/shared/cmd/launcher ∅ plugins/shared/cmd/launcher/command ∅ plugins/shared/hclspec ∅ plugins/shared/structs/proto ∅ version === Skipped === SKIP: client/allocdir TestLinuxUnprivilegedSecretDir (0.00s) fs_linux_test.go:113: Must not be run as root === SKIP: client/allocdir TestTaskDir_NonRoot_Image (0.00s) task_dir_test.go:91: test should be run as non-root user === SKIP: client/allocdir TestTaskDir_NonRoot (0.00s) task_dir_test.go:114: test should be run as non-root user === SKIP: client/allocrunner/taskrunner TestSIDSHook_recoverToken_unReadable (0.00s) sids_hook_test.go:97: test only works as non-root === SKIP: client/allocrunner/taskrunner TestSIDSHook_writeToken_unWritable (0.00s) sids_hook_test.go:144: test only works as non-root === SKIP: client/allocrunner/taskrunner TestTaskRunner_DeriveSIToken_UnWritableTokenFile (0.00s) sids_hook_test.go:272: test only works as non-root === SKIP: client/allocrunner/taskrunner TestEnvoyBootstrapHook_maybeLoadSIToken (0.00s) === PAUSE TestEnvoyBootstrapHook_maybeLoadSIToken === CONT TestEnvoyBootstrapHook_maybeLoadSIToken envoy_bootstrap_hook_test.go:60: test only works as non-root === SKIP: client/pluginmanager/csimanager TestVolumeManager_ensureStagingDir/Returns_positive_mount_info (0.00s) === SKIP: drivers/docker TestDockerDriver_AdvertiseIPv6Address (0.03s) === PAUSE TestDockerDriver_AdvertiseIPv6Address === CONT TestDockerDriver_AdvertiseIPv6Address 2021-09-13T07:10:09.446Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 driver_test.go:2671: IPv6 not enabled on bridge network, skipping === SKIP: drivers/exec TestExecDriver_Fingerprint_NonLinux (0.00s) === PAUSE TestExecDriver_Fingerprint_NonLinux === CONT TestExecDriver_Fingerprint_NonLinux driver_test.go:60: Test only available not on Linux === SKIP: drivers/qemu TestQemuDriver_Start_Wait_Stop (0.00s) driver_compatible.go:48: Must have Qemu installed for Qemu specific tests to run === SKIP: drivers/qemu TestQemuDriver_GetMonitorPathOldQemu (0.00s) driver_compatible.go:48: Must have Qemu installed for Qemu specific tests to run === SKIP: drivers/qemu TestQemuDriver_GetMonitorPathNewQemu (0.00s) driver_compatible.go:48: Must have Qemu installed for Qemu specific tests to run === SKIP: drivers/qemu TestQemuDriver_User (0.00s) driver_compatible.go:48: Must have Qemu installed for Qemu specific tests to run === SKIP: drivers/qemu TestQemuDriver_Stats (0.00s) driver_compatible.go:48: Must have Qemu installed for Qemu specific tests to run === SKIP: drivers/qemu TestQemuDriver_Fingerprint (0.00s) driver_compatible.go:48: Must have Qemu installed for Qemu specific tests to run === SKIP: e2e TestE2E (0.00s) e2e_test.go:48: Skipping e2e tests, NOMAD_E2E not set === SKIP: e2e/vaultcompat TestVaultCompatibility (0.00s) vault_test.go:304: skipping test in non-integration mode: add -integration flag to run === SKIP: helper/tlsutil TestConfig_outgoingWrapper_BadCert (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/bad-artifact.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/bad-ports.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/basic_wrong_key.hcl (0.01s) === SKIP: jobspec2 TestEquivalentToHCL1/multi-network.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/multi-resource.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/multi-vault.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/overlapping-ports.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/service-check-bad-header-2.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/service-check-bad-header.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/task-scaling-policy-invalid-resource.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/task-scaling-policy-invalid-type.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/task-scaling-policy-missing-name.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/task-scaling-policy-multi-cpu.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/task-scaling-policy-multi-name.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/tg-scaling-policy-invalid-type.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/tg-scaling-policy-missing-max.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/tg-scaling-policy-multi-policy.hcl (0.00s) === SKIP: jobspec2 TestEquivalentToHCL1/tg-scaling-policy-with-label.hcl (0.00s) === SKIP: nomad TestAutopilot_CleanupStaleRaftServer (0.00s) autopilot_test.go:263: TestAutopilot_CleanupDeadServer is very flaky, removing it for now === SKIP: nomad/state TestEventsFromChanges_JobBatchDeregisterRequestType (0.00s) === SKIP: nomad/state TestEventsFromChanges_AllocClientUpdateRequestType (0.00s) === SKIP: nomad/state TestEventsFromChanges_AllocUpdateRequestType (0.00s) === SKIP: nomad/state TestEventsFromChanges_JobDeregisterRequestType (0.00s) === SKIP: nomad/structs TestNetworkIndex_Overcommitted (0.00s) network_test.go:13: === SKIP: scheduler TestBinPackIterator_Network_Failure (0.00s) rank_test.go:382: === Failed === FAIL: client TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot (60.32s) === PAUSE TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot === CONT TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot 2021-09-13T07:05:06.938Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.938Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.938Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.938Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.938Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.938Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.939Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.939Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.939Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.939Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.940Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:05:06.940Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= nomad-023 2021-09-13T07:05:06.940Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/api.go:549: nomad.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:127.0.0.1:9645 Address:127.0.0.1:9645}]" nomad-023 2021-09-13T07:05:06.940Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:152: nomad.raft: entering follower state: follower="Node at 127.0.0.1:9645 [Follower]" leader= nomad-023 2021-09-13T07:05:06.941Z [INFO] go-hclog@v0.14.1/stdlog.go:47: nomad: serf: EventMemberJoin: nomad-023.global 127.0.0.1 nomad-023 2021-09-13T07:05:06.941Z [INFO] nomad/server.go:1465: nomad: starting scheduling worker(s): num_workers=2 schedulers=[service, batch, system, sysbatch, _core] nomad-023 2021-09-13T07:05:06.941Z [INFO] nomad/serf.go:60: nomad: adding server: server="nomad-023.global (Addr: 127.0.0.1:9645) (DC: dc1)" 2021-09-13T07:05:07.012Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest1265467260/client 2021-09-13T07:05:07.012Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest1265467260/allocs 2021-09-13T07:05:07.012Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.012Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.012Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.012Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.012Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.012Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.012Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.012Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.012Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.013Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.013Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.013Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= nomad-023 2021-09-13T07:05:07.028Z [WARN] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:214: nomad.raft: heartbeat timeout reached, starting election: last-leader= nomad-023 2021-09-13T07:05:07.028Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:250: nomad.raft: entering candidate state: node="Node at 127.0.0.1:9645 [Candidate]" term=2 nomad-023 2021-09-13T07:05:07.028Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:268: nomad.raft: votes: needed=1 nomad-023 2021-09-13T07:05:07.028Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:287: nomad.raft: vote granted: from=127.0.0.1:9645 term=2 tally=1 nomad-023 2021-09-13T07:05:07.028Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:292: nomad.raft: election won: tally=1 nomad-023 2021-09-13T07:05:07.028Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:363: nomad.raft: entering leader state: leader="Node at 127.0.0.1:9645 [Leader]" nomad-023 2021-09-13T07:05:07.028Z [INFO] nomad/leader.go:75: nomad: cluster leadership acquired nomad-023 2021-09-13T07:05:07.028Z [TRACE] nomad/fsm.go:337: nomad.fsm: ClusterSetMetadata: cluster_id=96446757-63fb-d15b-be50-bfbb2ec312e8 create_time=1631516707028596794 nomad-023 2021-09-13T07:05:07.028Z [INFO] nomad/leader.go:1628: nomad.core: established cluster id: cluster_id=96446757-63fb-d15b-be50-bfbb2ec312e8 create_time=1631516707028596794 nomad-023 2021-09-13T07:05:07.028Z [TRACE] drainer/watch_jobs.go:145: nomad.drain.job_watcher: getting job allocs at index: index=1 2021-09-13T07:05:07.033Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:05:07.033Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_azure, env_aws, env_gce] 2021-09-13T07:05:07.034Z [INFO] fingerprint/cgroup_linux.go:37: client.fingerprint_mgr.cgroup: cgroups are available 2021-09-13T07:05:07.034Z [DEBUG] fingerprint/cni.go:26: client.fingerprint_mgr: CNI config dir is not set or does not exist, skipping: cni_config_dir=/opt/cni/config 2021-09-13T07:05:07.034Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=cgroup period=15s 2021-09-13T07:05:07.034Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:05:07.034Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:05:07.034Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:05:07.034Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=consul period=15s 2021-09-13T07:05:07.037Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:05:07.037Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:05:07.037Z [DEBUG] fingerprint/network.go:92: client.fingerprint_mgr.network: link speed could not be detected and no speed specified by user, falling back to default speed: mbits=1000 2021-09-13T07:05:07.037Z [DEBUG] fingerprint/network.go:112: client.fingerprint_mgr.network: detected interface IP: interface=ens5 IP=172.31.11.103 2021-09-13T07:05:07.038Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=lo 2021-09-13T07:05:07.038Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/lo/speed 2021-09-13T07:05:07.038Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:05:07.041Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:05:07.041Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:05:07.041Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:05:07.044Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=docker0 2021-09-13T07:05:07.044Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/docker0/speed 2021-09-13T07:05:07.044Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:05:07.047Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=br-8a6b4474e385 2021-09-13T07:05:07.047Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/br-8a6b4474e385/speed 2021-09-13T07:05:07.047Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:05:07.050Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=vault period=15s 2021-09-13T07:05:07.056Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=instance-type 2021-09-13T07:05:07.060Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=placement/availability-zone 2021-09-13T07:05:07.062Z [DEBUG] fingerprint/env_aws.go:157: client.fingerprint_mgr.env_aws: read an empty value: attribute=network/interfaces/macs/02:2e:4a:38:c9:c6/ipv6s 2021-09-13T07:05:07.063Z [WARN] fingerprint/env_aws.go:220: client.fingerprint_mgr.env_aws: failed to read EC2 metadata instance-type: error="RequestError: send request failed caused by: Get "http://169.254.169.254/latest/meta-data/instance-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:05:07.063Z [WARN] fingerprint/env_aws.go:194: client.fingerprint_mgr.env_aws: failed to find the cpu specification for this instance type 2021-09-13T07:05:07.065Z [DEBUG] fingerprint/env_gce.go:107: client.fingerprint_mgr.env_gce: could not read value for attribute: attribute=machine-type error="Get "http://169.254.169.254/computeMetadata/v1/instance/machine-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:05:07.066Z [DEBUG] fingerprint/env_gce.go:282: client.fingerprint_mgr.env_gce: error querying GCE Metadata URL, skipping 2021-09-13T07:05:07.067Z [DEBUG] fingerprint/env_azure.go:92: client.fingerprint_mgr.env_azure: could not read value for attribute: attribute=compute/azEnvironment error="Get "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2019-06-04&format=text": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:05:07.067Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, bridge, cgroup, cpu, host, network, nomad, signal, storage, env_aws] 2021-09-13T07:05:07.068Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:05:07.068Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:05:07.068Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:05:07.068Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:05:07.080Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:05:07.080Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:05:07.080Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:05:07.081Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:05:07.081Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:05:07.081Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:05:07.083Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:05:07.083Z [DEBUG] servers/manager.go:205: client.server_mgr: new server list: new_servers=[127.0.0.1:9643] old_servers=[] 2021-09-13T07:05:07.111Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:05:07.141Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest72436120/client 2021-09-13T07:05:07.141Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest72436120/allocs 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.142Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:05:07.157Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:05:07.157Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_aws, env_gce, env_azure] 2021-09-13T07:05:07.157Z [INFO] fingerprint/cgroup_linux.go:37: client.fingerprint_mgr.cgroup: cgroups are available 2021-09-13T07:05:07.158Z [DEBUG] fingerprint/cni.go:26: client.fingerprint_mgr: CNI config dir is not set or does not exist, skipping: cni_config_dir=/opt/cni/config 2021-09-13T07:05:07.158Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=cgroup period=15s 2021-09-13T07:05:07.158Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:05:07.158Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:05:07.158Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:05:07.158Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=consul period=15s 2021-09-13T07:05:07.171Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:05:07.171Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:05:07.171Z [DEBUG] fingerprint/network.go:92: client.fingerprint_mgr.network: link speed could not be detected and no speed specified by user, falling back to default speed: mbits=1000 2021-09-13T07:05:07.171Z [DEBUG] fingerprint/network.go:112: client.fingerprint_mgr.network: detected interface IP: interface=ens5 IP=172.31.11.103 2021-09-13T07:05:07.173Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=lo 2021-09-13T07:05:07.173Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/lo/speed 2021-09-13T07:05:07.173Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:05:07.178Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:05:07.178Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[mock_driver exec docker java] undetected:[qemu raw_exec]]" 2021-09-13T07:05:07.178Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:05:07.178Z [INFO] client/client.go:567: client: started client: node_id=3f401452-7ebc-0733-3a76-f5053318f56b 2021-09-13T07:05:07.180Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:05:07.180Z [DEBUG] client/client.go:2166: client: updated allocations: index=1 total=0 pulled=0 filtered=0 2021-09-13T07:05:07.180Z [DEBUG] client/client.go:2244: client: allocation updates: added=0 removed=0 updated=0 ignored=0 2021-09-13T07:05:07.180Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=0 removed=0 updated=0 ignored=0 errors=0 2021-09-13T07:05:07.181Z [TRACE] client/client.go:1854: client: next heartbeat: period=16.995696481s 2021-09-13T07:05:07.181Z [DEBUG] client/client.go:1857: client: state updated: node_status=ready 2021-09-13T07:05:07.182Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:05:07.182Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:05:07.182Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:05:07.185Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=docker0 2021-09-13T07:05:07.185Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/docker0/speed 2021-09-13T07:05:07.185Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 nomad-022 2021-09-13T07:05:07.188Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=canonicalize warnings=[] error= nomad-022 2021-09-13T07:05:07.188Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=connect warnings=[] error= nomad-022 2021-09-13T07:05:07.188Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=expose-check warnings=[] error= nomad-022 2021-09-13T07:05:07.188Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=constraints warnings=[] error= 2021-09-13T07:05:07.188Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=br-8a6b4474e385 nomad-022 2021-09-13T07:05:07.188Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=connect warnings=[] error= 2021-09-13T07:05:07.188Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/br-8a6b4474e385/speed nomad-022 2021-09-13T07:05:07.188Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=expose-check warnings=[] error= 2021-09-13T07:05:07.188Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 nomad-022 2021-09-13T07:05:07.188Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=validate warnings=["Group "web" has warnings: 1 error occurred: * 1 error occurred: * Task "web": task network resources have been deprecated as of Nomad 0.12.0. Please configure networking via group network block. "] error= nomad-022 2021-09-13T07:05:07.188Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=memory_oversubscription warnings=[] error= === CONT TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot wait.go:185: Job "mock-batch-9c66e730-c008-82e6-2442-8384c682652b" registered nomad-023 2021-09-13T07:05:07.330Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=9f47c217-a266-c271-20b8-abbb994167b6 nomad-023 2021-09-13T07:05:07.330Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=9f47c217-a266-c271-20b8-abbb994167b6 job_id=mock-batch-9c66e730-c008-82e6-2442-8384c682652b namespace=default algorithm=binpack nomad-023 2021-09-13T07:05:07.330Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=9f47c217-a266-c271-20b8-abbb994167b6 job_id=mock-batch-9c66e730-c008-82e6-2442-8384c682652b namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-023 2021-09-13T07:05:07.330Z [TRACE] nomad/plan_apply.go:403: nomad: evaluating plan: plan="(eval 9f47c217, job mock-batch-9c66e730-c008-82e6-2442-8384c682652b, NodeAllocations: (node[0bc1f52f] (c47dc207 mock-batch-9c66e730-c008-82e6-2442-8384c682652b.web[0] run)))" nomad-023 2021-09-13T07:05:07.331Z [DEBUG] nomad/worker.go:315: worker: submitted plan for evaluation: eval_id=9f47c217-a266-c271-20b8-abbb994167b6 nomad-023 2021-09-13T07:05:07.331Z [DEBUG] scheduler/util.go:671: worker.batch_sched: setting eval status: eval_id=9f47c217-a266-c271-20b8-abbb994167b6 job_id=mock-batch-9c66e730-c008-82e6-2442-8384c682652b namespace=default status=complete nomad-023 2021-09-13T07:05:07.331Z [DEBUG] nomad/worker.go:377: worker: updated evaluation: eval="" nomad-023 2021-09-13T07:05:07.331Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=9f47c217-a266-c271-20b8-abbb994167b6 2021-09-13T07:05:07.332Z [DEBUG] client/client.go:2166: client: updated allocations: index=10 total=1 pulled=1 filtered=0 2021-09-13T07:05:07.332Z [TRACE] client/client.go:1854: client: next heartbeat: period=13.974771058s 2021-09-13T07:05:07.332Z [DEBUG] client/client.go:2244: client: allocation updates: added=1 removed=0 updated=0 ignored=0 2021-09-13T07:05:07.333Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=1 removed=0 updated=0 ignored=0 errors=0 2021-09-13T07:05:07.333Z [TRACE] allocrunner/alloc_runner_hooks.go:167: client.alloc_runner: running pre-run hooks: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc start="2021-09-13 07:05:07.333409212 +0000 UTC m=+26.812705900" 2021-09-13T07:05:07.333Z [TRACE] allocrunner/alloc_runner_hooks.go:184: client.alloc_runner: running pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=alloc_dir start="2021-09-13 07:05:07.333462725 +0000 UTC m=+26.812759422" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:193: client.alloc_runner: finished pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=alloc_dir end="2021-09-13 07:05:07.419096208 +0000 UTC m=+26.898392912" duration=85.63349ms 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:184: client.alloc_runner: running pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=cgroup start="2021-09-13 07:05:07.41915864 +0000 UTC m=+26.898455328" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:193: client.alloc_runner: finished pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=cgroup end="2021-09-13 07:05:07.419191862 +0000 UTC m=+26.898488542" duration=33.214µs 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:184: client.alloc_runner: running pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=await_previous_allocations start="2021-09-13 07:05:07.419212022 +0000 UTC m=+26.898508694" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:193: client.alloc_runner: finished pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=await_previous_allocations end="2021-09-13 07:05:07.419226725 +0000 UTC m=+26.898523406" duration=14.712µs 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:184: client.alloc_runner: running pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=migrate_disk start="2021-09-13 07:05:07.419243882 +0000 UTC m=+26.898540562" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:193: client.alloc_runner: finished pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=migrate_disk end="2021-09-13 07:05:07.419258504 +0000 UTC m=+26.898555184" duration=14.622µs 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:184: client.alloc_runner: running pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=network start="2021-09-13 07:05:07.41927635 +0000 UTC m=+26.898573022" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:193: client.alloc_runner: finished pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=network end="2021-09-13 07:05:07.419290832 +0000 UTC m=+26.898587512" duration=14.49µs 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:184: client.alloc_runner: running pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=group_services start="2021-09-13 07:05:07.419306569 +0000 UTC m=+26.898603249" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:193: client.alloc_runner: finished pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=group_services end="2021-09-13 07:05:07.419321051 +0000 UTC m=+26.898617731" duration=14.482µs 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:184: client.alloc_runner: running pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=consul_grpc_socket start="2021-09-13 07:05:07.419336066 +0000 UTC m=+26.898632738" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:193: client.alloc_runner: finished pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=consul_grpc_socket end="2021-09-13 07:05:07.419350228 +0000 UTC m=+26.898646900" duration=14.162µs 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:184: client.alloc_runner: running pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=consul_http_socket start="2021-09-13 07:05:07.419365145 +0000 UTC m=+26.898661817" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:193: client.alloc_runner: finished pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=consul_http_socket end="2021-09-13 07:05:07.419380004 +0000 UTC m=+26.898676684" duration=14.867µs 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:184: client.alloc_runner: running pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=csi_hook start="2021-09-13 07:05:07.419394248 +0000 UTC m=+26.898690928" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:193: client.alloc_runner: finished pre-run hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=csi_hook end="2021-09-13 07:05:07.419408558 +0000 UTC m=+26.898705230" duration=14.302µs 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner_hooks.go:170: client.alloc_runner: finished pre-run hooks: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc end="2021-09-13 07:05:07.419422432 +0000 UTC m=+26.898719113" duration=86.013213ms 2021-09-13T07:05:07.419Z [DEBUG] taskrunner/task_runner.go:509: client.alloc_runner.task_runner: lifecycle start condition has been met, proceeding: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc task=web 2021-09-13T07:05:07.419Z [TRACE] taskrunner/task_runner_hooks.go:186: client.alloc_runner.task_runner: running prestart hooks: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc task=web start="2021-09-13 07:05:07.419473944 +0000 UTC m=+26.898770624" 2021-09-13T07:05:07.419Z [TRACE] taskrunner/task_runner_hooks.go:234: client.alloc_runner.task_runner: running prestart hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc task=web name=validate start="2021-09-13 07:05:07.419537541 +0000 UTC m=+26.898834221" 2021-09-13T07:05:07.419Z [TRACE] taskrunner/task_runner_hooks.go:284: client.alloc_runner.task_runner: finished prestart hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc task=web name=validate end="2021-09-13 07:05:07.419566612 +0000 UTC m=+26.898863284" duration=29.063µs 2021-09-13T07:05:07.419Z [TRACE] taskrunner/task_runner_hooks.go:234: client.alloc_runner.task_runner: running prestart hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc task=web name=task_dir start="2021-09-13 07:05:07.419605487 +0000 UTC m=+26.898902159" 2021-09-13T07:05:07.419Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc done=false 2021-09-13T07:05:07.419Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc client_status=pending desired_status= 2021-09-13T07:05:07.430Z [DEBUG] client/client.go:2166: client: updated allocations: index=12 total=1 pulled=0 filtered=1 2021-09-13T07:05:07.430Z [DEBUG] client/client.go:2244: client: allocation updates: added=0 removed=0 updated=0 ignored=1 2021-09-13T07:05:07.431Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=0 removed=0 updated=0 ignored=1 errors=0 2021-09-13T07:05:07.431Z [TRACE] client/client.go:1854: client: next heartbeat: period=14.104891841s === CONT TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot wait.go:223: Error Trace: wait.go:223 wait.go:32 wait.go:18 wait.go:197 alloc_endpoint_test.go:1043 Error: Received unexpected error: alloc not running: id=c47dc207-3fa3-67b1-e349-1ef15331b3bc tg=web status=pending Test: TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot 2021-09-13T07:05:13.014Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:05:13.014Z [TRACE] allocrunner/alloc_runner_hooks.go:330: client.alloc_runner: running alloc pre shutdown hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=group_services start="2021-09-13 07:05:13.014955579 +0000 UTC m=+32.494252251" 2021-09-13T07:05:13.014Z [TRACE] allocrunner/alloc_runner_hooks.go:337: client.alloc_runner: finished alloc pre shutdown hook: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc name=group_services end="2021-09-13 07:05:13.014997983 +0000 UTC m=+32.494294663" duration=42.412µs 2021-09-13T07:05:13.015Z [TRACE] taskrunner/lifecycle.go:72: client.alloc_runner.task_runner: Kill requested: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc task=web event_type=Killing event_reason= 2021-09-13T07:05:13.015Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc done=false 2021-09-13T07:05:13.015Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=c47dc207-3fa3-67b1-e349-1ef15331b3bc client_status=pending desired_status= 2021-09-13T07:05:13.190Z [DEBUG] client/client.go:2166: client: updated allocations: index=12 total=1 pulled=0 filtered=1 2021-09-13T07:05:13.190Z [DEBUG] client/client.go:2244: client: allocation updates: added=0 removed=0 updated=0 ignored=1 2021-09-13T07:05:13.190Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=0 removed=0 updated=0 ignored=1 errors=0 2021-09-13T07:05:13.190Z [TRACE] client/client.go:1854: client: next heartbeat: period=17.986991549s 2021-09-13T07:05:13.195Z [TRACE] client/client.go:1854: client: next heartbeat: period=10.15686296s 2021-09-13T07:05:13.196Z [TRACE] client/client.go:1854: client: next heartbeat: period=11.705243003s 2021-09-13T07:05:13.196Z [TRACE] client/client.go:1854: client: next heartbeat: period=14.892579114s 2021-09-13T07:05:13.196Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:05:13.196Z [TRACE] allocrunner/alloc_runner_hooks.go:330: client.alloc_runner: running alloc pre shutdown hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=group_services start="2021-09-13 07:05:13.196692478 +0000 UTC m=+32.675989158" 2021-09-13T07:05:13.196Z [TRACE] allocrunner/alloc_runner_hooks.go:337: client.alloc_runner: finished alloc pre shutdown hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=group_services end="2021-09-13 07:05:13.196737688 +0000 UTC m=+32.676034368" duration=45.21µs 2021-09-13T07:05:13.196Z [TRACE] taskrunner/lifecycle.go:72: client.alloc_runner.task_runner: Kill requested: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web event_type=Killing event_reason= 2021-09-13T07:05:13.196Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d done=false 2021-09-13T07:05:13.196Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d client_status=running desired_status= 2021-09-13T07:05:13.196Z [TRACE] taskrunner/task_runner_hooks.go:502: client.alloc_runner.task_runner: running pre kill hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web start="2021-09-13 07:05:13.19686709 +0000 UTC m=+32.676163762" 2021-09-13T07:05:13.196Z [TRACE] taskrunner/task_runner_hooks.go:521: client.alloc_runner.task_runner: running prekill hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=consul_services start="2021-09-13 07:05:13.196894027 +0000 UTC m=+32.676190724" 2021-09-13T07:05:13.196Z [TRACE] taskrunner/task_runner_hooks.go:536: client.alloc_runner.task_runner: finished prekill hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=consul_services end="2021-09-13 07:05:13.19691655 +0000 UTC m=+32.676213230" duration=22.506µs 2021-09-13T07:05:13.196Z [TRACE] taskrunner/task_runner_hooks.go:505: client.alloc_runner.task_runner: finished pre kill hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web end="2021-09-13 07:05:13.196938195 +0000 UTC m=+32.676234875" duration=71.113µs 2021-09-13T07:05:13.196Z [DEBUG] mock/driver.go:526: client.driver_mgr.mock_driver: killing task: driver=mock_driver task_name=web kill_after=0s 2021-09-13T07:05:13.196Z [DEBUG] mock/driver.go:532: client.driver_mgr.mock_driver: killing task due to kill_after: driver=mock_driver task_name=web 2021-09-13T07:05:13.196Z [DEBUG] mock/command.go:37: client.driver_mgr.mock_driver: killed; exiting: driver=mock_driver task_name=web 2021-09-13T07:05:13.197Z [TRACE] taskrunner/task_runner_hooks.go:350: client.alloc_runner.task_runner: running exited hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web start="2021-09-13 07:05:13.197031019 +0000 UTC m=+32.676327707" 2021-09-13T07:05:13.197Z [TRACE] taskrunner/task_runner_hooks.go:368: client.alloc_runner.task_runner: running exited hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=stats_hook start="2021-09-13 07:05:13.197055174 +0000 UTC m=+32.676351871" 2021-09-13T07:05:13.197Z [TRACE] taskrunner/task_runner_hooks.go:382: client.alloc_runner.task_runner: finished exited hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=stats_hook end="2021-09-13 07:05:13.197075391 +0000 UTC m=+32.676372064" duration=20.193µs 2021-09-13T07:05:13.197Z [TRACE] taskrunner/task_runner_hooks.go:368: client.alloc_runner.task_runner: running exited hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=consul_services start="2021-09-13 07:05:13.197092245 +0000 UTC m=+32.676388925" 2021-09-13T07:05:13.197Z [TRACE] taskrunner/task_runner_hooks.go:382: client.alloc_runner.task_runner: finished exited hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=consul_services end="2021-09-13 07:05:13.197537878 +0000 UTC m=+32.676834550" duration=445.625µs 2021-09-13T07:05:13.197Z [TRACE] taskrunner/task_runner_hooks.go:353: client.alloc_runner.task_runner: finished exited hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web end="2021-09-13 07:05:13.197630841 +0000 UTC m=+32.676927521" duration=599.814µs 2021-09-13T07:05:13.197Z [TRACE] taskrunner/task_runner_hooks.go:394: client.alloc_runner.task_runner: running stop hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web start="2021-09-13 07:05:13.197655686 +0000 UTC m=+32.676952366" 2021-09-13T07:05:13.197Z [TRACE] taskrunner/task_runner_hooks.go:412: client.alloc_runner.task_runner: running stop hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=logmon start="2021-09-13 07:05:13.197752768 +0000 UTC m=+32.677049440" 2021-09-13T07:05:13.197Z [TRACE] client/client.go:1854: client: next heartbeat: period=17.929702075s 2021-09-13T07:05:13.197Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d done=false 2021-09-13T07:05:13.197Z [INFO] client/gc.go:340: client.gc: marking allocation for GC: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d 2021-09-13T07:05:13.197Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d client_status=complete desired_status= 2021-09-13T07:05:13.197Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d done=false 2021-09-13T07:05:13.198Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d client_status=complete desired_status= 2021-09-13T07:05:13.221Z [DEBUG] client/client.go:2166: client: updated allocations: index=15 total=1 pulled=0 filtered=1 2021-09-13T07:05:13.222Z [TRACE] client/client.go:1854: client: next heartbeat: period=14.157022161s 2021-09-13T07:05:13.442Z [DEBUG] client/client.go:2166: client: updated allocations: index=13 total=1 pulled=0 filtered=1 2021-09-13T07:05:13.442Z [TRACE] client/client.go:1854: client: next heartbeat: period=14.50395804s 2021-09-13T07:05:13.555Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: client.alloc_runner.task_runner.task_hook.logmon.stdio: received EOF, stopping recv loop: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:05:13.558Z [DEBUG] go-plugin@v1.4.0/client.go:635: client.alloc_runner.task_runner.task_hook.logmon: plugin process exited: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web path=/tmp/go-build124451399/b938/client.test pid=22265 2021-09-13T07:05:13.558Z [DEBUG] go-plugin@v1.4.0/client.go:454: client.alloc_runner.task_runner.task_hook.logmon: plugin exited: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web 2021-09-13T07:05:13.558Z [TRACE] taskrunner/task_runner_hooks.go:434: client.alloc_runner.task_runner: finished stop hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=logmon end="2021-09-13 07:05:13.558335835 +0000 UTC m=+33.037632515" duration=360.583075ms 2021-09-13T07:05:13.558Z [TRACE] taskrunner/task_runner_hooks.go:412: client.alloc_runner.task_runner: running stop hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=consul_services start="2021-09-13 07:05:13.558467789 +0000 UTC m=+33.037764485" 2021-09-13T07:05:13.558Z [TRACE] taskrunner/task_runner_hooks.go:434: client.alloc_runner.task_runner: finished stop hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=consul_services end="2021-09-13 07:05:13.558566938 +0000 UTC m=+33.037863635" duration=99.15µs 2021-09-13T07:05:13.558Z [TRACE] taskrunner/task_runner_hooks.go:412: client.alloc_runner.task_runner: running stop hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=script_checks start="2021-09-13 07:05:13.558659442 +0000 UTC m=+33.037956139" 2021-09-13T07:05:13.558Z [TRACE] taskrunner/task_runner_hooks.go:434: client.alloc_runner.task_runner: finished stop hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web name=script_checks end="2021-09-13 07:05:13.55874818 +0000 UTC m=+33.038044868" duration=88.729µs 2021-09-13T07:05:13.558Z [TRACE] taskrunner/task_runner_hooks.go:397: client.alloc_runner.task_runner: finished stop hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web end="2021-09-13 07:05:13.55881254 +0000 UTC m=+33.038109237" duration=361.156871ms 2021-09-13T07:05:13.559Z [DEBUG] taskrunner/task_runner.go:637: client.alloc_runner.task_runner: task run loop exiting: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d task=web 2021-09-13T07:05:13.559Z [TRACE] allocrunner/alloc_runner_hooks.go:247: client.alloc_runner: running post-run hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d start="2021-09-13 07:05:13.559666974 +0000 UTC m=+33.038963654" 2021-09-13T07:05:13.559Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=cgroup start="2021-09-13 07:05:13.559902918 +0000 UTC m=+33.039199615" 2021-09-13T07:05:13.559Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=cgroup end="2021-09-13 07:05:13.55998501 +0000 UTC m=+33.039281690" duration=82.075µs 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=network start="2021-09-13 07:05:13.560051455 +0000 UTC m=+33.039348151" 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=network end="2021-09-13 07:05:13.560116857 +0000 UTC m=+33.039413578" duration=65.427µs 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=group_services start="2021-09-13 07:05:13.560186124 +0000 UTC m=+33.039482821" 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=group_services end="2021-09-13 07:05:13.560249049 +0000 UTC m=+33.039545729" duration=62.908µs 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=consul_grpc_socket start="2021-09-13 07:05:13.560273172 +0000 UTC m=+33.039569868" 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=consul_grpc_socket end="2021-09-13 07:05:13.560345048 +0000 UTC m=+33.039641745" duration=71.877µs 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=consul_http_socket start="2021-09-13 07:05:13.560409277 +0000 UTC m=+33.039705998" 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=consul_http_socket end="2021-09-13 07:05:13.560940267 +0000 UTC m=+33.040237079" duration=531.081µs 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=csi_hook start="2021-09-13 07:05:13.560974925 +0000 UTC m=+33.040271597" 2021-09-13T07:05:13.560Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=csi_hook end="2021-09-13 07:05:13.560990827 +0000 UTC m=+33.040287507" duration=15.91µs 2021-09-13T07:05:13.561Z [TRACE] allocrunner/alloc_runner_hooks.go:250: client.alloc_runner: finished post-run hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d end="2021-09-13 07:05:13.56100777 +0000 UTC m=+33.040304450" duration=1.340796ms 2021-09-13T07:05:13.561Z [TRACE] allocrunner/alloc_runner_hooks.go:285: client.alloc_runner: running destroy hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d start="2021-09-13 07:05:13.561047942 +0000 UTC m=+33.040344622" 2021-09-13T07:05:13.561Z [TRACE] allocrunner/alloc_runner_hooks.go:303: client.alloc_runner: running destroy hook: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=alloc_dir start="2021-09-13 07:05:13.561064943 +0000 UTC m=+33.040361615" 2021-09-13T07:05:13.563Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d done=true 2021-09-13T07:05:13.564Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d client_status=complete desired_status= 2021-09-13T07:05:13.567Z [TRACE] allocrunner/alloc_runner_hooks.go:312: client.alloc_runner: finished destroy hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d name=alloc_dir end="2021-09-13 07:05:13.56729601 +0000 UTC m=+33.046592699" duration=6.231084ms 2021-09-13T07:05:13.567Z [TRACE] allocrunner/alloc_runner_hooks.go:288: client.alloc_runner: finished destroy hooks: alloc_id=d9db4b77-25bd-fdf3-f23d-9dd350168c3d end="2021-09-13 07:05:13.56734346 +0000 UTC m=+33.046640132" duration=6.29551ms 2021-09-13T07:05:13.567Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:05:13.567Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:05:13.567Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:05:13.567Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:05:13.567Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:05:13.567Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:05:13.567Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:05:13.567Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:05:13.573Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:05:13.573Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:05:13.573Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:05:13.574Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver nomad-029 2021-09-13T07:05:13.575Z [INFO] nomad/server.go:622: nomad: shutting down server nomad-029 2021-09-13T07:05:13.575Z [WARN] go-hclog@v0.14.1/stdlog.go:49: nomad: serf: Shutdown without a Leave 2021-09-13T07:05:13.578Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down nomad-029 2021-09-13T07:05:13.578Z [DEBUG] nomad/leader.go:84: nomad: shutting down leader loop === FAIL: client TestFS_Stream (8.13s) === PAUSE TestFS_Stream === CONT TestFS_Stream 2021-09-13T07:06:49.761Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.761Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.761Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.761Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.761Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.761Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.761Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= nomad-051 2021-09-13T07:06:49.764Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/api.go:549: nomad.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:127.0.0.1:9608 Address:127.0.0.1:9608}]" nomad-051 2021-09-13T07:06:49.765Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:152: nomad.raft: entering follower state: follower="Node at 127.0.0.1:9608 [Follower]" leader= nomad-051 2021-09-13T07:06:49.766Z [INFO] go-hclog@v0.14.1/stdlog.go:47: nomad: serf: EventMemberJoin: nomad-051.global 127.0.0.1 nomad-051 2021-09-13T07:06:49.766Z [INFO] nomad/server.go:1465: nomad: starting scheduling worker(s): num_workers=2 schedulers=[batch, system, sysbatch, service, _core] nomad-051 2021-09-13T07:06:49.766Z [INFO] nomad/serf.go:60: nomad: adding server: server="nomad-051.global (Addr: 127.0.0.1:9608) (DC: dc1)" 2021-09-13T07:06:49.765Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.765Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.765Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.765Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.765Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.810Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:49.810Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 nomad-051 2021-09-13T07:06:49.849Z [WARN] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:214: nomad.raft: heartbeat timeout reached, starting election: last-leader= nomad-051 2021-09-13T07:06:49.849Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:250: nomad.raft: entering candidate state: node="Node at 127.0.0.1:9608 [Candidate]" term=2 nomad-051 2021-09-13T07:06:49.851Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:268: nomad.raft: votes: needed=1 nomad-051 2021-09-13T07:06:49.851Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:287: nomad.raft: vote granted: from=127.0.0.1:9608 term=2 tally=1 nomad-051 2021-09-13T07:06:49.851Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:292: nomad.raft: election won: tally=1 nomad-051 2021-09-13T07:06:49.851Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:363: nomad.raft: entering leader state: leader="Node at 127.0.0.1:9608 [Leader]" nomad-051 2021-09-13T07:06:49.851Z [INFO] nomad/leader.go:75: nomad: cluster leadership acquired nomad-051 2021-09-13T07:06:49.861Z [TRACE] nomad/fsm.go:337: nomad.fsm: ClusterSetMetadata: cluster_id=d0ed3f07-7339-9ae0-b57e-6720f6c48195 create_time=1631516809861587598 nomad-051 2021-09-13T07:06:49.861Z [INFO] nomad/leader.go:1628: nomad.core: established cluster id: cluster_id=d0ed3f07-7339-9ae0-b57e-6720f6c48195 create_time=1631516809861587598 nomad-051 2021-09-13T07:06:49.861Z [TRACE] drainer/watch_jobs.go:145: nomad.drain.job_watcher: getting job allocs at index: index=1 2021-09-13T07:06:49.945Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:49.946Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:49.991Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest93535034/client 2021-09-13T07:06:49.992Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest93535034/allocs 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:49.994Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.014Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:06:50.015Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_azure, env_aws, env_gce] 2021-09-13T07:06:50.015Z [INFO] fingerprint/cgroup_linux.go:37: client.fingerprint_mgr.cgroup: cgroups are available 2021-09-13T07:06:50.015Z [DEBUG] fingerprint/cni.go:26: client.fingerprint_mgr: CNI config dir is not set or does not exist, skipping: cni_config_dir=/opt/cni/config 2021-09-13T07:06:50.021Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=cgroup period=15s 2021-09-13T07:06:50.022Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:06:50.023Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:06:50.023Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:06:50.025Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=consul period=15s 2021-09-13T07:06:50.050Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:06:50.050Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:06:50.050Z [DEBUG] fingerprint/network.go:92: client.fingerprint_mgr.network: link speed could not be detected and no speed specified by user, falling back to default speed: mbits=1000 2021-09-13T07:06:50.050Z [DEBUG] fingerprint/network.go:112: client.fingerprint_mgr.network: detected interface IP: interface=ens5 IP=172.31.11.103 2021-09-13T07:06:50.055Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=lo 2021-09-13T07:06:50.055Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/lo/speed 2021-09-13T07:06:50.055Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:50.078Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:06:50.078Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:06:50.078Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:50.085Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=docker0 2021-09-13T07:06:50.085Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/docker0/speed 2021-09-13T07:06:50.085Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:50.089Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=br-8a6b4474e385 2021-09-13T07:06:50.089Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/br-8a6b4474e385/speed 2021-09-13T07:06:50.090Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:50.094Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=vault period=15s 2021-09-13T07:06:50.095Z [DEBUG] fingerprint/env_gce.go:110: client.fingerprint_mgr.env_gce: could not read value for attribute: attribute=machine-type resp_code=404 2021-09-13T07:06:50.097Z [DEBUG] fingerprint/env_azure.go:92: client.fingerprint_mgr.env_azure: could not read value for attribute: attribute=compute/azEnvironment error="Get "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2019-06-04&format=text": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:06:50.100Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, bridge, cgroup, cpu, host, network, nomad, signal, storage] 2021-09-13T07:06:50.100Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:06:50.100Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:06:50.100Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:06:50.106Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:06:50.106Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:06:50.106Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:06:50.107Z [DEBUG] servers/manager.go:205: client.server_mgr: new server list: new_servers=[127.0.0.1:9608] old_servers=[] 2021-09-13T07:06:50.107Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:06:50.107Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:06:50.107Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:06:50.107Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:06:50.107Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:06:50.139Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:06:50.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:50.303Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:50.413Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:06:50.413Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[mock_driver exec docker java] undetected:[raw_exec qemu]]" 2021-09-13T07:06:50.413Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:06:50.413Z [INFO] client/client.go:567: client: started client: node_id=b6ca2917-d36a-b9da-7dcc-d1b3ee7361bd 2021-09-13T07:06:50.415Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:06:50.421Z [DEBUG] client/client.go:2166: client: updated allocations: index=1 total=0 pulled=0 filtered=0 2021-09-13T07:06:50.422Z [DEBUG] client/client.go:2244: client: allocation updates: added=0 removed=0 updated=0 ignored=0 2021-09-13T07:06:50.422Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=0 removed=0 updated=0 ignored=0 errors=0 2021-09-13T07:06:50.423Z [TRACE] client/client.go:1854: client: next heartbeat: period=13.134692144s 2021-09-13T07:06:50.423Z [DEBUG] client/client.go:1857: client: state updated: node_status=ready nomad-051 2021-09-13T07:06:50.423Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=canonicalize warnings=[] error= nomad-051 2021-09-13T07:06:50.423Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=connect warnings=[] error= nomad-051 2021-09-13T07:06:50.423Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=expose-check warnings=[] error= nomad-051 2021-09-13T07:06:50.424Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=constraints warnings=[] error= nomad-051 2021-09-13T07:06:50.424Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=connect warnings=[] error= nomad-051 2021-09-13T07:06:50.424Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=expose-check warnings=[] error= nomad-051 2021-09-13T07:06:50.424Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=validate warnings=["Group "web" has warnings: 1 error occurred: * 1 error occurred: * Task "web": task network resources have been deprecated as of Nomad 0.12.0. Please configure networking via group network block. "] error= nomad-051 2021-09-13T07:06:50.424Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=memory_oversubscription warnings=[] error= wait.go:185: Job "mock-batch-78a75873-e2ff-2576-8499-bdeef392e0db" registered nomad-051 2021-09-13T07:06:50.425Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=54b90019-37c9-dd0a-bc52-f967bb3f33e1 nomad-051 2021-09-13T07:06:50.425Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=54b90019-37c9-dd0a-bc52-f967bb3f33e1 job_id=mock-batch-78a75873-e2ff-2576-8499-bdeef392e0db namespace=default algorithm=binpack nomad-051 2021-09-13T07:06:50.425Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=54b90019-37c9-dd0a-bc52-f967bb3f33e1 job_id=mock-batch-78a75873-e2ff-2576-8499-bdeef392e0db namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-051 2021-09-13T07:06:50.425Z [DEBUG] nomad/worker.go:418: worker: created evaluation: eval="" nomad-051 2021-09-13T07:06:50.425Z [DEBUG] scheduler/generic_sched.go:275: worker.batch_sched: failed to place all allocations, blocked eval created: eval_id=54b90019-37c9-dd0a-bc52-f967bb3f33e1 job_id=mock-batch-78a75873-e2ff-2576-8499-bdeef392e0db namespace=default blocked_eval_id=930dac6d-cfc4-6a8c-c495-f5a071deeb2b nomad-051 2021-09-13T07:06:50.425Z [DEBUG] scheduler/util.go:671: worker.batch_sched: setting eval status: eval_id=54b90019-37c9-dd0a-bc52-f967bb3f33e1 job_id=mock-batch-78a75873-e2ff-2576-8499-bdeef392e0db namespace=default status=complete nomad-051 2021-09-13T07:06:50.426Z [DEBUG] nomad/worker.go:377: worker: updated evaluation: eval="" nomad-051 2021-09-13T07:06:50.426Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=54b90019-37c9-dd0a-bc52-f967bb3f33e1 2021-09-13T07:06:50.443Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:50.443Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:50.504Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:06:50.505Z [TRACE] allocrunner/alloc_runner_hooks.go:330: client.alloc_runner: running alloc pre shutdown hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=group_services start="2021-09-13 07:06:50.505045798 +0000 UTC m=+129.984342479" 2021-09-13T07:06:50.505Z [TRACE] allocrunner/alloc_runner_hooks.go:337: client.alloc_runner: finished alloc pre shutdown hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=group_services end="2021-09-13 07:06:50.505088514 +0000 UTC m=+129.984385194" duration=42.715µs 2021-09-13T07:06:50.505Z [TRACE] taskrunner/lifecycle.go:72: client.alloc_runner.task_runner: Kill requested: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web event_type=Killing event_reason= 2021-09-13T07:06:50.505Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b done=false 2021-09-13T07:06:50.505Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b client_status=running desired_status= 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:502: client.alloc_runner.task_runner: running pre kill hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web start="2021-09-13 07:06:50.505246453 +0000 UTC m=+129.984543133" 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:521: client.alloc_runner.task_runner: running prekill hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=consul_services start="2021-09-13 07:06:50.505276147 +0000 UTC m=+129.984572828" 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:536: client.alloc_runner.task_runner: finished prekill hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=consul_services end="2021-09-13 07:06:50.505291655 +0000 UTC m=+129.984588335" duration=15.507µs 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:505: client.alloc_runner.task_runner: finished pre kill hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web end="2021-09-13 07:06:50.505307786 +0000 UTC m=+129.984604458" duration=61.325µs 2021-09-13T07:06:50.505Z [DEBUG] mock/driver.go:526: client.driver_mgr.mock_driver: killing task: driver=mock_driver task_name=web kill_after=0s 2021-09-13T07:06:50.505Z [DEBUG] mock/driver.go:532: client.driver_mgr.mock_driver: killing task due to kill_after: driver=mock_driver task_name=web 2021-09-13T07:06:50.505Z [DEBUG] mock/command.go:37: client.driver_mgr.mock_driver: killed; exiting: driver=mock_driver task_name=web 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:350: client.alloc_runner.task_runner: running exited hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web start="2021-09-13 07:06:50.505393553 +0000 UTC m=+129.984690234" 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:368: client.alloc_runner.task_runner: running exited hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=stats_hook start="2021-09-13 07:06:50.505410513 +0000 UTC m=+129.984707185" 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:382: client.alloc_runner.task_runner: finished exited hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=stats_hook end="2021-09-13 07:06:50.505426029 +0000 UTC m=+129.984722709" duration=15.524µs 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:368: client.alloc_runner.task_runner: running exited hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=consul_services start="2021-09-13 07:06:50.505441151 +0000 UTC m=+129.984737831" 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:382: client.alloc_runner.task_runner: finished exited hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=consul_services end="2021-09-13 07:06:50.505455223 +0000 UTC m=+129.984751895" duration=14.064µs 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:353: client.alloc_runner.task_runner: finished exited hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web end="2021-09-13 07:06:50.505470993 +0000 UTC m=+129.984767673" duration=77.439µs 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:394: client.alloc_runner.task_runner: running stop hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web start="2021-09-13 07:06:50.505501171 +0000 UTC m=+129.984797851" 2021-09-13T07:06:50.505Z [TRACE] taskrunner/task_runner_hooks.go:412: client.alloc_runner.task_runner: running stop hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=logmon start="2021-09-13 07:06:50.505516621 +0000 UTC m=+129.984813301" 2021-09-13T07:06:50.505Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b done=false 2021-09-13T07:06:50.505Z [INFO] client/gc.go:340: client.gc: marking allocation for GC: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b 2021-09-13T07:06:50.505Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b client_status=complete desired_status= 2021-09-13T07:06:50.505Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b done=false 2021-09-13T07:06:50.505Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b client_status=complete desired_status= 2021-09-13T07:06:50.519Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: client.alloc_runner.task_runner.task_hook.logmon.stdio: received EOF, stopping recv loop: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:06:50.519Z [DEBUG] go-plugin@v1.4.0/client.go:635: client.alloc_runner.task_runner.task_hook.logmon: plugin process exited: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web path=/tmp/go-build124451399/b938/client.test pid=25227 2021-09-13T07:06:50.519Z [DEBUG] go-plugin@v1.4.0/client.go:454: client.alloc_runner.task_runner.task_hook.logmon: plugin exited: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web 2021-09-13T07:06:50.519Z [TRACE] taskrunner/task_runner_hooks.go:434: client.alloc_runner.task_runner: finished stop hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=logmon end="2021-09-13 07:06:50.519450681 +0000 UTC m=+129.998747378" duration=13.934077ms 2021-09-13T07:06:50.519Z [TRACE] taskrunner/task_runner_hooks.go:412: client.alloc_runner.task_runner: running stop hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=consul_services start="2021-09-13 07:06:50.519544924 +0000 UTC m=+129.998841621" 2021-09-13T07:06:50.519Z [TRACE] taskrunner/task_runner_hooks.go:434: client.alloc_runner.task_runner: finished stop hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=consul_services end="2021-09-13 07:06:50.519623504 +0000 UTC m=+129.998920201" duration=78.58µs 2021-09-13T07:06:50.519Z [TRACE] taskrunner/task_runner_hooks.go:412: client.alloc_runner.task_runner: running stop hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=script_checks start="2021-09-13 07:06:50.519677198 +0000 UTC m=+129.998973895" 2021-09-13T07:06:50.519Z [TRACE] taskrunner/task_runner_hooks.go:434: client.alloc_runner.task_runner: finished stop hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web name=script_checks end="2021-09-13 07:06:50.519774026 +0000 UTC m=+129.999070706" duration=96.811µs 2021-09-13T07:06:50.519Z [TRACE] taskrunner/task_runner_hooks.go:397: client.alloc_runner.task_runner: finished stop hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web end="2021-09-13 07:06:50.519846862 +0000 UTC m=+129.999143551" duration=14.3457ms 2021-09-13T07:06:50.519Z [DEBUG] taskrunner/task_runner.go:637: client.alloc_runner.task_runner: task run loop exiting: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b task=web 2021-09-13T07:06:50.519Z [TRACE] allocrunner/alloc_runner_hooks.go:247: client.alloc_runner: running post-run hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b start="2021-09-13 07:06:50.519980867 +0000 UTC m=+129.999277547" 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=cgroup start="2021-09-13 07:06:50.520087156 +0000 UTC m=+129.999383836" 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=cgroup end="2021-09-13 07:06:50.520136763 +0000 UTC m=+129.999433468" duration=49.632µs 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=network start="2021-09-13 07:06:50.520210798 +0000 UTC m=+129.999507494" 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=network end="2021-09-13 07:06:50.520272861 +0000 UTC m=+129.999569541" duration=62.047µs 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=group_services start="2021-09-13 07:06:50.520347363 +0000 UTC m=+129.999644059" 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=group_services end="2021-09-13 07:06:50.520434139 +0000 UTC m=+129.999730828" duration=86.769µs 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=consul_grpc_socket start="2021-09-13 07:06:50.52048465 +0000 UTC m=+129.999781346" 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=consul_grpc_socket end="2021-09-13 07:06:50.520550036 +0000 UTC m=+129.999846732" duration=65.386µs 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=consul_http_socket start="2021-09-13 07:06:50.520622413 +0000 UTC m=+129.999919109" 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=consul_http_socket end="2021-09-13 07:06:50.520684894 +0000 UTC m=+129.999981591" duration=62.482µs 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:264: client.alloc_runner: running post-run hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=csi_hook start="2021-09-13 07:06:50.520756467 +0000 UTC m=+130.000053163" 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:273: client.alloc_runner: finished post-run hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=csi_hook end="2021-09-13 07:06:50.520849348 +0000 UTC m=+130.000146028" duration=92.865µs 2021-09-13T07:06:50.520Z [TRACE] allocrunner/alloc_runner_hooks.go:250: client.alloc_runner: finished post-run hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b end="2021-09-13 07:06:50.52090401 +0000 UTC m=+130.000200715" duration=923.168µs 2021-09-13T07:06:50.521Z [TRACE] allocrunner/alloc_runner_hooks.go:285: client.alloc_runner: running destroy hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b start="2021-09-13 07:06:50.52101454 +0000 UTC m=+130.000311212" 2021-09-13T07:06:50.521Z [TRACE] allocrunner/alloc_runner_hooks.go:303: client.alloc_runner: running destroy hook: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=alloc_dir start="2021-09-13 07:06:50.521086179 +0000 UTC m=+130.000382867" 2021-09-13T07:06:50.521Z [TRACE] allocrunner/alloc_runner.go:493: client.alloc_runner: handling task state update: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b done=true 2021-09-13T07:06:50.521Z [TRACE] structs/broadcaster.go:61: client.alloc_runner: sending updated alloc: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b client_status=complete desired_status= 2021-09-13T07:06:50.531Z [TRACE] allocrunner/alloc_runner_hooks.go:312: client.alloc_runner: finished destroy hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b name=alloc_dir end="2021-09-13 07:06:50.531899363 +0000 UTC m=+130.011196059" duration=10.813192ms 2021-09-13T07:06:50.532Z [TRACE] allocrunner/alloc_runner_hooks.go:288: client.alloc_runner: finished destroy hooks: alloc_id=445d572b-e745-f52b-1bd2-47a6164b102b end="2021-09-13 07:06:50.532071373 +0000 UTC m=+130.011368070" duration=11.056858ms 2021-09-13T07:06:50.532Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:06:50.532Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:06:50.532Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:06:50.532Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:06:50.532Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:06:50.532Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:06:50.533Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:06:50.533Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:06:50.533Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:06:50.533Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:06:50.533Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:06:50.533Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec nomad-050 2021-09-13T07:06:50.534Z [INFO] nomad/server.go:622: nomad: shutting down server nomad-050 2021-09-13T07:06:50.534Z [WARN] go-hclog@v0.14.1/stdlog.go:49: nomad: serf: Shutdown without a Leave 2021-09-13T07:06:50.534Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down nomad-050 2021-09-13T07:06:50.534Z [DEBUG] nomad/leader.go:84: nomad: shutting down leader loop nomad-050 2021-09-13T07:06:50.536Z [INFO] nomad/leader.go:88: nomad: cluster leadership lost === CONT TestFS_Stream wait.go:223: Error Trace: wait.go:223 wait.go:32 wait.go:18 wait.go:197 wait.go:231 fs_endpoint_test.go:549 Error: Received unexpected error: 0 allocations; evals: []*structs.Evaluation{ &structs.Evaluation{ _struct: false, ID: "930dac6d-cfc4-6a8c-c495-f5a071deeb2b", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "queued-allocs", JobID: "mock-batch-78a75873-e2ff-2576-8499-bdeef392e0db", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "blocked", StatusDescription: "created to place remaining allocations", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "54b90019-37c9-dd0a-bc52-f967bb3f33e1", BlockedEval: "", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 101571, CoalescedFailures: 0, }, }, ClassEligibility: {"v1:9704016006244050741":true}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {}, LeaderACL: "", SnapshotIndex: 0x9, CreateIndex: 0xa, ModifyIndex: 0xa, CreateTime: 1631516810425583960, ModifyTime: 1631516810425583960, }, &structs.Evaluation{ _struct: false, ID: "54b90019-37c9-dd0a-bc52-f967bb3f33e1", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "job-register", JobID: "mock-batch-78a75873-e2ff-2576-8499-bdeef392e0db", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "complete", StatusDescription: "", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "", BlockedEval: "930dac6d-cfc4-6a8c-c495-f5a071deeb2b", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 101571, CoalescedFailures: 0, }, }, ClassEligibility: {}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {"web":1}, LeaderACL: "", SnapshotIndex: 0x9, CreateIndex: 0x9, ModifyIndex: 0xb, CreateTime: 1631516810424608444, ModifyTime: 1631516810425964248, }, } Test: TestFS_Stream 2021-09-13T07:06:57.874Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:06:57.874Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:06:57.874Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:06:57.874Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:06:57.874Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:06:57.874Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:06:57.874Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:06:57.874Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:06:57.875Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:06:57.875Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:06:57.875Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:06:57.875Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:06:57.875Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:06:57.875Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down nomad-051 2021-09-13T07:06:57.886Z [INFO] nomad/server.go:622: nomad: shutting down server nomad-051 2021-09-13T07:06:57.886Z [WARN] go-hclog@v0.14.1/stdlog.go:49: nomad: serf: Shutdown without a Leave nomad-051 2021-09-13T07:06:57.886Z [DEBUG] nomad/leader.go:84: nomad: shutting down leader loop === FAIL: client TestFS_Stream_ACL (8.49s) === PAUSE TestFS_Stream_ACL === CONT TestFS_Stream_ACL nomad-052 2021-09-13T07:06:50.547Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/api.go:549: nomad.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:127.0.0.1:9610 Address:127.0.0.1:9610}]" nomad-052 2021-09-13T07:06:50.548Z [INFO] go-hclog@v0.14.1/stdlog.go:47: nomad: serf: EventMemberJoin: nomad-052.global 127.0.0.1 nomad-052 2021-09-13T07:06:50.548Z [INFO] nomad/server.go:1465: nomad: starting scheduling worker(s): num_workers=2 schedulers=[batch, system, sysbatch, service, _core] 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.547Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= nomad-052 2021-09-13T07:06:50.550Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:152: nomad.raft: entering follower state: follower="Node at 127.0.0.1:9610 [Follower]" leader= nomad-052 2021-09-13T07:06:50.550Z [INFO] nomad/serf.go:60: nomad: adding server: server="nomad-052.global (Addr: 127.0.0.1:9610) (DC: dc1)" nomad-052 2021-09-13T07:06:50.625Z [WARN] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:214: nomad.raft: heartbeat timeout reached, starting election: last-leader= nomad-052 2021-09-13T07:06:50.625Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:250: nomad.raft: entering candidate state: node="Node at 127.0.0.1:9610 [Candidate]" term=2 nomad-052 2021-09-13T07:06:50.625Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:268: nomad.raft: votes: needed=1 nomad-052 2021-09-13T07:06:50.625Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:287: nomad.raft: vote granted: from=127.0.0.1:9610 term=2 tally=1 nomad-052 2021-09-13T07:06:50.625Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:292: nomad.raft: election won: tally=1 nomad-052 2021-09-13T07:06:50.625Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:363: nomad.raft: entering leader state: leader="Node at 127.0.0.1:9610 [Leader]" nomad-052 2021-09-13T07:06:50.625Z [INFO] nomad/leader.go:75: nomad: cluster leadership acquired nomad-052 2021-09-13T07:06:50.625Z [TRACE] nomad/fsm.go:337: nomad.fsm: ClusterSetMetadata: cluster_id=6fddc354-0ba9-29e0-060e-5d87d8730856 create_time=1631516810625860967 nomad-052 2021-09-13T07:06:50.625Z [INFO] nomad/leader.go:1628: nomad.core: established cluster id: cluster_id=6fddc354-0ba9-29e0-060e-5d87d8730856 create_time=1631516810625860967 nomad-052 2021-09-13T07:06:50.626Z [TRACE] drainer/watch_jobs.go:145: nomad.drain.job_watcher: getting job allocs at index: index=1 2021-09-13T07:06:50.815Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:50.815Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:50.881Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.894Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.900Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest3800419063/client 2021-09-13T07:06:50.905Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest3800419063/allocs 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.902Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:50.925Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:06:50.926Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_aws, env_gce, env_azure] 2021-09-13T07:06:50.926Z [INFO] fingerprint/cgroup_linux.go:37: client.fingerprint_mgr.cgroup: cgroups are available 2021-09-13T07:06:50.926Z [DEBUG] fingerprint/cni.go:26: client.fingerprint_mgr: CNI config dir is not set or does not exist, skipping: cni_config_dir=/opt/cni/config 2021-09-13T07:06:50.926Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:06:50.927Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:06:50.927Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:06:50.928Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=consul period=15s 2021-09-13T07:06:50.929Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:06:50.930Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:06:50.930Z [DEBUG] fingerprint/network.go:92: client.fingerprint_mgr.network: link speed could not be detected and no speed specified by user, falling back to default speed: mbits=1000 2021-09-13T07:06:50.930Z [DEBUG] fingerprint/network.go:112: client.fingerprint_mgr.network: detected interface IP: interface=ens5 IP=172.31.11.103 2021-09-13T07:06:50.931Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=lo 2021-09-13T07:06:50.931Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/lo/speed 2021-09-13T07:06:50.931Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:50.940Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:50.941Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=cgroup period=15s 2021-09-13T07:06:50.942Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:50.997Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:06:51.105Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:06:51.105Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:51.168Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=docker0 2021-09-13T07:06:51.169Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/docker0/speed 2021-09-13T07:06:51.169Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:51.188Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=br-8a6b4474e385 2021-09-13T07:06:51.189Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/br-8a6b4474e385/speed 2021-09-13T07:06:51.189Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:51.196Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=vault period=15s 2021-09-13T07:06:51.216Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=instance-id 2021-09-13T07:06:51.221Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=instance-type 2021-09-13T07:06:51.223Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=placement/availability-zone 2021-09-13T07:06:51.228Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=public-hostname 2021-09-13T07:06:51.233Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=public-ipv4 2021-09-13T07:06:51.234Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=mac 2021-09-13T07:06:51.237Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=ami-id 2021-09-13T07:06:51.238Z [ERROR] fingerprint/env_aws.go:252: client.fingerprint_mgr.env_aws: error reading instance-type: error="RequestError: send request failed caused by: Get "http://169.254.169.254/latest/meta-data/instance-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:06:51.239Z [WARN] fingerprint/env_aws.go:220: client.fingerprint_mgr.env_aws: failed to read EC2 metadata instance-type: error="RequestError: send request failed caused by: Get "http://169.254.169.254/latest/meta-data/instance-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:06:51.239Z [WARN] fingerprint/env_aws.go:194: client.fingerprint_mgr.env_aws: failed to find the cpu specification for this instance type 2021-09-13T07:06:51.241Z [DEBUG] fingerprint/env_gce.go:107: client.fingerprint_mgr.env_gce: could not read value for attribute: attribute=machine-type error="Get "http://169.254.169.254/computeMetadata/v1/instance/machine-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:06:51.241Z [DEBUG] fingerprint/env_gce.go:282: client.fingerprint_mgr.env_gce: error querying GCE Metadata URL, skipping 2021-09-13T07:06:51.246Z [DEBUG] fingerprint/env_azure.go:92: client.fingerprint_mgr.env_azure: could not read value for attribute: attribute=compute/azEnvironment error="Get "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2019-06-04&format=text": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:06:51.252Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, bridge, cgroup, cpu, host, network, nomad, signal, storage, env_aws] 2021-09-13T07:06:51.252Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:06:51.252Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:06:51.252Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:06:51.253Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:06:51.253Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:06:51.254Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:06:51.254Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:06:51.254Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:06:51.255Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:06:51.255Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:06:51.255Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:06:51.271Z [DEBUG] servers/manager.go:205: client.server_mgr: new server list: new_servers=[127.0.0.1:9610] old_servers=[] 2021-09-13T07:06:51.298Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:06:51.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:51.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:51.422Z [DEBUG] client/client.go:2217: client: state changed, updating node and re-registering nomad-051 2021-09-13T07:06:51.424Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=930dac6d-cfc4-6a8c-c495-f5a071deeb2b nomad-051 2021-09-13T07:06:51.424Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=930dac6d-cfc4-6a8c-c495-f5a071deeb2b job_id=mock-batch-78a75873-e2ff-2576-8499-bdeef392e0db namespace=default algorithm=binpack nomad-051 2021-09-13T07:06:51.424Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=930dac6d-cfc4-6a8c-c495-f5a071deeb2b job_id=mock-batch-78a75873-e2ff-2576-8499-bdeef392e0db namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-051 2021-09-13T07:06:51.424Z [DEBUG] nomad/worker.go:480: worker: reblocked evaluation: eval="" nomad-051 2021-09-13T07:06:51.424Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=930dac6d-cfc4-6a8c-c495-f5a071deeb2b 2021-09-13T07:06:51.424Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:06:51.425Z [TRACE] client/client.go:1854: client: next heartbeat: period=15.396851305s 2021-09-13T07:06:51.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:51.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:51.538Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:06:51.538Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[exec mock_driver docker java] undetected:[raw_exec qemu]]" 2021-09-13T07:06:51.538Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:06:51.538Z [INFO] client/client.go:567: client: started client: node_id=8c03c202-230e-f18d-2e03-fc96e84e7e8a nomad-052 2021-09-13T07:06:51.555Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=canonicalize warnings=[] error= nomad-052 2021-09-13T07:06:51.556Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=connect warnings=[] error= nomad-052 2021-09-13T07:06:51.556Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=expose-check warnings=[] error= nomad-052 2021-09-13T07:06:51.556Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=constraints warnings=[] error= nomad-052 2021-09-13T07:06:51.556Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=connect warnings=[] error= nomad-052 2021-09-13T07:06:51.556Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=expose-check warnings=[] error= nomad-052 2021-09-13T07:06:51.556Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=validate warnings=["Group "web" has warnings: 1 error occurred: * 1 error occurred: * Task "web": task network resources have been deprecated as of Nomad 0.12.0. Please configure networking via group network block. "] error= nomad-052 2021-09-13T07:06:51.556Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=memory_oversubscription warnings=[] error= wait.go:185: Job "mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23" registered nomad-052 2021-09-13T07:06:51.556Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=51608604-616f-31dc-6b13-8a131c28105d nomad-052 2021-09-13T07:06:51.556Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=51608604-616f-31dc-6b13-8a131c28105d job_id=mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23 namespace=default algorithm=binpack nomad-052 2021-09-13T07:06:51.556Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=51608604-616f-31dc-6b13-8a131c28105d job_id=mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-052 2021-09-13T07:06:51.557Z [DEBUG] nomad/worker.go:418: worker: created evaluation: eval="" nomad-052 2021-09-13T07:06:51.557Z [DEBUG] scheduler/generic_sched.go:275: worker.batch_sched: failed to place all allocations, blocked eval created: eval_id=51608604-616f-31dc-6b13-8a131c28105d job_id=mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23 namespace=default blocked_eval_id=3c5feb18-dc80-c11f-4359-64dabbb5b9d4 nomad-052 2021-09-13T07:06:51.557Z [DEBUG] scheduler/util.go:671: worker.batch_sched: setting eval status: eval_id=51608604-616f-31dc-6b13-8a131c28105d job_id=mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23 namespace=default status=complete nomad-052 2021-09-13T07:06:51.561Z [DEBUG] nomad/worker.go:377: worker: updated evaluation: eval="" nomad-052 2021-09-13T07:06:51.561Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=51608604-616f-31dc-6b13-8a131c28105d 2021-09-13T07:06:51.562Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:06:51.565Z [DEBUG] client/client.go:2166: client: updated allocations: index=1 total=0 pulled=0 filtered=0 2021-09-13T07:06:51.565Z [DEBUG] client/client.go:2244: client: allocation updates: added=0 removed=0 updated=0 ignored=0 2021-09-13T07:06:51.565Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=0 removed=0 updated=0 ignored=0 errors=0 nomad-052 2021-09-13T07:06:51.566Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=3c5feb18-dc80-c11f-4359-64dabbb5b9d4 nomad-052 2021-09-13T07:06:51.566Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=3c5feb18-dc80-c11f-4359-64dabbb5b9d4 job_id=mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23 namespace=default algorithm=binpack nomad-052 2021-09-13T07:06:51.566Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=3c5feb18-dc80-c11f-4359-64dabbb5b9d4 job_id=mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-052 2021-09-13T07:06:51.567Z [DEBUG] nomad/worker.go:480: worker: reblocked evaluation: eval="" nomad-052 2021-09-13T07:06:51.567Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=3c5feb18-dc80-c11f-4359-64dabbb5b9d4 2021-09-13T07:06:51.567Z [TRACE] client/client.go:1854: client: next heartbeat: period=10.742814526s 2021-09-13T07:06:51.567Z [DEBUG] client/client.go:1857: client: state updated: node_status=ready 2021-09-13T07:06:51.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:51.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:51.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:51.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:52.307Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:52.308Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:52.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:52.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:52.566Z [DEBUG] client/client.go:2217: client: state changed, updating node and re-registering nomad-052 2021-09-13T07:06:52.636Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=3c5feb18-dc80-c11f-4359-64dabbb5b9d4 nomad-052 2021-09-13T07:06:52.636Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=3c5feb18-dc80-c11f-4359-64dabbb5b9d4 job_id=mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23 namespace=default algorithm=binpack nomad-052 2021-09-13T07:06:52.636Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=3c5feb18-dc80-c11f-4359-64dabbb5b9d4 job_id=mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-052 2021-09-13T07:06:52.637Z [DEBUG] nomad/worker.go:480: worker: reblocked evaluation: eval="" nomad-052 2021-09-13T07:06:52.637Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=3c5feb18-dc80-c11f-4359-64dabbb5b9d4 2021-09-13T07:06:52.640Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:06:52.702Z [TRACE] client/client.go:1854: client: next heartbeat: period=18.927017014s 2021-09-13T07:06:52.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:52.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:52.939Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:52.939Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:53.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:53.305Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:53.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:53.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:53.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:53.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:53.941Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:53.942Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:54.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:54.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:54.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:54.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:54.805Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:54.812Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:54.949Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:54.950Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:55.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:55.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:55.443Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:55.446Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:55.806Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:55.804Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:55.943Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:55.944Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:56.305Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:56.307Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:56.441Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:56.464Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:56.813Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:56.816Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:56.942Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:56.942Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:57.321Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:57.321Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:57.449Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:57.450Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:57.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:57.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 === CONT TestFS_Stream_ACL wait.go:223: Error Trace: wait.go:223 wait.go:32 wait.go:18 wait.go:197 fs_endpoint_test.go:427 Error: Received unexpected error: 0 allocations; evals: []*structs.Evaluation{ &structs.Evaluation{ _struct: false, ID: "3c5feb18-dc80-c11f-4359-64dabbb5b9d4", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "queued-allocs", JobID: "mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23", JobModifyIndex: 0x7, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "blocked", StatusDescription: "created to place remaining allocations", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "51608604-616f-31dc-6b13-8a131c28105d", BlockedEval: "", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 0, NodesFiltered: 0, NodesAvailable: {"dc1":0}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 0, ClassExhausted: {}, DimensionExhausted: {}, QuotaExhausted: nil, ResourcesExhausted: {}, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 5333, CoalescedFailures: 0, }, }, ClassEligibility: {}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {}, LeaderACL: "", SnapshotIndex: 0x3f2, CreateIndex: 0x8, ModifyIndex: 0x8, CreateTime: 1631516811556980637, ModifyTime: 1631516811556980637, }, &structs.Evaluation{ _struct: false, ID: "51608604-616f-31dc-6b13-8a131c28105d", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "job-register", JobID: "mock-batch-2de34cd8-48d4-7d45-5489-c9d06f2bff23", JobModifyIndex: 0x7, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "complete", StatusDescription: "", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "", BlockedEval: "3c5feb18-dc80-c11f-4359-64dabbb5b9d4", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 0, NodesFiltered: 0, NodesAvailable: {"dc1":0}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 0, ClassExhausted: {}, DimensionExhausted: {}, QuotaExhausted: nil, ResourcesExhausted: {}, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 5333, CoalescedFailures: 0, }, }, ClassEligibility: {}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {"web":1}, LeaderACL: "", SnapshotIndex: 0x3f2, CreateIndex: 0x7, ModifyIndex: 0x9, CreateTime: 1631516811556255737, ModifyTime: 1631516811560669443, }, } Test: TestFS_Stream_ACL 2021-09-13T07:06:59.035Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:06:59.035Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:06:59.035Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:06:59.035Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:06:59.035Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:06:59.035Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:06:59.035Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:06:59.035Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:06:59.035Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:06:59.035Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:06:59.035Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:06:59.035Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:06:59.035Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu nomad-052 2021-09-13T07:06:59.036Z [INFO] nomad/server.go:622: nomad: shutting down server nomad-052 2021-09-13T07:06:59.036Z [WARN] go-hclog@v0.14.1/stdlog.go:49: nomad: serf: Shutdown without a Leave 2021-09-13T07:06:59.036Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down nomad-052 2021-09-13T07:06:59.036Z [DEBUG] nomad/leader.go:84: nomad: shutting down leader loop === FAIL: client TestFS_List (6.89s) === PAUSE TestFS_List === CONT TestFS_List nomad-054 2021-09-13T07:06:59.039Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/api.go:549: nomad.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:127.0.0.1:9614 Address:127.0.0.1:9614}]" nomad-054 2021-09-13T07:06:59.041Z [INFO] go-hclog@v0.14.1/stdlog.go:47: nomad: serf: EventMemberJoin: nomad-054.global 127.0.0.1 nomad-054 2021-09-13T07:06:59.041Z [INFO] nomad/server.go:1465: nomad: starting scheduling worker(s): num_workers=2 schedulers=[system, sysbatch, service, batch, _core] nomad-052 2021-09-13T07:06:59.041Z [INFO] nomad/leader.go:88: nomad: cluster leadership lost 2021-09-13T07:06:59.041Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.041Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.042Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= nomad-054 2021-09-13T07:06:59.043Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:152: nomad.raft: entering follower state: follower="Node at 127.0.0.1:9614 [Follower]" leader= nomad-054 2021-09-13T07:06:59.043Z [INFO] nomad/serf.go:60: nomad: adding server: server="nomad-054.global (Addr: 127.0.0.1:9614) (DC: dc1)" 2021-09-13T07:06:59.049Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:06:59.049Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[mock_driver exec docker java] undetected:[raw_exec qemu]]" 2021-09-13T07:06:59.050Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:06:59.050Z [INFO] client/client.go:567: client: started client: node_id=cdc12eb7-79f6-1004-1603-4100156564d1 2021-09-13T07:06:59.053Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:06:59.057Z [TRACE] client/client.go:1854: client: next heartbeat: period=11.933185025s 2021-09-13T07:06:59.057Z [DEBUG] client/client.go:1857: client: state updated: node_status=ready 2021-09-13T07:06:59.057Z [DEBUG] client/client.go:2166: client: updated allocations: index=1 total=0 pulled=0 filtered=0 2021-09-13T07:06:59.057Z [DEBUG] client/client.go:2244: client: allocation updates: added=0 removed=0 updated=0 ignored=0 2021-09-13T07:06:59.057Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=0 removed=0 updated=0 ignored=0 errors=0 2021-09-13T07:06:59.058Z [TRACE] client/client.go:1854: client: next heartbeat: period=15.780277811s nomad-053 2021-09-13T07:06:59.065Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=canonicalize warnings=[] error= nomad-053 2021-09-13T07:06:59.065Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=connect warnings=[] error= nomad-053 2021-09-13T07:06:59.065Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=expose-check warnings=[] error= nomad-053 2021-09-13T07:06:59.065Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=constraints warnings=[] error= nomad-053 2021-09-13T07:06:59.065Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=connect warnings=[] error= nomad-053 2021-09-13T07:06:59.065Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=expose-check warnings=[] error= nomad-053 2021-09-13T07:06:59.066Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=validate warnings=["Group "web" has warnings: 1 error occurred: * 1 error occurred: * Task "web": task network resources have been deprecated as of Nomad 0.12.0. Please configure networking via group network block. "] error= nomad-053 2021-09-13T07:06:59.066Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=memory_oversubscription warnings=[] error= === CONT TestFS_List wait.go:185: Job "mock-batch-5920a488-f314-1712-eb5d-4e637262e0b1" registered nomad-054 2021-09-13T07:06:59.565Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=5bec0684-f829-3c8e-0fc1-468ab858ca68 nomad-054 2021-09-13T07:06:59.565Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=5bec0684-f829-3c8e-0fc1-468ab858ca68 job_id=mock-batch-5920a488-f314-1712-eb5d-4e637262e0b1 namespace=default algorithm=binpack nomad-054 2021-09-13T07:06:59.565Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=5bec0684-f829-3c8e-0fc1-468ab858ca68 job_id=mock-batch-5920a488-f314-1712-eb5d-4e637262e0b1 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-054 2021-09-13T07:06:59.565Z [DEBUG] nomad/worker.go:418: worker: created evaluation: eval="" nomad-054 2021-09-13T07:06:59.565Z [DEBUG] scheduler/generic_sched.go:275: worker.batch_sched: failed to place all allocations, blocked eval created: eval_id=5bec0684-f829-3c8e-0fc1-468ab858ca68 job_id=mock-batch-5920a488-f314-1712-eb5d-4e637262e0b1 namespace=default blocked_eval_id=7984531e-e82d-d696-edf0-255d7d31f6df nomad-054 2021-09-13T07:06:59.566Z [DEBUG] scheduler/util.go:671: worker.batch_sched: setting eval status: eval_id=5bec0684-f829-3c8e-0fc1-468ab858ca68 job_id=mock-batch-5920a488-f314-1712-eb5d-4e637262e0b1 namespace=default status=complete nomad-054 2021-09-13T07:06:59.566Z [DEBUG] nomad/worker.go:377: worker: updated evaluation: eval="" nomad-054 2021-09-13T07:06:59.566Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=5bec0684-f829-3c8e-0fc1-468ab858ca68 2021-09-13T07:06:59.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:59.803Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:59.937Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:59.937Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:00.054Z [DEBUG] client/client.go:2217: client: state changed, updating node and re-registering 2021-09-13T07:07:00.055Z [TRACE] client/client.go:1854: client: next heartbeat: period=14.124096248s nomad-053 2021-09-13T07:07:00.056Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=b02af59e-35e5-e898-4c94-a051a5ce5740 nomad-053 2021-09-13T07:07:00.056Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=b02af59e-35e5-e898-4c94-a051a5ce5740 job_id=mock-batch-06855827-c751-b575-bb20-b8b48eda6093 namespace=default algorithm=binpack nomad-053 2021-09-13T07:07:00.056Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=b02af59e-35e5-e898-4c94-a051a5ce5740 job_id=mock-batch-06855827-c751-b575-bb20-b8b48eda6093 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-053 2021-09-13T07:07:00.056Z [DEBUG] nomad/worker.go:480: worker: reblocked evaluation: eval="" nomad-053 2021-09-13T07:07:00.056Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=b02af59e-35e5-e898-4c94-a051a5ce5740 2021-09-13T07:07:00.056Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:07:00.057Z [TRACE] client/client.go:1854: client: next heartbeat: period=19.846935806s 2021-09-13T07:07:00.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:00.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:00.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:00.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:00.554Z [DEBUG] client/client.go:2217: client: state changed, updating node and re-registering nomad-054 2021-09-13T07:07:00.555Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=7984531e-e82d-d696-edf0-255d7d31f6df nomad-054 2021-09-13T07:07:00.555Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=7984531e-e82d-d696-edf0-255d7d31f6df job_id=mock-batch-5920a488-f314-1712-eb5d-4e637262e0b1 namespace=default algorithm=binpack nomad-054 2021-09-13T07:07:00.555Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=7984531e-e82d-d696-edf0-255d7d31f6df job_id=mock-batch-5920a488-f314-1712-eb5d-4e637262e0b1 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-054 2021-09-13T07:07:00.556Z [DEBUG] nomad/worker.go:480: worker: reblocked evaluation: eval="" nomad-054 2021-09-13T07:07:00.556Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=7984531e-e82d-d696-edf0-255d7d31f6df 2021-09-13T07:07:00.556Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:07:00.556Z [TRACE] client/client.go:1854: client: next heartbeat: period=10.67224217s 2021-09-13T07:07:00.827Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:00.827Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:00.937Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:00.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:01.312Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:01.313Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:01.446Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:01.446Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:01.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:01.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:01.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:01.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:02.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:02.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:02.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:02.439Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:02.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:02.805Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:02.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:02.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:03.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:03.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:03.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:03.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:03.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:03.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:03.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:03.939Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:04.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:04.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:04.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:04.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:04.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:04.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:04.946Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:04.946Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:05.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:05.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 === CONT TestFS_List wait.go:223: Error Trace: wait.go:223 wait.go:32 wait.go:18 wait.go:197 wait.go:231 fs_endpoint_test.go:232 Error: Received unexpected error: 0 allocations; evals: []*structs.Evaluation{ &structs.Evaluation{ _struct: false, ID: "7984531e-e82d-d696-edf0-255d7d31f6df", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "queued-allocs", JobID: "mock-batch-5920a488-f314-1712-eb5d-4e637262e0b1", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "blocked", StatusDescription: "created to place remaining allocations", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "5bec0684-f829-3c8e-0fc1-468ab858ca68", BlockedEval: "", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 91560, CoalescedFailures: 0, }, }, ClassEligibility: {"v1:17255589972121564491":true}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {}, LeaderACL: "", SnapshotIndex: 0x9, CreateIndex: 0xa, ModifyIndex: 0xa, CreateTime: 1631516819565741019, ModifyTime: 1631516819565741019, }, &structs.Evaluation{ _struct: false, ID: "5bec0684-f829-3c8e-0fc1-468ab858ca68", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "job-register", JobID: "mock-batch-5920a488-f314-1712-eb5d-4e637262e0b1", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "complete", StatusDescription: "", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "", BlockedEval: "7984531e-e82d-d696-edf0-255d7d31f6df", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 91560, CoalescedFailures: 0, }, }, ClassEligibility: {}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {"web":1}, LeaderACL: "", SnapshotIndex: 0x9, CreateIndex: 0x9, ModifyIndex: 0xb, CreateTime: 1631516819564102141, ModifyTime: 1631516819566015520, }, } Test: TestFS_List 2021-09-13T07:07:05.702Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:07:05.702Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:07:05.702Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:07:05.702Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:07:05.702Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:07:05.702Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:07:05.702Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:07:05.702Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:07:05.702Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:07:05.702Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:07:05.702Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:07:05.702Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:07:05.702Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:07:05.703Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down 2021-09-13T07:07:05.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:05.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 nomad-054 2021-09-13T07:07:05.924Z [INFO] nomad/server.go:622: nomad: shutting down server nomad-054 2021-09-13T07:07:05.924Z [WARN] go-hclog@v0.14.1/stdlog.go:49: nomad: serf: Shutdown without a Leave nomad-054 2021-09-13T07:07:05.927Z [DEBUG] nomad/leader.go:84: nomad: shutting down leader loop nomad-053 2021-09-13T07:07:05.928Z [INFO] nomad/server.go:622: nomad: shutting down server nomad-053 2021-09-13T07:07:05.928Z [WARN] go-hclog@v0.14.1/stdlog.go:49: nomad: serf: Shutdown without a Leave === FAIL: client TestFS_List_ACL (7.63s) === PAUSE TestFS_List_ACL === CONT TestFS_List_ACL nomad-053 2021-09-13T07:06:58.302Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/api.go:549: nomad.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:127.0.0.1:9612 Address:127.0.0.1:9612}]" nomad-053 2021-09-13T07:06:58.302Z [INFO] go-hclog@v0.14.1/stdlog.go:47: nomad: serf: EventMemberJoin: nomad-053.global 127.0.0.1 nomad-053 2021-09-13T07:06:58.303Z [INFO] nomad/server.go:1465: nomad: starting scheduling worker(s): num_workers=2 schedulers=[sysbatch, service, batch, system, _core] 2021-09-13T07:06:58.303Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.304Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.305Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= nomad-053 2021-09-13T07:06:58.305Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:152: nomad.raft: entering follower state: follower="Node at 127.0.0.1:9612 [Follower]" leader= nomad-053 2021-09-13T07:06:58.305Z [INFO] nomad/serf.go:60: nomad: adding server: server="nomad-053.global (Addr: 127.0.0.1:9612) (DC: dc1)" 2021-09-13T07:06:58.309Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:58.309Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 nomad-053 2021-09-13T07:06:58.378Z [WARN] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:214: nomad.raft: heartbeat timeout reached, starting election: last-leader= nomad-053 2021-09-13T07:06:58.378Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:250: nomad.raft: entering candidate state: node="Node at 127.0.0.1:9612 [Candidate]" term=2 nomad-053 2021-09-13T07:06:58.388Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:268: nomad.raft: votes: needed=1 nomad-053 2021-09-13T07:06:58.389Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:287: nomad.raft: vote granted: from=127.0.0.1:9612 term=2 tally=1 nomad-053 2021-09-13T07:06:58.389Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:292: nomad.raft: election won: tally=1 nomad-053 2021-09-13T07:06:58.389Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:363: nomad.raft: entering leader state: leader="Node at 127.0.0.1:9612 [Leader]" nomad-053 2021-09-13T07:06:58.389Z [INFO] nomad/leader.go:75: nomad: cluster leadership acquired nomad-053 2021-09-13T07:06:58.389Z [TRACE] nomad/fsm.go:337: nomad.fsm: ClusterSetMetadata: cluster_id=fb0e89eb-9ce0-beed-5209-eb3c2732d02d create_time=1631516818389585254 nomad-053 2021-09-13T07:06:58.389Z [INFO] nomad/leader.go:1628: nomad.core: established cluster id: cluster_id=fb0e89eb-9ce0-beed-5209-eb3c2732d02d create_time=1631516818389585254 nomad-053 2021-09-13T07:06:58.389Z [TRACE] drainer/watch_jobs.go:145: nomad.drain.job_watcher: getting job allocs at index: index=1 2021-09-13T07:06:58.452Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:58.452Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:58.526Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest3592766366/client 2021-09-13T07:06:58.526Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest3592766366/allocs 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.541Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:58.553Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:06:58.553Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_aws, env_gce, env_azure] 2021-09-13T07:06:58.554Z [INFO] fingerprint/cgroup_linux.go:37: client.fingerprint_mgr.cgroup: cgroups are available 2021-09-13T07:06:58.554Z [DEBUG] fingerprint/cni.go:26: client.fingerprint_mgr: CNI config dir is not set or does not exist, skipping: cni_config_dir=/opt/cni/config 2021-09-13T07:06:58.571Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=cgroup period=15s 2021-09-13T07:06:58.572Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:06:58.572Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:06:58.572Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:06:58.573Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=consul period=15s 2021-09-13T07:06:58.579Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:06:58.579Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:06:58.579Z [DEBUG] fingerprint/network.go:92: client.fingerprint_mgr.network: link speed could not be detected and no speed specified by user, falling back to default speed: mbits=1000 2021-09-13T07:06:58.579Z [DEBUG] fingerprint/network.go:112: client.fingerprint_mgr.network: detected interface IP: interface=ens5 IP=172.31.11.103 2021-09-13T07:06:58.584Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=lo 2021-09-13T07:06:58.584Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/lo/speed 2021-09-13T07:06:58.584Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:58.678Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:06:58.678Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:06:58.678Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:58.751Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=docker0 2021-09-13T07:06:58.751Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/docker0/speed 2021-09-13T07:06:58.751Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:58.772Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=br-8a6b4474e385 2021-09-13T07:06:58.772Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/br-8a6b4474e385/speed 2021-09-13T07:06:58.772Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:58.775Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=vault period=15s 2021-09-13T07:06:58.780Z [DEBUG] fingerprint/env_gce.go:110: client.fingerprint_mgr.env_gce: could not read value for attribute: attribute=machine-type resp_code=404 2021-09-13T07:06:58.780Z [DEBUG] fingerprint/env_azure.go:95: client.fingerprint_mgr.env_azure: could not read value for attribute: attribute=compute/azEnvironment resp_code=404 2021-09-13T07:06:58.781Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, bridge, cgroup, cpu, host, network, nomad, signal, storage] 2021-09-13T07:06:58.781Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:06:58.781Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:06:58.781Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:06:58.781Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:06:58.782Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:06:58.786Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:06:58.786Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:06:58.787Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:06:58.793Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:06:58.793Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:06:58.794Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:06:58.808Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:58.808Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:58.809Z [DEBUG] servers/manager.go:205: client.server_mgr: new server list: new_servers=[127.0.0.1:9612] old_servers=[] 2021-09-13T07:06:58.831Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:06:58.937Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:58.937Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 === CONT TestFS_List_ACL wait.go:185: Job "mock-batch-06855827-c751-b575-bb20-b8b48eda6093" registered nomad-053 2021-09-13T07:06:59.067Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=5504de73-7aa9-8aa0-e8dc-9ee63c91ee6f nomad-053 2021-09-13T07:06:59.068Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=5504de73-7aa9-8aa0-e8dc-9ee63c91ee6f job_id=mock-batch-06855827-c751-b575-bb20-b8b48eda6093 namespace=default algorithm=binpack nomad-053 2021-09-13T07:06:59.068Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=5504de73-7aa9-8aa0-e8dc-9ee63c91ee6f job_id=mock-batch-06855827-c751-b575-bb20-b8b48eda6093 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-053 2021-09-13T07:06:59.068Z [DEBUG] nomad/worker.go:418: worker: created evaluation: eval="" nomad-053 2021-09-13T07:06:59.069Z [DEBUG] scheduler/generic_sched.go:275: worker.batch_sched: failed to place all allocations, blocked eval created: eval_id=5504de73-7aa9-8aa0-e8dc-9ee63c91ee6f job_id=mock-batch-06855827-c751-b575-bb20-b8b48eda6093 namespace=default blocked_eval_id=b02af59e-35e5-e898-4c94-a051a5ce5740 nomad-053 2021-09-13T07:06:59.069Z [DEBUG] scheduler/util.go:671: worker.batch_sched: setting eval status: eval_id=5504de73-7aa9-8aa0-e8dc-9ee63c91ee6f job_id=mock-batch-06855827-c751-b575-bb20-b8b48eda6093 namespace=default status=complete nomad-053 2021-09-13T07:06:59.069Z [DEBUG] nomad/worker.go:377: worker: updated evaluation: eval="" nomad-053 2021-09-13T07:06:59.070Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=5504de73-7aa9-8aa0-e8dc-9ee63c91ee6f nomad-054 2021-09-13T07:06:59.138Z [WARN] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:214: nomad.raft: heartbeat timeout reached, starting election: last-leader= nomad-054 2021-09-13T07:06:59.152Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:250: nomad.raft: entering candidate state: node="Node at 127.0.0.1:9614 [Candidate]" term=2 nomad-054 2021-09-13T07:06:59.152Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:268: nomad.raft: votes: needed=1 nomad-054 2021-09-13T07:06:59.152Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:287: nomad.raft: vote granted: from=127.0.0.1:9614 term=2 tally=1 nomad-054 2021-09-13T07:06:59.152Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:292: nomad.raft: election won: tally=1 nomad-054 2021-09-13T07:06:59.152Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:363: nomad.raft: entering leader state: leader="Node at 127.0.0.1:9614 [Leader]" nomad-054 2021-09-13T07:06:59.153Z [INFO] nomad/leader.go:75: nomad: cluster leadership acquired nomad-054 2021-09-13T07:06:59.153Z [TRACE] nomad/fsm.go:337: nomad.fsm: ClusterSetMetadata: cluster_id=71ad90fc-eb67-c1bc-cc2a-cb3bf2f01247 create_time=1631516819153698273 nomad-054 2021-09-13T07:06:59.153Z [INFO] nomad/leader.go:1628: nomad.core: established cluster id: cluster_id=71ad90fc-eb67-c1bc-cc2a-cb3bf2f01247 create_time=1631516819153698273 nomad-054 2021-09-13T07:06:59.154Z [TRACE] drainer/watch_jobs.go:145: nomad.drain.job_watcher: getting job allocs at index: index=1 2021-09-13T07:06:59.174Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest1064750086/client 2021-09-13T07:06:59.175Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest1064750086/allocs 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.178Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.179Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.179Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:06:59.195Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:06:59.195Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_gce, env_azure, env_aws] 2021-09-13T07:06:59.196Z [INFO] fingerprint/cgroup_linux.go:37: client.fingerprint_mgr.cgroup: cgroups are available 2021-09-13T07:06:59.197Z [DEBUG] fingerprint/cni.go:26: client.fingerprint_mgr: CNI config dir is not set or does not exist, skipping: cni_config_dir=/opt/cni/config 2021-09-13T07:06:59.197Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=cgroup period=15s 2021-09-13T07:06:59.198Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:06:59.198Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:06:59.198Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:06:59.198Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=consul period=15s 2021-09-13T07:06:59.246Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:06:59.247Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:06:59.247Z [DEBUG] fingerprint/network.go:92: client.fingerprint_mgr.network: link speed could not be detected and no speed specified by user, falling back to default speed: mbits=1000 2021-09-13T07:06:59.247Z [DEBUG] fingerprint/network.go:112: client.fingerprint_mgr.network: detected interface IP: interface=ens5 IP=172.31.11.103 2021-09-13T07:06:59.248Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=lo 2021-09-13T07:06:59.248Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/lo/speed 2021-09-13T07:06:59.248Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:59.265Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:06:59.265Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:06:59.265Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:59.268Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=docker0 2021-09-13T07:06:59.268Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/docker0/speed 2021-09-13T07:06:59.268Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:59.272Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=br-8a6b4474e385 2021-09-13T07:06:59.272Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/br-8a6b4474e385/speed 2021-09-13T07:06:59.272Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:06:59.275Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=vault period=15s 2021-09-13T07:06:59.280Z [DEBUG] fingerprint/env_gce.go:107: client.fingerprint_mgr.env_gce: could not read value for attribute: attribute=machine-type error="Get "http://169.254.169.254/computeMetadata/v1/instance/machine-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:06:59.280Z [DEBUG] fingerprint/env_gce.go:282: client.fingerprint_mgr.env_gce: error querying GCE Metadata URL, skipping 2021-09-13T07:06:59.282Z [DEBUG] fingerprint/env_azure.go:92: client.fingerprint_mgr.env_azure: could not read value for attribute: attribute=compute/azEnvironment error="Get "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2019-06-04&format=text": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:06:59.282Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, bridge, cgroup, cpu, host, network, nomad, signal, storage] 2021-09-13T07:06:59.282Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:06:59.282Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:06:59.282Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:06:59.282Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:06:59.283Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:06:59.283Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:06:59.283Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:06:59.283Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:06:59.285Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:06:59.285Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:06:59.285Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:06:59.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:06:59.304Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:06:59.304Z [DEBUG] servers/manager.go:205: client.server_mgr: new server list: new_servers=[127.0.0.1:9614] old_servers=[] 2021-09-13T07:06:59.336Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:06:59.441Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:06:59.441Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:06:59.550Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:06:59.550Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[mock_driver exec docker java] undetected:[raw_exec qemu]]" 2021-09-13T07:06:59.550Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:06:59.551Z [INFO] client/client.go:567: client: started client: node_id=84ca3583-8932-a4a1-40b9-31491f0bd853 2021-09-13T07:06:59.553Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:06:59.553Z [DEBUG] client/client.go:2166: client: updated allocations: index=1 total=0 pulled=0 filtered=0 2021-09-13T07:06:59.553Z [DEBUG] client/client.go:2244: client: allocation updates: added=0 removed=0 updated=0 ignored=0 2021-09-13T07:06:59.553Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=0 removed=0 updated=0 ignored=0 errors=0 2021-09-13T07:06:59.554Z [TRACE] client/client.go:1854: client: next heartbeat: period=19.312832408s 2021-09-13T07:06:59.554Z [DEBUG] client/client.go:1857: client: state updated: node_status=ready nomad-054 2021-09-13T07:06:59.563Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=canonicalize warnings=[] error= nomad-054 2021-09-13T07:06:59.563Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=connect warnings=[] error= nomad-054 2021-09-13T07:06:59.563Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=expose-check warnings=[] error= nomad-054 2021-09-13T07:06:59.563Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=constraints warnings=[] error= nomad-054 2021-09-13T07:06:59.563Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=connect warnings=[] error= nomad-054 2021-09-13T07:06:59.563Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=expose-check warnings=[] error= nomad-054 2021-09-13T07:06:59.564Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=validate warnings=["Group "web" has warnings: 1 error occurred: * 1 error occurred: * Task "web": task network resources have been deprecated as of Nomad 0.12.0. Please configure networking via group network block. "] error= nomad-054 2021-09-13T07:06:59.564Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=memory_oversubscription warnings=[] error= === CONT TestFS_List_ACL wait.go:223: Error Trace: wait.go:223 wait.go:32 wait.go:18 wait.go:197 fs_endpoint_test.go:277 Error: Received unexpected error: 0 allocations; evals: []*structs.Evaluation{ &structs.Evaluation{ _struct: false, ID: "b02af59e-35e5-e898-4c94-a051a5ce5740", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "queued-allocs", JobID: "mock-batch-06855827-c751-b575-bb20-b8b48eda6093", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "blocked", StatusDescription: "created to place remaining allocations", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "5504de73-7aa9-8aa0-e8dc-9ee63c91ee6f", BlockedEval: "", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 119999, CoalescedFailures: 0, }, }, ClassEligibility: {"v1:8392187951761676814":true}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {}, LeaderACL: "", SnapshotIndex: 0x3f2, CreateIndex: 0xa, ModifyIndex: 0xa, CreateTime: 1631516819068795724, ModifyTime: 1631516819068795724, }, &structs.Evaluation{ _struct: false, ID: "5504de73-7aa9-8aa0-e8dc-9ee63c91ee6f", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "job-register", JobID: "mock-batch-06855827-c751-b575-bb20-b8b48eda6093", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "complete", StatusDescription: "", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "", BlockedEval: "b02af59e-35e5-e898-4c94-a051a5ce5740", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 119999, CoalescedFailures: 0, }, }, ClassEligibility: {}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {"web":1}, LeaderACL: "", SnapshotIndex: 0x3f2, CreateIndex: 0x9, ModifyIndex: 0xb, CreateTime: 1631516819066960220, ModifyTime: 1631516819069701899, }, } Test: TestFS_List_ACL 2021-09-13T07:07:05.326Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:07:05.326Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:07:05.326Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:07:05.326Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:07:05.326Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:07:05.326Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:07:05.326Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:07:05.326Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:07:05.326Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:07:05.326Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:07:05.326Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:07:05.326Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:07:05.326Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:07:05.326Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down 2021-09-13T07:07:05.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:05.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 === FAIL: client TestFS_Stat_ACL (6.86s) === PAUSE TestFS_Stat_ACL === CONT TestFS_Stat_ACL 2021-09-13T07:07:05.933Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.933Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.933Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.933Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.933Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.934Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.934Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.934Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.934Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.934Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.934Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.934Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= nomad-053 2021-09-13T07:07:05.934Z [INFO] nomad/leader.go:88: nomad: cluster leadership lost 2021-09-13T07:07:05.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:05.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 nomad-055 2021-09-13T07:07:05.947Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/api.go:549: nomad.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:127.0.0.1:9616 Address:127.0.0.1:9616}]" nomad-055 2021-09-13T07:07:05.948Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:152: nomad.raft: entering follower state: follower="Node at 127.0.0.1:9616 [Follower]" leader= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:05.947Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= nomad-055 2021-09-13T07:07:05.949Z [INFO] go-hclog@v0.14.1/stdlog.go:47: nomad: serf: EventMemberJoin: nomad-055.global 127.0.0.1 nomad-055 2021-09-13T07:07:05.949Z [INFO] nomad/server.go:1465: nomad: starting scheduling worker(s): num_workers=2 schedulers=[service, batch, system, sysbatch, _core] nomad-055 2021-09-13T07:07:05.950Z [INFO] nomad/serf.go:60: nomad: adding server: server="nomad-055.global (Addr: 127.0.0.1:9616) (DC: dc1)" 2021-09-13T07:07:05.950Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:07:05.950Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_aws, env_gce, env_azure] 2021-09-13T07:07:05.951Z [INFO] fingerprint/cgroup_linux.go:37: client.fingerprint_mgr.cgroup: cgroups are available 2021-09-13T07:07:05.951Z [DEBUG] fingerprint/cni.go:26: client.fingerprint_mgr: CNI config dir is not set or does not exist, skipping: cni_config_dir=/opt/cni/config 2021-09-13T07:07:05.951Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=cgroup period=15s 2021-09-13T07:07:05.951Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:07:05.951Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:07:05.951Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:07:05.956Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=consul period=15s 2021-09-13T07:07:05.959Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:07:05.960Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:07:05.960Z [DEBUG] fingerprint/network.go:92: client.fingerprint_mgr.network: link speed could not be detected and no speed specified by user, falling back to default speed: mbits=1000 2021-09-13T07:07:05.960Z [DEBUG] fingerprint/network.go:112: client.fingerprint_mgr.network: detected interface IP: interface=ens5 IP=172.31.11.103 2021-09-13T07:07:05.965Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=lo 2021-09-13T07:07:05.965Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/lo/speed 2021-09-13T07:07:05.965Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:05.977Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:07:05.977Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:07:05.977Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:05.995Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=docker0 2021-09-13T07:07:05.996Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/docker0/speed 2021-09-13T07:07:05.996Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:06.003Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=br-8a6b4474e385 2021-09-13T07:07:06.003Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/br-8a6b4474e385/speed 2021-09-13T07:07:06.003Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:06.008Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=vault period=15s nomad-055 2021-09-13T07:07:06.009Z [WARN] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:214: nomad.raft: heartbeat timeout reached, starting election: last-leader= nomad-055 2021-09-13T07:07:06.009Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:250: nomad.raft: entering candidate state: node="Node at 127.0.0.1:9616 [Candidate]" term=2 nomad-055 2021-09-13T07:07:06.009Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:268: nomad.raft: votes: needed=1 nomad-055 2021-09-13T07:07:06.009Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:287: nomad.raft: vote granted: from=127.0.0.1:9616 term=2 tally=1 nomad-055 2021-09-13T07:07:06.009Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:292: nomad.raft: election won: tally=1 nomad-055 2021-09-13T07:07:06.009Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:363: nomad.raft: entering leader state: leader="Node at 127.0.0.1:9616 [Leader]" nomad-055 2021-09-13T07:07:06.009Z [INFO] nomad/leader.go:75: nomad: cluster leadership acquired nomad-055 2021-09-13T07:07:06.010Z [TRACE] nomad/fsm.go:337: nomad.fsm: ClusterSetMetadata: cluster_id=4b8dff29-674c-d995-b613-5e5aa807cd0e create_time=1631516826010262186 nomad-055 2021-09-13T07:07:06.010Z [INFO] nomad/leader.go:1628: nomad.core: established cluster id: cluster_id=4b8dff29-674c-d995-b613-5e5aa807cd0e create_time=1631516826010262186 nomad-055 2021-09-13T07:07:06.010Z [TRACE] drainer/watch_jobs.go:145: nomad.drain.job_watcher: getting job allocs at index: index=1 2021-09-13T07:07:06.014Z [DEBUG] fingerprint/env_gce.go:107: client.fingerprint_mgr.env_gce: could not read value for attribute: attribute=machine-type error="Get "http://169.254.169.254/computeMetadata/v1/instance/machine-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:07:06.014Z [DEBUG] fingerprint/env_gce.go:282: client.fingerprint_mgr.env_gce: error querying GCE Metadata URL, skipping 2021-09-13T07:07:06.016Z [DEBUG] fingerprint/env_azure.go:92: client.fingerprint_mgr.env_azure: could not read value for attribute: attribute=compute/azEnvironment error="Get "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2019-06-04&format=text": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:07:06.016Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, bridge, cgroup, cpu, host, network, nomad, signal, storage] 2021-09-13T07:07:06.016Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:07:06.016Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:07:06.016Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:07:06.020Z [TRACE] consul/catalog_testing.go:61: mock_consul: Datacenters(): dcs=[dc1] error=nil 2021-09-13T07:07:06.020Z [DEBUG] client/client.go:2717: client.consul: bootstrap contacting Consul DCs: consul_dcs=[dc1] 2021-09-13T07:07:06.020Z [TRACE] consul/catalog_testing.go:66: mock_consul: Services(): service=nomad tag=rpc query_options="&{ dc1 true false false 0s 0s 0 2s _agent map[] 0 false false }" 2021-09-13T07:07:06.020Z [ERROR] client/client.go:2679: client: error discovering nomad servers: error="no Nomad Servers advertising service "nomad" in Consul datacenters: ["dc1"]" 2021-09-13T07:07:06.020Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:07:06.020Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:07:06.021Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:07:06.021Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:06.021Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:06.036Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:06.034Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:07:06.034Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:07:06.073Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:07:06.135Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest3007373657/client 2021-09-13T07:07:06.140Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest3007373657/allocs 2021-09-13T07:07:06.146Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.147Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.148Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.165Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:07:06.166Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_aws, env_gce, env_azure] 2021-09-13T07:07:06.166Z [INFO] fingerprint/cgroup_linux.go:37: client.fingerprint_mgr.cgroup: cgroups are available 2021-09-13T07:07:06.166Z [DEBUG] fingerprint/cni.go:26: client.fingerprint_mgr: CNI config dir is not set or does not exist, skipping: cni_config_dir=/opt/cni/config 2021-09-13T07:07:06.166Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=cgroup period=15s 2021-09-13T07:07:06.169Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:07:06.169Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:07:06.169Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:07:06.170Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=consul period=15s 2021-09-13T07:07:06.252Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:07:06.252Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[exec mock_driver docker java] undetected:[qemu raw_exec]]" 2021-09-13T07:07:06.252Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:06.252Z [INFO] client/client.go:567: client: started client: node_id=b8870937-ae03-02c3-11ef-3120ca02d919 2021-09-13T07:07:06.252Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:07:06.252Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:07:06.252Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:07:06.252Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:07:06.252Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:07:06.252Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:07:06.252Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:07:06.252Z [WARN] servers/manager.go:238: client.server_mgr: no servers available 2021-09-13T07:07:06.252Z [DEBUG] client/client.go:1781: client: registration waiting on servers 2021-09-13T07:07:06.252Z [WARN] servers/manager.go:238: client.server_mgr: no servers available 2021-09-13T07:07:06.252Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:07:06.252Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:07:06.252Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:07:06.252Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:07:06.252Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:07:06.252Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec === CONT TestFS_Stat_ACL wait.go:185: Job "mock-batch-5915e571-48f7-8a87-0318-da2e3a43fc81" registered nomad-055 2021-09-13T07:07:06.660Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=2186bb2c-97ca-bc62-a15a-78aec1e93fcf nomad-055 2021-09-13T07:07:06.660Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=2186bb2c-97ca-bc62-a15a-78aec1e93fcf job_id=mock-batch-5915e571-48f7-8a87-0318-da2e3a43fc81 namespace=default algorithm=binpack nomad-055 2021-09-13T07:07:06.660Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=2186bb2c-97ca-bc62-a15a-78aec1e93fcf job_id=mock-batch-5915e571-48f7-8a87-0318-da2e3a43fc81 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-055 2021-09-13T07:07:06.660Z [DEBUG] nomad/worker.go:418: worker: created evaluation: eval="" nomad-055 2021-09-13T07:07:06.660Z [DEBUG] scheduler/generic_sched.go:275: worker.batch_sched: failed to place all allocations, blocked eval created: eval_id=2186bb2c-97ca-bc62-a15a-78aec1e93fcf job_id=mock-batch-5915e571-48f7-8a87-0318-da2e3a43fc81 namespace=default blocked_eval_id=c34d05ee-5a42-cba4-3160-a92de7500c47 nomad-055 2021-09-13T07:07:06.661Z [DEBUG] scheduler/util.go:671: worker.batch_sched: setting eval status: eval_id=2186bb2c-97ca-bc62-a15a-78aec1e93fcf job_id=mock-batch-5915e571-48f7-8a87-0318-da2e3a43fc81 namespace=default status=complete nomad-055 2021-09-13T07:07:06.661Z [DEBUG] nomad/worker.go:377: worker: updated evaluation: eval="" nomad-055 2021-09-13T07:07:06.661Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=2186bb2c-97ca-bc62-a15a-78aec1e93fcf 2021-09-13T07:07:06.685Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=vault period=15s 2021-09-13T07:07:06.692Z [DEBUG] fingerprint/env_gce.go:107: client.fingerprint_mgr.env_gce: could not read value for attribute: attribute=machine-type error="Get "http://169.254.169.254/computeMetadata/v1/instance/machine-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:07:06.693Z [DEBUG] fingerprint/env_gce.go:282: client.fingerprint_mgr.env_gce: error querying GCE Metadata URL, skipping 2021-09-13T07:07:06.694Z [DEBUG] fingerprint/env_azure.go:92: client.fingerprint_mgr.env_azure: could not read value for attribute: attribute=compute/azEnvironment error="Get "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2019-06-04&format=text": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:07:06.694Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, bridge, cgroup, cpu, host, network, nomad, signal, storage] 2021-09-13T07:07:06.694Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:07:06.694Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:07:06.694Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:07:06.695Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:07:06.696Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:07:06.696Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:07:06.696Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:07:06.696Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:07:06.710Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:06.710Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:06.710Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:06.712Z [DEBUG] servers/manager.go:205: client.server_mgr: new server list: new_servers=[127.0.0.1:9618] old_servers=[] 2021-09-13T07:07:06.755Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:07:06.805Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:06.805Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:06.924Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:07:06.924Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[mock_driver exec docker java] undetected:[raw_exec qemu]]" 2021-09-13T07:07:06.924Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:06.924Z [INFO] client/client.go:567: client: started client: node_id=3f47fd51-9b03-b1b6-ccc0-7c8809ad981f 2021-09-13T07:07:06.926Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:07:06.926Z [DEBUG] client/client.go:2166: client: updated allocations: index=1 total=0 pulled=0 filtered=0 2021-09-13T07:07:06.926Z [DEBUG] client/client.go:2244: client: allocation updates: added=0 removed=0 updated=0 ignored=0 2021-09-13T07:07:06.926Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=0 removed=0 updated=0 ignored=0 errors=0 2021-09-13T07:07:06.935Z [TRACE] client/client.go:1854: client: next heartbeat: period=19.403160486s 2021-09-13T07:07:06.935Z [DEBUG] client/client.go:1857: client: state updated: node_status=ready nomad-056 2021-09-13T07:07:06.935Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=canonicalize warnings=[] error= nomad-056 2021-09-13T07:07:06.935Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=connect warnings=[] error= nomad-056 2021-09-13T07:07:06.935Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=expose-check warnings=[] error= nomad-056 2021-09-13T07:07:06.935Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=constraints warnings=[] error= nomad-056 2021-09-13T07:07:06.935Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=connect warnings=[] error= nomad-056 2021-09-13T07:07:06.935Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=expose-check warnings=[] error= nomad-056 2021-09-13T07:07:06.935Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=validate warnings=["Group "web" has warnings: 1 error occurred: * 1 error occurred: * Task "web": task network resources have been deprecated as of Nomad 0.12.0. Please configure networking via group network block. "] error= nomad-056 2021-09-13T07:07:06.935Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=memory_oversubscription warnings=[] error= === CONT TestFS_Stat_ACL wait.go:223: Error Trace: wait.go:223 wait.go:32 wait.go:18 wait.go:197 fs_endpoint_test.go:144 Error: Received unexpected error: 0 allocations; evals: []*structs.Evaluation{ &structs.Evaluation{ _struct: false, ID: "c34d05ee-5a42-cba4-3160-a92de7500c47", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "queued-allocs", JobID: "mock-batch-5915e571-48f7-8a87-0318-da2e3a43fc81", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "blocked", StatusDescription: "created to place remaining allocations", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "2186bb2c-97ca-bc62-a15a-78aec1e93fcf", BlockedEval: "", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 90190, CoalescedFailures: 0, }, }, ClassEligibility: {"v1:8337484102197316252":true}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {}, LeaderACL: "", SnapshotIndex: 0x3f2, CreateIndex: 0xa, ModifyIndex: 0xa, CreateTime: 1631516826660746703, ModifyTime: 1631516826660746703, }, &structs.Evaluation{ _struct: false, ID: "2186bb2c-97ca-bc62-a15a-78aec1e93fcf", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "job-register", JobID: "mock-batch-5915e571-48f7-8a87-0318-da2e3a43fc81", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "complete", StatusDescription: "", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "", BlockedEval: "c34d05ee-5a42-cba4-3160-a92de7500c47", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 90190, CoalescedFailures: 0, }, }, ClassEligibility: {}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {"web":1}, LeaderACL: "", SnapshotIndex: 0x3f2, CreateIndex: 0x9, ModifyIndex: 0xb, CreateTime: 1631516826660041126, ModifyTime: 1631516826661026529, }, } Test: TestFS_Stat_ACL 2021-09-13T07:07:12.793Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:07:12.793Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:07:12.793Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:07:12.793Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:07:12.793Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:07:12.793Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:07:12.793Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:07:12.793Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:07:12.793Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:07:12.793Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:07:12.793Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:07:12.793Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:07:12.793Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi nomad-055 2021-09-13T07:07:12.793Z [INFO] nomad/server.go:622: nomad: shutting down server nomad-055 2021-09-13T07:07:12.793Z [WARN] go-hclog@v0.14.1/stdlog.go:49: nomad: serf: Shutdown without a Leave 2021-09-13T07:07:12.793Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down nomad-055 2021-09-13T07:07:12.793Z [DEBUG] nomad/leader.go:84: nomad: shutting down leader loop === FAIL: client TestFS_Stat (7.23s) === PAUSE TestFS_Stat === CONT TestFS_Stat nomad-056 2021-09-13T07:07:06.255Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/api.go:549: nomad.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:127.0.0.1:9618 Address:127.0.0.1:9618}]" nomad-056 2021-09-13T07:07:06.256Z [INFO] go-hclog@v0.14.1/stdlog.go:47: nomad: serf: EventMemberJoin: nomad-056.global 127.0.0.1 nomad-056 2021-09-13T07:07:06.256Z [INFO] nomad/server.go:1465: nomad: starting scheduling worker(s): num_workers=2 schedulers=[batch, system, sysbatch, service, _core] 2021-09-13T07:07:06.256Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down 2021-09-13T07:07:06.256Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.256Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.256Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.257Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.257Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.257Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.257Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.257Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.257Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.257Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.257Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.257Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= nomad-056 2021-09-13T07:07:06.257Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:152: nomad.raft: entering follower state: follower="Node at 127.0.0.1:9618 [Follower]" leader= nomad-056 2021-09-13T07:07:06.258Z [INFO] nomad/serf.go:60: nomad: adding server: server="nomad-056.global (Addr: 127.0.0.1:9618) (DC: dc1)" 2021-09-13T07:07:06.270Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:07:06.270Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:07:06.270Z [DEBUG] fingerprint/network.go:92: client.fingerprint_mgr.network: link speed could not be detected and no speed specified by user, falling back to default speed: mbits=1000 2021-09-13T07:07:06.270Z [DEBUG] fingerprint/network.go:112: client.fingerprint_mgr.network: detected interface IP: interface=ens5 IP=172.31.11.103 2021-09-13T07:07:06.271Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=lo 2021-09-13T07:07:06.271Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/lo/speed 2021-09-13T07:07:06.271Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:06.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:06.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:06.322Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:07:06.322Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:07:06.322Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:06.348Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=docker0 2021-09-13T07:07:06.348Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/docker0/speed 2021-09-13T07:07:06.348Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 nomad-056 2021-09-13T07:07:06.356Z [WARN] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:214: nomad.raft: heartbeat timeout reached, starting election: last-leader= nomad-056 2021-09-13T07:07:06.356Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:250: nomad.raft: entering candidate state: node="Node at 127.0.0.1:9618 [Candidate]" term=2 nomad-056 2021-09-13T07:07:06.356Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:268: nomad.raft: votes: needed=1 nomad-056 2021-09-13T07:07:06.356Z [DEBUG] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:287: nomad.raft: vote granted: from=127.0.0.1:9618 term=2 tally=1 nomad-056 2021-09-13T07:07:06.356Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:292: nomad.raft: election won: tally=1 nomad-056 2021-09-13T07:07:06.356Z [INFO] raft@v1.1.3-0.20200211192230-365023de17e6/raft.go:363: nomad.raft: entering leader state: leader="Node at 127.0.0.1:9618 [Leader]" nomad-056 2021-09-13T07:07:06.356Z [INFO] nomad/leader.go:75: nomad: cluster leadership acquired nomad-056 2021-09-13T07:07:06.357Z [TRACE] nomad/fsm.go:337: nomad.fsm: ClusterSetMetadata: cluster_id=b3286411-0fad-eb03-c2c7-3244dbbec804 create_time=1631516826357306188 nomad-056 2021-09-13T07:07:06.357Z [INFO] nomad/leader.go:1628: nomad.core: established cluster id: cluster_id=b3286411-0fad-eb03-c2c7-3244dbbec804 create_time=1631516826357306188 nomad-056 2021-09-13T07:07:06.360Z [TRACE] drainer/watch_jobs.go:145: nomad.drain.job_watcher: getting job allocs at index: index=1 2021-09-13T07:07:06.365Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=br-8a6b4474e385 2021-09-13T07:07:06.365Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/br-8a6b4474e385/speed 2021-09-13T07:07:06.365Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:06.373Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=vault period=15s 2021-09-13T07:07:06.376Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=ami-id 2021-09-13T07:07:06.378Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=hostname 2021-09-13T07:07:06.379Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=public-hostname 2021-09-13T07:07:06.383Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=local-hostname 2021-09-13T07:07:06.384Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=local-ipv4 2021-09-13T07:07:06.385Z [DEBUG] fingerprint/env_aws.go:107: client.fingerprint_mgr.env_aws: read an empty value: attribute=mac 2021-09-13T07:07:06.387Z [WARN] fingerprint/env_aws.go:220: client.fingerprint_mgr.env_aws: failed to read EC2 metadata instance-type: error="RequestError: send request failed caused by: Get "http://169.254.169.254/latest/meta-data/instance-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:07:06.387Z [WARN] fingerprint/env_aws.go:194: client.fingerprint_mgr.env_aws: failed to find the cpu specification for this instance type 2021-09-13T07:07:06.392Z [DEBUG] fingerprint/env_gce.go:107: client.fingerprint_mgr.env_gce: could not read value for attribute: attribute=machine-type error="Get "http://169.254.169.254/computeMetadata/v1/instance/machine-type": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:07:06.393Z [DEBUG] fingerprint/env_gce.go:282: client.fingerprint_mgr.env_gce: error querying GCE Metadata URL, skipping 2021-09-13T07:07:06.396Z [DEBUG] fingerprint/env_azure.go:92: client.fingerprint_mgr.env_azure: could not read value for attribute: attribute=compute/azEnvironment error="Get "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2019-06-04&format=text": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" 2021-09-13T07:07:06.396Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, bridge, cgroup, cpu, host, network, nomad, signal, storage, env_aws] 2021-09-13T07:07:06.396Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:07:06.396Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:07:06.397Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:07:06.397Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:07:06.398Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:07:06.398Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:07:06.398Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:07:06.398Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:07:06.398Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:06.399Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:06.399Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:06.400Z [DEBUG] servers/manager.go:205: client.server_mgr: new server list: new_servers=[127.0.0.1:9616] old_servers=[] 2021-09-13T07:07:06.442Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:06.443Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:06.452Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:07:06.581Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.581Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.581Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.581Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.581Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.581Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.581Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.582Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.582Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.582Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.582Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.583Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest3338218565/client 2021-09-13T07:07:06.583Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:06.583Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest3338218565/allocs 2021-09-13T07:07:06.601Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:07:06.601Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_gce, env_azure, env_aws] 2021-09-13T07:07:06.602Z [INFO] fingerprint/cgroup_linux.go:37: client.fingerprint_mgr.cgroup: cgroups are available 2021-09-13T07:07:06.602Z [DEBUG] fingerprint/cni.go:26: client.fingerprint_mgr: CNI config dir is not set or does not exist, skipping: cni_config_dir=/opt/cni/config 2021-09-13T07:07:06.602Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=cgroup period=15s 2021-09-13T07:07:06.602Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:07:06.603Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:07:06.603Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:07:06.603Z [DEBUG] client/fingerprint_manager.go:161: client.fingerprint_mgr: fingerprinting periodically: fingerprinter=consul period=15s 2021-09-13T07:07:06.619Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:07:06.619Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:07:06.619Z [DEBUG] fingerprint/network.go:92: client.fingerprint_mgr.network: link speed could not be detected and no speed specified by user, falling back to default speed: mbits=1000 2021-09-13T07:07:06.619Z [DEBUG] fingerprint/network.go:112: client.fingerprint_mgr.network: detected interface IP: interface=ens5 IP=172.31.11.103 2021-09-13T07:07:06.621Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=lo 2021-09-13T07:07:06.621Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/lo/speed 2021-09-13T07:07:06.621Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:06.629Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=ens5 2021-09-13T07:07:06.629Z [DEBUG] fingerprint/network_linux.go:19: client.fingerprint_mgr.network: unable to read link speed: path=/sys/class/net/ens5/speed 2021-09-13T07:07:06.629Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:06.636Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=docker0 2021-09-13T07:07:06.636Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/docker0/speed 2021-09-13T07:07:06.636Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:06.649Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:07:06.649Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[mock_driver exec docker java] undetected:[raw_exec qemu]]" 2021-09-13T07:07:06.649Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:06.649Z [INFO] client/client.go:567: client: started client: node_id=fb962db3-6976-f037-074b-bb08ad12f706 2021-09-13T07:07:06.650Z [WARN] fingerprint/network_linux.go:62: client.fingerprint_mgr.network: unable to parse speed: path=/usr/sbin/ethtool device=br-8a6b4474e385 2021-09-13T07:07:06.650Z [DEBUG] fingerprint/network_linux.go:26: client.fingerprint_mgr.network: unable to parse link speed: path=/sys/class/net/br-8a6b4474e385/speed 2021-09-13T07:07:06.650Z [DEBUG] fingerprint/network.go:141: client.fingerprint_mgr.network: link speed could not be detected, falling back to default speed: mbits=1000 2021-09-13T07:07:06.652Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:07:06.653Z [DEBUG] client/client.go:2166: client: updated allocations: index=1 total=0 pulled=0 filtered=0 2021-09-13T07:07:06.653Z [DEBUG] client/client.go:2244: client: allocation updates: added=0 removed=0 updated=0 ignored=0 2021-09-13T07:07:06.653Z [DEBUG] client/client.go:2288: client: allocation updates applied: added=0 removed=0 updated=0 ignored=0 errors=0 2021-09-13T07:07:06.654Z [TRACE] client/client.go:1854: client: next heartbeat: period=11.793162241s 2021-09-13T07:07:06.654Z [DEBUG] client/client.go:1857: client: state updated: node_status=ready nomad-055 2021-09-13T07:07:06.659Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=canonicalize warnings=[] error= nomad-055 2021-09-13T07:07:06.659Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=connect warnings=[] error= nomad-055 2021-09-13T07:07:06.659Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=expose-check warnings=[] error= nomad-055 2021-09-13T07:07:06.659Z [TRACE] nomad/job_endpoint_hooks.go:64: nomad.job: job mutate results: mutator=constraints warnings=[] error= nomad-055 2021-09-13T07:07:06.659Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=connect warnings=[] error= nomad-055 2021-09-13T07:07:06.659Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=expose-check warnings=[] error= nomad-055 2021-09-13T07:07:06.659Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=validate warnings=["Group "web" has warnings: 1 error occurred: * 1 error occurred: * Task "web": task network resources have been deprecated as of Nomad 0.12.0. Please configure networking via group network block. "] error= nomad-055 2021-09-13T07:07:06.659Z [TRACE] nomad/job_endpoint_hooks.go:84: nomad.job: job validate results: validator=memory_oversubscription warnings=[] error= === CONT TestFS_Stat wait.go:185: Job "mock-batch-287a4aef-1f7d-3fb8-6f89-a8483b76e424" registered nomad-056 2021-09-13T07:07:06.936Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=aadd0303-af53-5000-cb63-b280539720c7 nomad-056 2021-09-13T07:07:06.936Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=aadd0303-af53-5000-cb63-b280539720c7 job_id=mock-batch-287a4aef-1f7d-3fb8-6f89-a8483b76e424 namespace=default algorithm=binpack nomad-056 2021-09-13T07:07:06.938Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=aadd0303-af53-5000-cb63-b280539720c7 job_id=mock-batch-287a4aef-1f7d-3fb8-6f89-a8483b76e424 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" 2021-09-13T07:07:06.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 nomad-056 2021-09-13T07:07:06.938Z [DEBUG] nomad/worker.go:418: worker: created evaluation: eval="" nomad-056 2021-09-13T07:07:06.938Z [DEBUG] scheduler/generic_sched.go:275: worker.batch_sched: failed to place all allocations, blocked eval created: eval_id=aadd0303-af53-5000-cb63-b280539720c7 job_id=mock-batch-287a4aef-1f7d-3fb8-6f89-a8483b76e424 namespace=default blocked_eval_id=80669f91-85b9-cf08-11b3-d4a72f1b5e01 nomad-056 2021-09-13T07:07:06.938Z [DEBUG] scheduler/util.go:671: worker.batch_sched: setting eval status: eval_id=aadd0303-af53-5000-cb63-b280539720c7 job_id=mock-batch-287a4aef-1f7d-3fb8-6f89-a8483b76e424 namespace=default status=complete nomad-056 2021-09-13T07:07:06.938Z [DEBUG] nomad/worker.go:377: worker: updated evaluation: eval="" nomad-056 2021-09-13T07:07:06.939Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=aadd0303-af53-5000-cb63-b280539720c7 2021-09-13T07:07:06.940Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:07.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:07.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:07.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:07.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:07.653Z [DEBUG] client/client.go:2217: client: state changed, updating node and re-registering nomad-055 2021-09-13T07:07:07.654Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=c34d05ee-5a42-cba4-3160-a92de7500c47 nomad-055 2021-09-13T07:07:07.654Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=c34d05ee-5a42-cba4-3160-a92de7500c47 job_id=mock-batch-5915e571-48f7-8a87-0318-da2e3a43fc81 namespace=default algorithm=binpack nomad-055 2021-09-13T07:07:07.654Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=c34d05ee-5a42-cba4-3160-a92de7500c47 job_id=mock-batch-5915e571-48f7-8a87-0318-da2e3a43fc81 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" nomad-055 2021-09-13T07:07:07.655Z [DEBUG] nomad/worker.go:480: worker: reblocked evaluation: eval="" nomad-055 2021-09-13T07:07:07.655Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=c34d05ee-5a42-cba4-3160-a92de7500c47 2021-09-13T07:07:07.656Z [INFO] client/client.go:1814: client: node registration complete 2021-09-13T07:07:07.656Z [TRACE] client/client.go:1854: client: next heartbeat: period=15.087891968s 2021-09-13T07:07:07.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:07.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:07.927Z [DEBUG] client/client.go:2217: client: state changed, updating node and re-registering 2021-09-13T07:07:07.928Z [TRACE] client/client.go:1854: client: next heartbeat: period=13.422154011s nomad-056 2021-09-13T07:07:07.928Z [DEBUG] nomad/worker.go:185: worker: dequeued evaluation: eval_id=80669f91-85b9-cf08-11b3-d4a72f1b5e01 2021-09-13T07:07:07.928Z [INFO] client/client.go:1814: client: node registration complete nomad-056 2021-09-13T07:07:07.928Z [TRACE] scheduler/rank.go:180: worker.batch_sched.binpack: NewBinPackIterator created: eval_id=80669f91-85b9-cf08-11b3-d4a72f1b5e01 job_id=mock-batch-287a4aef-1f7d-3fb8-6f89-a8483b76e424 namespace=default algorithm=binpack nomad-056 2021-09-13T07:07:07.929Z [DEBUG] scheduler/generic_sched.go:356: worker.batch_sched: reconciled current state with desired state: eval_id=80669f91-85b9-cf08-11b3-d4a72f1b5e01 job_id=mock-batch-287a4aef-1f7d-3fb8-6f89-a8483b76e424 namespace=default results="Total changes: (place 1) (destructive 0) (inplace 0) (stop 0) Desired Changes for "web": (place 1) (inplace 0) (destructive 0) (stop 0) (migrate 0) (ignore 0) (canary 0)" 2021-09-13T07:07:07.929Z [TRACE] client/client.go:1854: client: next heartbeat: period=14.688500496s nomad-056 2021-09-13T07:07:07.929Z [DEBUG] nomad/worker.go:480: worker: reblocked evaluation: eval="" nomad-056 2021-09-13T07:07:07.929Z [DEBUG] nomad/worker.go:223: worker: ack evaluation: eval_id=80669f91-85b9-cf08-11b3-d4a72f1b5e01 2021-09-13T07:07:07.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:07.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:08.302Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:08.304Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:08.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:08.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:08.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:08.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:08.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:08.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:09.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:09.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:09.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:09.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:09.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:09.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:09.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:09.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:10.318Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:10.318Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:10.448Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:10.450Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:10.801Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:10.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:10.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:10.938Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:11.304Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:11.305Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:11.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:11.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:11.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:11.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:11.937Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:11.937Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 2021-09-13T07:07:12.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:12.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:12.437Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:12.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 === CONT TestFS_Stat wait.go:223: Error Trace: wait.go:223 wait.go:32 wait.go:18 wait.go:197 wait.go:231 fs_endpoint_test.go:99 Error: Received unexpected error: 0 allocations; evals: []*structs.Evaluation{ &structs.Evaluation{ _struct: false, ID: "80669f91-85b9-cf08-11b3-d4a72f1b5e01", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "queued-allocs", JobID: "mock-batch-287a4aef-1f7d-3fb8-6f89-a8483b76e424", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "blocked", StatusDescription: "created to place remaining allocations", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "aadd0303-af53-5000-cb63-b280539720c7", BlockedEval: "", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 103901, CoalescedFailures: 0, }, }, ClassEligibility: {"v1:15417496649062846160":true}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {}, LeaderACL: "", SnapshotIndex: 0x9, CreateIndex: 0xa, ModifyIndex: 0xa, CreateTime: 1631516826938504351, ModifyTime: 1631516826938504351, }, &structs.Evaluation{ _struct: false, ID: "aadd0303-af53-5000-cb63-b280539720c7", Namespace: "default", Priority: 50, Type: "batch", TriggeredBy: "job-register", JobID: "mock-batch-287a4aef-1f7d-3fb8-6f89-a8483b76e424", JobModifyIndex: 0x9, NodeID: "", NodeModifyIndex: 0x0, DeploymentID: "", Status: "complete", StatusDescription: "", Wait: 0, WaitUntil: time.Time{}, NextEval: "", PreviousEval: "", BlockedEval: "80669f91-85b9-cf08-11b3-d4a72f1b5e01", FailedTGAllocs: { "web": &structs.AllocMetric{ NodesEvaluated: 1, NodesFiltered: 0, NodesAvailable: {"dc1":1}, ClassFiltered: {}, ConstraintFiltered: {}, NodesExhausted: 1, ClassExhausted: {"linux-medium-pci":1}, DimensionExhausted: {"disk":1}, QuotaExhausted: nil, ResourcesExhausted: { "web": &structs.Resources{}, }, Scores: {}, ScoreMetaData: nil, nodeScoreMeta: (*structs.NodeScoreMeta)(nil), topScores: (*kheap.ScoreHeap)(nil), AllocationTime: 103901, CoalescedFailures: 0, }, }, ClassEligibility: {}, QuotaLimitReached: "", EscapedComputedClass: false, AnnotatePlan: false, QueuedAllocations: {"web":1}, LeaderACL: "", SnapshotIndex: 0x9, CreateIndex: 0x9, ModifyIndex: 0xb, CreateTime: 1631516826935989541, ModifyTime: 1631516826938826810, }, } Test: TestFS_Stat 2021-09-13T07:07:13.163Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:07:13.164Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:07:13.164Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:07:13.164Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:07:13.164Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:07:13.164Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:07:13.164Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:07:13.164Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:07:13.164Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:07:13.164Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:07:13.164Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:07:13.164Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:07:13.164Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:07:13.164Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down 2021-09-13T07:07:13.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:13.301Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:13.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=d2bbab80-612c-e6a0-c2a5-72cd5ed8f866 2021-09-13T07:07:13.438Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=604e3bb6-376c-8ed3-7f73-d73e25cafdf8 nomad-056 2021-09-13T07:07:13.480Z [INFO] nomad/server.go:622: nomad: shutting down server nomad-056 2021-09-13T07:07:13.480Z [WARN] go-hclog@v0.14.1/stdlog.go:49: nomad: serf: Shutdown without a Leave nomad-056 2021-09-13T07:07:13.480Z [DEBUG] nomad/leader.go:84: nomad: shutting down leader loop === FAIL: client TestFingerprintManager_Run_CombinationLegacyNames (0.39s) === PAUSE TestFingerprintManager_Run_CombinationLegacyNames === CONT TestFingerprintManager_Run_CombinationLegacyNames nomad-056 2021-09-13T07:07:13.480Z [INFO] nomad/leader.go:88: nomad: cluster leadership lost 2021-09-13T07:07:13.483Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest1548627898/client 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.485Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.488Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:07:13.488Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest1548627898/allocs 2021-09-13T07:07:13.488Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[mock_driver exec docker java] undetected:[qemu raw_exec]]" 2021-09-13T07:07:13.488Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:13.488Z [INFO] client/client.go:567: client: started client: node_id=99bd4cb1-53c5-8bc5-9cda-a1cb5de06564 2021-09-13T07:07:13.488Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:07:13.488Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:07:13.488Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:07:13.488Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:07:13.488Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:07:13.488Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:07:13.488Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:07:13.497Z [WARN] servers/manager.go:238: client.server_mgr: no servers available 2021-09-13T07:07:13.497Z [DEBUG] client/client.go:1781: client: registration waiting on servers 2021-09-13T07:07:13.497Z [WARN] servers/manager.go:238: client.server_mgr: no servers available 2021-09-13T07:07:13.497Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:07:13.498Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:07:13.498Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:07:13.498Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:07:13.498Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:07:13.498Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java === CONT TestFingerprintManager_Run_CombinationLegacyNames fingerprint_manager_test.go:151: Error Trace: fingerprint_manager_test.go:151 Error: Should not be: "" Test: TestFingerprintManager_Run_CombinationLegacyNames 2021-09-13T07:07:13.871Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:07:13.871Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:07:13.871Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:07:13.871Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:07:13.871Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:07:13.871Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:07:13.871Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:07:13.871Z [WARN] servers/manager.go:238: client.server_mgr: no servers available 2021-09-13T07:07:13.872Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:07:13.872Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:07:13.872Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:07:13.872Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:07:13.872Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:07:13.872Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java === FAIL: client TestFingerprintManager_Run_Combination (0.52s) === PAUSE TestFingerprintManager_Run_Combination === CONT TestFingerprintManager_Run_Combination 2021-09-13T07:07:13.503Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:07:13.504Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_aws, env_gce, env_azure] 2021-09-13T07:07:13.504Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:07:13.505Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:07:13.505Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:07:13.505Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, cpu] 2021-09-13T07:07:13.505Z [DEBUG] client/fingerprint_manager.go:104: client.fingerprint_mgr: fingerprint modules skipped due to allow/denylist: skipped_fingerprinters=[bridge, cgroup, cni, consul, host, memory, network, nomad, signal, storage, vault, env_aws, env_gce, env_azure] 2021-09-13T07:07:13.505Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:07:13.505Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:07:13.505Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:07:13.505Z [TRACE] consul/catalog_testing.go:61: mock_consul: Datacenters(): dcs=[dc1] error=nil 2021-09-13T07:07:13.506Z [DEBUG] client/client.go:2717: client.consul: bootstrap contacting Consul DCs: consul_dcs=[dc1] 2021-09-13T07:07:13.506Z [TRACE] consul/catalog_testing.go:66: mock_consul: Services(): service=nomad tag=rpc query_options="&{ dc1 true false false 0s 0s 0 2s _agent map[] 0 false false }" 2021-09-13T07:07:13.506Z [ERROR] client/client.go:2679: client: error discovering nomad servers: error="no Nomad Servers advertising service "nomad" in Consul datacenters: ["dc1"]" 2021-09-13T07:07:13.508Z [INFO] client/client.go:599: client: using state directory: state_dir=/tmp/nomadtest2882666238/client 2021-09-13T07:07:13.508Z [INFO] client/client.go:644: client: using alloc directory: alloc_dir=/tmp/nomadtest2882666238/allocs 2021-09-13T07:07:13.509Z [INFO] cgutil/cpuset_manager_linux.go:165: client.cpuset_manager: initialized cpuset cgroup manager: parent=/nomad cpuset=0-1 2021-09-13T07:07:13.509Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_aws, env_gce, env_azure] 2021-09-13T07:07:13.513Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:07:13.513Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:07:13.513Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:07:13.513Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, cpu] 2021-09-13T07:07:13.513Z [DEBUG] client/fingerprint_manager.go:104: client.fingerprint_mgr: fingerprint modules skipped due to allow/denylist: skipped_fingerprinters=[bridge, cgroup, cni, consul, host, memory, network, nomad, signal, storage, vault, env_aws, env_gce, env_azure] 2021-09-13T07:07:13.513Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=csi 2021-09-13T07:07:13.513Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=driver 2021-09-13T07:07:13.513Z [INFO] pluginmanager/group.go:43: client.plugin: starting plugin manager: plugin-type=device 2021-09-13T07:07:13.513Z [TRACE] consul/catalog_testing.go:61: mock_consul: Datacenters(): dcs=[dc1] error=nil 2021-09-13T07:07:13.514Z [DEBUG] client/client.go:2717: client.consul: bootstrap contacting Consul DCs: consul_dcs=[dc1] 2021-09-13T07:07:13.514Z [TRACE] consul/catalog_testing.go:66: mock_consul: Services(): service=nomad tag=rpc query_options="&{ dc1 true false false 0s 0s 0 2s _agent map[] 0 false false }" 2021-09-13T07:07:13.514Z [ERROR] client/client.go:2679: client: error discovering nomad servers: error="no Nomad Servers advertising service "nomad" in Consul datacenters: ["dc1"]" 2021-09-13T07:07:13.514Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down 2021-09-13T07:07:13.514Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:07:13.515Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:07:13.515Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:07:13.529Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:07:13.529Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:13.529Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:13.529Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:13.533Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.533Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:07:13.534Z [TRACE] devicemanager/instance.go:359: client.device_mgr: exiting since fingerprinting gracefully shutdown: plugin=nvidia-gpu 2021-09-13T07:07:13.537Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=mock_driver health=healthy description=Healthy 2021-09-13T07:07:13.537Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=raw_exec health=undetected description=disabled 2021-09-13T07:07:13.539Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=exec health=healthy description=Healthy 2021-09-13T07:07:13.539Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=qemu health=undetected description= 2021-09-13T07:07:13.540Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:13.540Z [DEBUG] pluginmanager/group.go:67: client.plugin: waiting on plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:13.540Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=device 2021-09-13T07:07:13.575Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:07:13.592Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=docker health=healthy description=Healthy 2021-09-13T07:07:13.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=5c9d4f47-fed7-880d-434c-39cffb59e035 2021-09-13T07:07:13.802Z [TRACE] consul/consul_testing.go:90: mock_consul: AllocRegistrations: alloc_id=98c57617-e146-ac4f-1664-dbd634bef552 2021-09-13T07:07:13.864Z [DEBUG] drivermanager/instance.go:386: client.driver_mgr: initial driver fingerprint: driver=java health=healthy description=Healthy 2021-09-13T07:07:13.864Z [DEBUG] drivermanager/manager.go:278: client.driver_mgr: detected drivers: drivers="map[healthy:[mock_driver exec docker java] undetected:[raw_exec qemu]]" 2021-09-13T07:07:13.865Z [DEBUG] pluginmanager/group.go:71: client.plugin: finished plugin manager initial fingerprint: plugin-type=driver 2021-09-13T07:07:13.865Z [INFO] client/client.go:567: client: started client: node_id=f3d28adc-c82d-38bf-dd7e-a9c40fac178f 2021-09-13T07:07:13.865Z [DEBUG] client/fingerprint_manager.go:80: client.fingerprint_mgr: built-in fingerprints: fingerprinters=[arch, bridge, cgroup, cni, consul, cpu, host, memory, network, nomad, signal, storage, vault, env_gce, env_azure, env_aws] 2021-09-13T07:07:13.865Z [DEBUG] fingerprint/cpu.go:66: client.fingerprint_mgr.cpu: detected core count: cores=2 2021-09-13T07:07:13.865Z [DEBUG] fingerprint/cpu.go:80: client.fingerprint_mgr.cpu: detected reservable cores: cpuset=[0, 1] 2021-09-13T07:07:13.870Z [INFO] fingerprint/cpu.go:95: client.fingerprint_mgr.cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override 2021-09-13T07:07:13.866Z [WARN] servers/manager.go:238: client.server_mgr: no servers available 2021-09-13T07:07:13.871Z [DEBUG] client/client.go:1781: client: registration waiting on servers 2021-09-13T07:07:13.871Z [TRACE] consul/catalog_testing.go:61: mock_consul: Datacenters(): dcs=[dc1] error=nil 2021-09-13T07:07:13.871Z [DEBUG] client/client.go:2717: client.consul: bootstrap contacting Consul DCs: consul_dcs=[dc1] 2021-09-13T07:07:13.871Z [TRACE] consul/catalog_testing.go:66: mock_consul: Services(): service=nomad tag=rpc query_options="&{ dc1 true false false 0s 0s 0 2s _agent map[] 0 false false }" 2021-09-13T07:07:13.871Z [ERROR] client/client.go:2679: client: error discovering nomad servers: error="no Nomad Servers advertising service "nomad" in Consul datacenters: ["dc1"]" 2021-09-13T07:07:13.871Z [DEBUG] client/fingerprint_manager.go:155: client.fingerprint_mgr: detected fingerprints: node_attrs=[arch, cpu] 2021-09-13T07:07:13.871Z [DEBUG] client/fingerprint_manager.go:104: client.fingerprint_mgr: fingerprint modules skipped due to allow/denylist: skipped_fingerprinters=[bridge, cgroup, cni, consul, host, memory, network, nomad, signal, storage, vault, env_gce, env_azure, env_aws] === CONT TestFingerprintManager_Run_Combination fingerprint_manager_test.go:119: Error Trace: fingerprint_manager_test.go:119 Error: Should not be: "" Test: TestFingerprintManager_Run_Combination 2021-09-13T07:07:14.018Z [INFO] client/client.go:761: client: shutting down 2021-09-13T07:07:14.018Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=device 2021-09-13T07:07:14.018Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=device 2021-09-13T07:07:14.018Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=driver 2021-09-13T07:07:14.018Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=driver 2021-09-13T07:07:14.018Z [INFO] pluginmanager/group.go:92: client.plugin: shutting down plugin manager: plugin-type=csi 2021-09-13T07:07:14.018Z [INFO] pluginmanager/group.go:94: client.plugin: plugin manager finished: plugin-type=csi 2021-09-13T07:07:14.018Z [WARN] servers/manager.go:238: client.server_mgr: no servers available 2021-09-13T07:07:14.019Z [DEBUG] client/client.go:1781: client: registration waiting on servers 2021-09-13T07:07:14.019Z [WARN] servers/manager.go:238: client.server_mgr: no servers available 2021-09-13T07:07:14.019Z [TRACE] eventer/eventer.go:68: client.driver_mgr.raw_exec: task event loop shutdown: driver=raw_exec 2021-09-13T07:07:14.019Z [TRACE] eventer/eventer.go:68: client.driver_mgr.exec: task event loop shutdown: driver=exec 2021-09-13T07:07:14.019Z [TRACE] eventer/eventer.go:68: client.driver_mgr.qemu: task event loop shutdown: driver=qemu 2021-09-13T07:07:14.019Z [TRACE] eventer/eventer.go:68: client.driver_mgr.java: task event loop shutdown: driver=java 2021-09-13T07:07:14.019Z [TRACE] eventer/eventer.go:68: client.driver_mgr.docker: task event loop shutdown: driver=docker 2021-09-13T07:07:14.019Z [TRACE] eventer/eventer.go:68: client.driver_mgr.mock_driver: task event loop shutdown: driver=mock_driver 2021-09-13T07:07:14.019Z [DEBUG] servers/manager.go:183: client.server_mgr: shutting down === FAIL: client/allocrunner/taskrunner TestTaskRunner_Download_ChrootExec (79.10s) === PAUSE TestTaskRunner_Download_ChrootExec === CONT TestTaskRunner_Download_ChrootExec 2021-09-13T07:04:48.166Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.166Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.166Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.166Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.166Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.167Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.167Z [TRACE] eventer/eventer.go:68: plugin_loader.docker: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.168Z [DEBUG] taskrunner/task_runner.go:509: task_runner: lifecycle start condition has been met, proceeding: task=web 2021-09-13T07:04:48.168Z [TRACE] taskrunner/task_runner_hooks.go:186: task_runner: running prestart hooks: task=web start="2021-09-13 07:04:48.168719211 +0000 UTC m=+9.051296985" 2021-09-13T07:04:48.168Z [TRACE] taskrunner/task_runner_hooks.go:234: task_runner: running prestart hook: task=web name=validate start="2021-09-13 07:04:48.168777672 +0000 UTC m=+9.051355454" 2021-09-13T07:04:48.168Z [TRACE] taskrunner/task_runner_hooks.go:284: task_runner: finished prestart hook: task=web name=validate end="2021-09-13 07:04:48.168800646 +0000 UTC m=+9.051378420" duration=22.966µs 2021-09-13T07:04:48.168Z [TRACE] taskrunner/task_runner_hooks.go:234: task_runner: running prestart hook: task=web name=task_dir start="2021-09-13 07:04:48.168838816 +0000 UTC m=+9.051416590" 2021-09-13T07:04:48.169Z [TRACE] eventer/eventer.go:68: plugin_loader.qemu: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.169Z [TRACE] eventer/eventer.go:68: plugin_loader.mock_driver: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.169Z [TRACE] eventer/eventer.go:68: plugin_loader.raw_exec: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.169Z [TRACE] eventer/eventer.go:68: plugin_loader.exec: task event loop shutdown: plugin_dir= 2021-09-13T07:04:48.169Z [TRACE] eventer/eventer.go:68: plugin_loader.java: task event loop shutdown: plugin_dir= 2021-09-13T07:04:50.138Z [WARN] go-plugin@v1.4.0/client.go:1026: task_runner.task_hook.logmon.taskrunner.test: timed out waiting for read-side of process output pipe to close: task=web @module=logmon timestamp=2021-09-13T07:04:50.138Z 2021-09-13T07:04:50.138Z [WARN] go-plugin@v1.4.0/client.go:1026: task_runner.task_hook.logmon.taskrunner.test: timed out waiting for read-side of process output pipe to close: task=web @module=logmon timestamp=2021-09-13T07:04:50.138Z 2021-09-13T07:04:50.139Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: task_runner.task_hook.logmon.stdio: received EOF, stopping recv loop: task=web err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:04:50.141Z [DEBUG] go-plugin@v1.4.0/client.go:635: task_runner.task_hook.logmon: plugin process exited: task=web path=/tmp/go-build124451399/b957/taskrunner.test pid=20293 2021-09-13T07:04:50.141Z [DEBUG] go-plugin@v1.4.0/client.go:454: task_runner.task_hook.logmon: plugin exited: task=web 2021-09-13T07:04:50.141Z [TRACE] taskrunner/task_runner_hooks.go:434: task_runner: finished stop hook: task=web name=logmon end="2021-09-13 07:04:50.141165473 +0000 UTC m=+11.023743247" duration=4.00466133s 2021-09-13T07:04:50.141Z [TRACE] taskrunner/task_runner_hooks.go:412: task_runner: running stop hook: task=web name=consul_services start="2021-09-13 07:04:50.14119451 +0000 UTC m=+11.023772276" 2021-09-13T07:04:50.141Z [TRACE] taskrunner/task_runner_hooks.go:434: task_runner: finished stop hook: task=web name=consul_services end="2021-09-13 07:04:50.141211634 +0000 UTC m=+11.023789408" duration=17.132µs 2021-09-13T07:04:50.141Z [TRACE] taskrunner/task_runner_hooks.go:412: task_runner: running stop hook: task=web name=script_checks start="2021-09-13 07:04:50.141227577 +0000 UTC m=+11.023805351" 2021-09-13T07:04:50.141Z [TRACE] taskrunner/task_runner_hooks.go:434: task_runner: finished stop hook: task=web name=script_checks end="2021-09-13 07:04:50.141245989 +0000 UTC m=+11.023823763" duration=18.412µs 2021-09-13T07:04:50.141Z [TRACE] taskrunner/task_runner_hooks.go:397: task_runner: finished stop hooks: task=web end="2021-09-13 07:04:50.141261242 +0000 UTC m=+11.023839008" duration=4.004774921s 2021-09-13T07:04:50.141Z [DEBUG] taskrunner/task_runner.go:637: task_runner: task run loop exiting: task=web 2021-09-13T07:04:50.142Z [TRACE] taskrunner/lifecycle.go:72: task_runner: Kill requested: task=web event_type=cleanup event_reason= === CONT TestTaskRunner_Download_ChrootExec task_runner_test.go:1642: Error Trace: task_runner_test.go:1642 Error: timed out waiting for task runner to exit Test: TestTaskRunner_Download_ChrootExec 2021-09-13T07:05:03.169Z [TRACE] taskrunner/lifecycle.go:72: task_runner: Kill requested: task=web event_type=cleanup event_reason= 2021/09/13 07:05:03 [TRACE] (view) vault.read(foo/secret) starting fetch 2021/09/13 07:05:03 [TRACE] vault.read(foo/secret): GET /v1/foo/secret 2021-09-13T07:05:04.822Z [WARN] go-plugin@v1.4.0/client.go:1026: task_runner.task_hook.logmon.taskrunner.test: timed out waiting for read-side of process output pipe to close: task=connect-proxy-testconnect @module=logmon timestamp=2021-09-13T07:05:04.819Z 2021-09-13T07:05:04.822Z [WARN] go-plugin@v1.4.0/client.go:1026: task_runner.task_hook.logmon.taskrunner.test: timed out waiting for read-side of process output pipe to close: task=connect-proxy-testconnect @module=logmon timestamp=2021-09-13T07:05:04.819Z 2021-09-13T07:05:04.825Z [DEBUG] go-plugin@v1.4.0/client.go:635: task_runner.task_hook.logmon: plugin process exited: task=connect-proxy-testconnect path=/tmp/go-build124451399/b957/taskrunner.test pid=21117 2021-09-13T07:05:04.825Z [DEBUG] go-plugin@v1.4.0/client.go:454: task_runner.task_hook.logmon: plugin exited: task=connect-proxy-testconnect 2021-09-13T07:05:04.825Z [TRACE] taskrunner/task_runner_hooks.go:434: task_runner: finished stop hook: task=connect-proxy-testconnect name=logmon end="2021-09-13 07:05:04.825395393 +0000 UTC m=+25.707973167" duration=4.006821736s 2021-09-13T07:05:04.825Z [TRACE] taskrunner/task_runner_hooks.go:412: task_runner: running stop hook: task=connect-proxy-testconnect name=consul_services start="2021-09-13 07:05:04.825443318 +0000 UTC m=+25.708021109" 2021-09-13T07:05:04.825Z [TRACE] taskrunner/task_runner_hooks.go:434: task_runner: finished stop hook: task=connect-proxy-testconnect name=consul_services end="2021-09-13 07:05:04.825461739 +0000 UTC m=+25.708039513" duration=18.404µs 2021-09-13T07:05:04.825Z [TRACE] taskrunner/task_runner_hooks.go:412: task_runner: running stop hook: task=connect-proxy-testconnect name=script_checks start="2021-09-13 07:05:04.825482924 +0000 UTC m=+25.708060698" 2021-09-13T07:05:04.825Z [TRACE] taskrunner/task_runner_hooks.go:434: task_runner: finished stop hook: task=connect-proxy-testconnect name=script_checks end="2021-09-13 07:05:04.825504257 +0000 UTC m=+25.708082032" duration=21.334µs 2021-09-13T07:05:04.825Z [TRACE] taskrunner/task_runner_hooks.go:397: task_runner: finished stop hooks: task=connect-proxy-testconnect end="2021-09-13 07:05:04.825520265 +0000 UTC m=+25.708098040" duration=4.00696466s 2021-09-13T07:05:04.825Z [DEBUG] taskrunner/task_runner.go:637: task_runner: task run loop exiting: task=connect-proxy-testconnect 2021-09-13T07:05:04.825Z [TRACE] taskrunner/lifecycle.go:72: task_runner: Kill requested: task=connect-proxy-testconnect event_type=cleanup event_reason= 2021-09-13T07:05:04.832Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: task_runner.task_hook.logmon.stdio: received EOF, stopping recv loop: task=connect-proxy-testconnect err="rpc error: code = Canceled desc = context canceled" === FAIL: client/fingerprint TestCPUFingerprint (0.00s) 2021-09-13T07:06:32.325Z [DEBUG] fingerprint/cpu.go:66: cpu: detected core count: cores=2 2021-09-13T07:06:32.325Z [DEBUG] fingerprint/cpu.go:80: cpu: detected reservable cores: cpuset=[] 2021-09-13T07:06:32.325Z [INFO] fingerprint/cpu.go:95: cpu: fallback to default cpu total compute, set client config option cpu_total_compute to override cpu_test.go:38: Missing Model Name === FAIL: client/lib/cgutil TestCpusetManager_AddAlloc (0.00s) 2021-09-13T07:06:36.038Z [INFO] cgutil/cpuset_manager_linux.go:165: initialized cpuset cgroup manager: parent=/gotest-7f7095cf cpuset=0-1 cpuset_manager_linux_test.go:78: Error Trace: cpuset_manager_linux_test.go:78 Error: Should be empty, but was [0 1] Test: TestCpusetManager_AddAlloc === FAIL: client/lib/cgutil TestCpusetManager_RemoveAlloc (0.00s) 2021-09-13T07:06:36.039Z [INFO] cgutil/cpuset_manager_linux.go:165: initialized cpuset cgroup manager: parent=/gotest-4f2bd0f6 cpuset=0-1 cpuset_manager_linux_test.go:132: Error Trace: cpuset_manager_linux_test.go:132 Error: Should be false Test: TestCpusetManager_RemoveAlloc === FAIL: client/stats TestHostStats_CPU (1.05s) cpu_test.go:45: Error Trace: cpu_test.go:45 Error: Should not be zero, but was 0 Test: TestHostStats_CPU === FAIL: command TestCommand_Metrics_Cases/bad_address_-_fake_host (0.00s) metrics_test.go:87: Error Trace: metrics_test.go:87 Error: "Error getting metrics: Get \"http://foo/v1/metrics\": dial tcp: lookup foo: Temporary failure in name resolution\n" does not contain "no such host" Test: TestCommand_Metrics_Cases/bad_address_-_fake_host Messages: expected error "no such host", got "Error getting metrics: Get "http://foo/v1/metrics": dial tcp: lookup foo: Temporary failure in name resolution " 127.0.0.1:9111 2021-09-13T07:08:07.720Z [DEBUG] http: shutting down http server 127.0.0.1:9111 2021-09-13T07:08:07.721Z [INFO] agent: requesting shutdown 127.0.0.1:9111 2021-09-13T07:08:07.721Z [INFO] nomad: shutting down server 127.0.0.1:9111 2021-09-13T07:08:07.721Z [WARN] nomad: serf: Shutdown without a Leave 127.0.0.1:9111 2021-09-13T07:08:07.721Z [DEBUG] nomad: shutting down leader loop 127.0.0.1:9111 2021-09-13T07:08:07.721Z [INFO] agent: shutdown complete --- FAIL: TestCommand_Metrics_Cases/bad_address_-_fake_host (0.00s) === FAIL: command TestCommand_Metrics_Cases (0.27s) === PAUSE TestCommand_Metrics_Cases === CONT TestCommand_Metrics_Cases 127.0.0.1:9114 2021-09-13T07:08:07.449Z [INFO] nomad: cluster leadership lost 127.0.0.1:9111 2021-09-13T07:08:07.449Z [INFO] agent: detected plugin: name=raw_exec type=driver plugin_version=0.1.0 127.0.0.1:9111 2021-09-13T07:08:07.449Z [INFO] agent: detected plugin: name=exec type=driver plugin_version=0.1.0 127.0.0.1:9111 2021-09-13T07:08:07.449Z [INFO] agent: detected plugin: name=qemu type=driver plugin_version=0.1.0 127.0.0.1:9111 2021-09-13T07:08:07.450Z [INFO] agent: detected plugin: name=java type=driver plugin_version=0.1.0 127.0.0.1:9111 2021-09-13T07:08:07.450Z [INFO] agent: detected plugin: name=docker type=driver plugin_version=0.1.0 127.0.0.1:9111 2021-09-13T07:08:07.450Z [INFO] agent: detected plugin: name=mock_driver type=driver plugin_version=0.1.0 127.0.0.1:9111 2021-09-13T07:08:07.450Z [INFO] agent: detected plugin: name=nvidia-gpu type=device plugin_version=0.1.0 127.0.0.1:9111 2021-09-13T07:08:07.451Z [INFO] nomad.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:127.0.0.1:9111 Address:127.0.0.1:9111}]" 127.0.0.1:9111 2021-09-13T07:08:07.451Z [INFO] nomad.raft: entering follower state: follower="Node at 127.0.0.1:9111 [Follower]" leader= 127.0.0.1:9111 2021-09-13T07:08:07.457Z [INFO] nomad: serf: EventMemberJoin: TestCommand_Metrics_Cases.global 127.0.0.1 127.0.0.1:9111 2021-09-13T07:08:07.457Z [INFO] nomad: starting scheduling worker(s): num_workers=2 schedulers=[batch, system, sysbatch, service, _core] 127.0.0.1:9111 2021-09-13T07:08:07.458Z [INFO] nomad: adding server: server="TestCommand_Metrics_Cases.global (Addr: 127.0.0.1:9111) (DC: dc1)" 127.0.0.1:9111 2021-09-13T07:08:07.510Z [WARN] nomad.raft: heartbeat timeout reached, starting election: last-leader= 127.0.0.1:9111 2021-09-13T07:08:07.510Z [INFO] nomad.raft: entering candidate state: node="Node at 127.0.0.1:9111 [Candidate]" term=2 127.0.0.1:9111 2021-09-13T07:08:07.510Z [DEBUG] nomad.raft: votes: needed=1 127.0.0.1:9111 2021-09-13T07:08:07.510Z [DEBUG] nomad.raft: vote granted: from=127.0.0.1:9111 term=2 tally=1 127.0.0.1:9111 2021-09-13T07:08:07.510Z [INFO] nomad.raft: election won: tally=1 127.0.0.1:9111 2021-09-13T07:08:07.510Z [INFO] nomad.raft: entering leader state: leader="Node at 127.0.0.1:9111 [Leader]" 127.0.0.1:9111 2021-09-13T07:08:07.510Z [INFO] nomad: cluster leadership acquired 127.0.0.1:9111 2021-09-13T07:08:07.511Z [INFO] nomad.core: established cluster id: cluster_id=575e96fc-7372-ec00-1d97-23dc89d38b81 create_time=1631516887511088030 === FAIL: drivers/docker TestDockerDriver_EnableImageGC (1.26s) docker.go:36: Successfully connected to docker daemon running version 20.10.7 [INFO] freeport: detected ephemeral port range of [32768, 60999] docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:08:43.601Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:08:43.604Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2361143687/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:08:43.605Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2361143687/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:08:43.831Z [DEBUG] docker/driver.go:618: docker: loading image from disk: archive=/tmp/nomad_driver_harness-2361143687/redis-demo/local/busybox.tar 2021-09-13T07:08:43.933Z [DEBUG] docker/coordinator.go:250: docker: image reference count incremented: image_name=busybox:1.29.3 image_id=sha256:59788edf1f3e78cd0ebe6ce1446e9d10788225db3dedcfd1a59f764bad2b2690 references=1 2021-09-13T07:08:43.933Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-2361143687/alloc:/alloc, /tmp/nomad_driver_harness-2361143687/redis-demo/local:/local, /tmp/nomad_driver_harness-2361143687/redis-demo/secrets:/secrets] 2021-09-13T07:08:43.933Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:08:43.933Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:08:43.933Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-2361143687/alloc:/alloc", "/tmp/nomad_driver_harness-2361143687/redis-demo/local:/local", "/tmp/nomad_driver_harness-2361143687/redis-demo/secrets:/secrets"}" 2021-09-13T07:08:43.933Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:08:43.933Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9101 label=main 2021-09-13T07:08:43.933Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9101 label=main 2021-09-13T07:08:43.933Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9102 label=REDIS 2021-09-13T07:08:43.933Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9102 label=REDIS 2021-09-13T07:08:43.933Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="echo hello" 2021-09-13T07:08:43.933Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:bb7e1924-90ab-a3e7-cdd2-2f95a9b90c44] 2021-09-13T07:08:43.933Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-bb7e1924-90ab-a3e7-cdd2-2f95a9b90c44 2021-09-13T07:08:44.074Z [INFO] docker/driver.go:306: docker: created container: container_id=0662dc0d4a8f2177869c5b7806709a3039d277fecfa9043a7228750939f168ca 2021-09-13T07:08:44.810Z [INFO] docker/driver.go:347: docker: started container: container_id=0662dc0d4a8f2177869c5b7806709a3039d277fecfa9043a7228750939f168ca 2021-09-13T07:08:44.810Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:08:44.810Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=40702 2021-09-13T07:08:44.810Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:08:44.822Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin2694875247 network=unix timestamp=2021-09-13T07:08:44.821Z 2021-09-13T07:08:44.822Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:08:44.823Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:08:44.824Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:08:44.823Z driver_test.go:1906: err: &{1 0 false Docker container exited with non-zero exit code: 1} 2021-09-13T07:08:44.827Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:08:44.834Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:08:44.834Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=40702 2021-09-13T07:08:44.834Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === FAIL: drivers/docker TestDockerDriver_DisableImageGC (1.05s) docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:08:44.863Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:08:44.869Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3101609248/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:08:44.869Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3101609248/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:08:44.894Z [DEBUG] docker/driver.go:618: docker: loading image from disk: archive=/tmp/nomad_driver_harness-3101609248/redis-demo/local/busybox.tar 2021-09-13T07:08:44.961Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3101609248/alloc:/alloc, /tmp/nomad_driver_harness-3101609248/redis-demo/local:/local, /tmp/nomad_driver_harness-3101609248/redis-demo/secrets:/secrets] 2021-09-13T07:08:44.961Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:08:44.961Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:08:44.961Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3101609248/alloc:/alloc", "/tmp/nomad_driver_harness-3101609248/redis-demo/local:/local", "/tmp/nomad_driver_harness-3101609248/redis-demo/secrets:/secrets"}" 2021-09-13T07:08:44.962Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:08:44.962Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9103 label=main 2021-09-13T07:08:44.962Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9103 label=main 2021-09-13T07:08:44.962Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9104 label=REDIS 2021-09-13T07:08:44.962Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9104 label=REDIS 2021-09-13T07:08:44.962Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="echo hello" 2021-09-13T07:08:44.962Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:37d24eec-312d-4d92-eef4-f22ce7b26db1] 2021-09-13T07:08:44.962Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-37d24eec-312d-4d92-eef4-f22ce7b26db1 2021-09-13T07:08:45.012Z [INFO] docker/driver.go:306: docker: created container: container_id=bd0abfa303b8cec9fd9b9adcf4d65db1ca6014e36e46083c3cb4de41e5d53043 2021-09-13T07:08:45.845Z [INFO] docker/driver.go:347: docker: started container: container_id=bd0abfa303b8cec9fd9b9adcf4d65db1ca6014e36e46083c3cb4de41e5d53043 2021-09-13T07:08:45.845Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:08:45.849Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=40945 2021-09-13T07:08:45.849Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:08:45.870Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1831685023 network=unix timestamp=2021-09-13T07:08:45.870Z 2021-09-13T07:08:45.870Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:08:45.871Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:08:45.872Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:08:45.872Z driver_test.go:1972: err: &{1 0 false Docker container exited with non-zero exit code: 1} 2021-09-13T07:08:45.881Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" === FAIL: drivers/docker TestDockerDriver_MissingContainer_Cleanup (1.12s) 2021-09-13T07:08:45.882Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:08:45.891Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=40945 2021-09-13T07:08:45.891Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:08:45.915Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:08:45.922Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-821556404/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:08:45.922Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-821556404/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:08:45.947Z [DEBUG] docker/driver.go:618: docker: loading image from disk: archive=/tmp/nomad_driver_harness-821556404/redis-demo/local/busybox.tar 2021-09-13T07:08:46.019Z [DEBUG] docker/coordinator.go:250: docker: image reference count incremented: image_name=busybox:1.29.3 image_id=sha256:59788edf1f3e78cd0ebe6ce1446e9d10788225db3dedcfd1a59f764bad2b2690 references=1 2021-09-13T07:08:46.019Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-821556404/alloc:/alloc, /tmp/nomad_driver_harness-821556404/redis-demo/local:/local, /tmp/nomad_driver_harness-821556404/redis-demo/secrets:/secrets] 2021-09-13T07:08:46.019Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:08:46.019Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:08:46.019Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-821556404/alloc:/alloc", "/tmp/nomad_driver_harness-821556404/redis-demo/local:/local", "/tmp/nomad_driver_harness-821556404/redis-demo/secrets:/secrets"}" 2021-09-13T07:08:46.020Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:08:46.020Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9105 label=main 2021-09-13T07:08:46.020Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9105 label=main 2021-09-13T07:08:46.020Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9106 label=REDIS 2021-09-13T07:08:46.020Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9106 label=REDIS 2021-09-13T07:08:46.020Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="echo hello" 2021-09-13T07:08:46.020Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:5e5f8174-e84e-f970-3946-d32e28c4a6bc] 2021-09-13T07:08:46.020Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-5e5f8174-e84e-f970-3946-d32e28c4a6bc 2021-09-13T07:08:46.077Z [INFO] docker/driver.go:306: docker: created container: container_id=588985ac3703ba29740c24f6dd1790232a67ddb5d3dca0c166ef5850abd3942e 2021-09-13T07:08:46.978Z [INFO] docker/driver.go:347: docker: started container: container_id=588985ac3703ba29740c24f6dd1790232a67ddb5d3dca0c166ef5850abd3942e 2021-09-13T07:08:46.979Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:08:46.979Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=41166 2021-09-13T07:08:46.979Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:08:46.991Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1438723484 network=unix timestamp=2021-09-13T07:08:46.991Z 2021-09-13T07:08:46.991Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:08:46.992Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:08:46.992Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:08:46.992Z driver_test.go:2034: err: &{1 0 false Docker container exited with non-zero exit code: 1} 2021-09-13T07:08:47.004Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=41166 2021-09-13T07:08:47.004Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:08:47.004Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" === FAIL: drivers/docker TestDockerDriver_StopSignal/stopsignal-only (5.62s) 2021-09-13T07:08:47.022Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:08:47.029Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3413642397/alloc/logs/.stopsignal-only.stdout.fifo 2021-09-13T07:08:47.029Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3413642397/alloc/logs/.stopsignal-only.stderr.fifo docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:08:47.057Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=stopsignal-only volumes=[/tmp/nomad_driver_harness-3413642397/alloc:/alloc, /tmp/nomad_driver_harness-3413642397/stopsignal-only/local:/local, /tmp/nomad_driver_harness-3413642397/stopsignal-only/secrets:/secrets] 2021-09-13T07:08:47.057Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=stopsignal-only 2021-09-13T07:08:47.057Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=stopsignal-only memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:08:47.057Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=stopsignal-only binds="[]string{"/tmp/nomad_driver_harness-3413642397/alloc:/alloc", "/tmp/nomad_driver_harness-3413642397/stopsignal-only/local:/local", "/tmp/nomad_driver_harness-3413642397/stopsignal-only/secrets:/secrets"}" 2021-09-13T07:08:47.057Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=stopsignal-only 2021-09-13T07:08:47.057Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=stopsignal-only command="sleep 9901" 2021-09-13T07:08:47.057Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=stopsignal-only labels=map[com.hashicorp.nomad.alloc_id:547e2860-cbdc-be52-d39a-415394740cdf] 2021-09-13T07:08:47.057Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=stopsignal-only container_name=stopsignal-only-547e2860-cbdc-be52-d39a-415394740cdf 2021-09-13T07:08:47.107Z [INFO] docker/driver.go:306: docker: created container: container_id=c7bcc9478475b2919b9182295fb6439b1062a66c8eb73d36668aa19f1494f4ec 2021-09-13T07:08:47.613Z [INFO] docker/driver.go:347: docker: started container: container_id=c7bcc9478475b2919b9182295fb6439b1062a66c8eb73d36668aa19f1494f4ec 2021-09-13T07:08:47.614Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:08:47.614Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=41291 2021-09-13T07:08:47.614Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:08:47.626Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin3522748057 network=unix timestamp=2021-09-13T07:08:47.626Z 2021-09-13T07:08:47.626Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:08:47.627Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:08:47.628Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:08:47.628Z 2021-09-13T07:08:47.631Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:08:47.632Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=41291 2021-09-13T07:08:47.633Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited driver_test.go:3033: Error Trace: driver_test.go:3033 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_StopSignal/stopsignal-only --- FAIL: TestDockerDriver_StopSignal/stopsignal-only (5.62s) === FAIL: drivers/docker TestDockerDriver_StopSignal/stopsignal-killsignal (5.65s) 2021-09-13T07:08:52.639Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:08:52.641Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:08:52.645Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2246509380/alloc/logs/.stopsignal-killsignal.stdout.fifo 2021-09-13T07:08:52.645Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2246509380/alloc/logs/.stopsignal-killsignal.stderr.fifo docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:08:52.673Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=stopsignal-killsignal volumes=[/tmp/nomad_driver_harness-2246509380/alloc:/alloc, /tmp/nomad_driver_harness-2246509380/stopsignal-killsignal/local:/local, /tmp/nomad_driver_harness-2246509380/stopsignal-killsignal/secrets:/secrets] 2021-09-13T07:08:52.673Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=stopsignal-killsignal 2021-09-13T07:08:52.673Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=stopsignal-killsignal memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:08:52.673Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=stopsignal-killsignal binds="[]string{"/tmp/nomad_driver_harness-2246509380/alloc:/alloc", "/tmp/nomad_driver_harness-2246509380/stopsignal-killsignal/local:/local", "/tmp/nomad_driver_harness-2246509380/stopsignal-killsignal/secrets:/secrets"}" 2021-09-13T07:08:52.673Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=stopsignal-killsignal 2021-09-13T07:08:52.673Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=stopsignal-killsignal command="sleep 9901" 2021-09-13T07:08:52.673Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=stopsignal-killsignal labels=map[com.hashicorp.nomad.alloc_id:57183d72-17ee-761a-b5c7-ffafaadd5bb7] 2021-09-13T07:08:52.673Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=stopsignal-killsignal container_name=stopsignal-killsignal-57183d72-17ee-761a-b5c7-ffafaadd5bb7 2021-09-13T07:08:52.721Z [INFO] docker/driver.go:306: docker: created container: container_id=540d0247e0eadfbdb8f5af80ffe6e53b53bee66521d0fe552b5b7746ad3efe1f 2021-09-13T07:08:53.268Z [INFO] docker/driver.go:347: docker: started container: container_id=540d0247e0eadfbdb8f5af80ffe6e53b53bee66521d0fe552b5b7746ad3efe1f 2021-09-13T07:08:53.269Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:08:53.269Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=41417 2021-09-13T07:08:53.269Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:08:53.282Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:08:53.282Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1267278077 network=unix timestamp=2021-09-13T07:08:53.282Z 2021-09-13T07:08:53.283Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:08:53.284Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:08:53.284Z 2021-09-13T07:08:53.288Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:08:53.290Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=41417 2021-09-13T07:08:53.290Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited driver_test.go:3033: Error Trace: driver_test.go:3033 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_StopSignal/stopsignal-killsignal --- FAIL: TestDockerDriver_StopSignal/stopsignal-killsignal (5.65s) === FAIL: drivers/docker TestDockerDriver_StopSignal/killsignal-only (5.85s) 2021-09-13T07:08:58.294Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:08:58.298Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:08:58.301Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1808286833/alloc/logs/.killsignal-only.stdout.fifo 2021-09-13T07:08:58.301Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1808286833/alloc/logs/.killsignal-only.stderr.fifo docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:08:58.321Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=killsignal-only volumes=[/tmp/nomad_driver_harness-1808286833/alloc:/alloc, /tmp/nomad_driver_harness-1808286833/killsignal-only/local:/local, /tmp/nomad_driver_harness-1808286833/killsignal-only/secrets:/secrets] 2021-09-13T07:08:58.321Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=killsignal-only 2021-09-13T07:08:58.321Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=killsignal-only memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:08:58.321Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=killsignal-only binds="[]string{"/tmp/nomad_driver_harness-1808286833/alloc:/alloc", "/tmp/nomad_driver_harness-1808286833/killsignal-only/local:/local", "/tmp/nomad_driver_harness-1808286833/killsignal-only/secrets:/secrets"}" 2021-09-13T07:08:58.321Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=killsignal-only 2021-09-13T07:08:58.321Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=killsignal-only command="sleep 9901" 2021-09-13T07:08:58.321Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=killsignal-only labels=map[com.hashicorp.nomad.alloc_id:63f2c9cd-fb86-2294-bb9b-b18dfb6c0b2f] 2021-09-13T07:08:58.321Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=killsignal-only container_name=killsignal-only-63f2c9cd-fb86-2294-bb9b-b18dfb6c0b2f 2021-09-13T07:08:58.402Z [INFO] docker/driver.go:306: docker: created container: container_id=713a00eb9560ed5ad239d0e6800ad7d26acfcd05aea2899014e9236e8d5c6256 2021-09-13T07:08:59.019Z [INFO] docker/driver.go:347: docker: started container: container_id=713a00eb9560ed5ad239d0e6800ad7d26acfcd05aea2899014e9236e8d5c6256 2021-09-13T07:08:59.019Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:08:59.019Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=41982 2021-09-13T07:08:59.020Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:08:59.031Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin4135923138 network=unix timestamp=2021-09-13T07:08:59.031Z 2021-09-13T07:08:59.031Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:08:59.032Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:08:59.032Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:08:59.032Z 2021-09-13T07:08:59.039Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:08:59.040Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=41982 2021-09-13T07:08:59.040Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited driver_test.go:3033: Error Trace: driver_test.go:3033 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_StopSignal/killsignal-only --- FAIL: TestDockerDriver_StopSignal/killsignal-only (5.85s) === FAIL: drivers/docker TestDockerDriver_StopSignal/nosignals-default (5.70s) 2021-09-13T07:09:04.141Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:09:04.144Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:09:04.150Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2677406947/alloc/logs/.nosignals-default.stdout.fifo 2021-09-13T07:09:04.150Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2677406947/alloc/logs/.nosignals-default.stderr.fifo docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:09:04.177Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=nosignals-default volumes=[/tmp/nomad_driver_harness-2677406947/alloc:/alloc, /tmp/nomad_driver_harness-2677406947/nosignals-default/local:/local, /tmp/nomad_driver_harness-2677406947/nosignals-default/secrets:/secrets] 2021-09-13T07:09:04.177Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=nosignals-default 2021-09-13T07:09:04.177Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=nosignals-default memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:09:04.177Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=nosignals-default binds="[]string{"/tmp/nomad_driver_harness-2677406947/alloc:/alloc", "/tmp/nomad_driver_harness-2677406947/nosignals-default/local:/local", "/tmp/nomad_driver_harness-2677406947/nosignals-default/secrets:/secrets"}" 2021-09-13T07:09:04.177Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=nosignals-default 2021-09-13T07:09:04.177Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=nosignals-default command="sleep 9901" 2021-09-13T07:09:04.177Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=nosignals-default labels=map[com.hashicorp.nomad.alloc_id:b6892385-32bf-f241-6ada-b4c470d7827f] 2021-09-13T07:09:04.177Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=nosignals-default container_name=nosignals-default-b6892385-32bf-f241-6ada-b4c470d7827f 2021-09-13T07:09:04.233Z [INFO] docker/driver.go:306: docker: created container: container_id=24862a6b9af7b77a3a40f256638b63595cece75492cbd1298955be7d30b78811 2021-09-13T07:09:04.809Z [INFO] docker/driver.go:347: docker: started container: container_id=24862a6b9af7b77a3a40f256638b63595cece75492cbd1298955be7d30b78811 2021-09-13T07:09:04.809Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:09:04.809Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=42100 2021-09-13T07:09:04.809Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:09:04.821Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1923997986 network=unix timestamp=2021-09-13T07:09:04.821Z 2021-09-13T07:09:04.821Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:09:04.822Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:09:04.823Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:09:04.823Z 2021-09-13T07:09:04.837Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:09:04.838Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=42100 2021-09-13T07:09:04.838Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited driver_test.go:3033: Error Trace: driver_test.go:3033 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_StopSignal/nosignals-default --- FAIL: TestDockerDriver_StopSignal/nosignals-default (5.70s) === FAIL: drivers/docker TestDockerDriver_StopSignal (22.84s) 2021-09-13T07:08:47.005Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 === FAIL: drivers/docker TestDockerDriver_Sysctl_Ulimit (6.45s) 2021-09-13T07:09:09.844Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:09:09.874Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:09:09.881Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1085100965/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:09:09.881Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1085100965/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:09:09.903Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-1085100965/alloc:/alloc, /tmp/nomad_driver_harness-1085100965/redis-demo/local:/local, /tmp/nomad_driver_harness-1085100965/redis-demo/secrets:/secrets] 2021-09-13T07:09:09.903Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:09:09.903Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:09:09.903Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-1085100965/alloc:/alloc", "/tmp/nomad_driver_harness-1085100965/redis-demo/local:/local", "/tmp/nomad_driver_harness-1085100965/redis-demo/secrets:/secrets"}" 2021-09-13T07:09:09.903Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:09:09.903Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9107 label=main 2021-09-13T07:09:09.903Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9107 label=main 2021-09-13T07:09:09.903Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9108 label=REDIS 2021-09-13T07:09:09.903Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9108 label=REDIS 2021-09-13T07:09:09.903Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:09:09.903Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:404a49d2-8b7f-e8bc-5447-69dfb4baf760] 2021-09-13T07:09:09.903Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-404a49d2-8b7f-e8bc-5447-69dfb4baf760 2021-09-13T07:09:10.129Z [INFO] docker/driver.go:306: docker: created container: container_id=59804b6ac803dda0ecf6bb371630833784bb8c64c716062c63d76d5fc3596c06 2021-09-13T07:09:11.160Z [INFO] docker/driver.go:347: docker: started container: container_id=59804b6ac803dda0ecf6bb371630833784bb8c64c716062c63d76d5fc3596c06 2021-09-13T07:09:11.160Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:09:11.160Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=42381 2021-09-13T07:09:11.160Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:09:11.177Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:09:11.177Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1799217096 network=unix timestamp=2021-09-13T07:09:11.177Z 2021-09-13T07:09:11.178Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:09:11.179Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:09:11.179Z 2021-09-13T07:09:11.188Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:09:11.189Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=42381 2021-09-13T07:09:11.189Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited driver_unix_test.go:189: Error Trace: driver_unix_test.go:189 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Sysctl_Ulimit === FAIL: drivers/docker TestDanglingContainerRemoval (7.48s) 2021-09-13T07:09:31.459Z [DEBUG] docker/coordinator.go:348: docker: cleanup removed downloaded image: image_id=sha256:2fca105641ea33595842008328fd42b62bacc61deee8e15e4e6f983d136b2bb4 docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:09:31.484Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:09:31.487Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4031988080/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:09:31.487Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4031988080/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:09:31.501Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-4031988080/alloc:/alloc, /tmp/nomad_driver_harness-4031988080/redis-demo/local:/local, /tmp/nomad_driver_harness-4031988080/redis-demo/secrets:/secrets] 2021-09-13T07:09:31.502Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:09:31.502Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:09:31.502Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-4031988080/alloc:/alloc", "/tmp/nomad_driver_harness-4031988080/redis-demo/local:/local", "/tmp/nomad_driver_harness-4031988080/redis-demo/secrets:/secrets"}" 2021-09-13T07:09:31.502Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:09:31.502Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9115 label=main 2021-09-13T07:09:31.502Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9115 label=main 2021-09-13T07:09:31.502Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9116 label=REDIS 2021-09-13T07:09:31.502Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9116 label=REDIS 2021-09-13T07:09:31.502Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:09:31.502Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:d2535589-897f-db16-1ab7-65b549105176] 2021-09-13T07:09:31.502Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-d2535589-897f-db16-1ab7-65b549105176 2021-09-13T07:09:32.133Z [INFO] docker/driver.go:306: docker: created container: container_id=dfe82b96a850863581b88623282ccd3f177df9b6a4a3b792aecd2e793e8e8787 2021-09-13T07:09:33.692Z [INFO] docker/driver.go:347: docker: started container: container_id=dfe82b96a850863581b88623282ccd3f177df9b6a4a3b792aecd2e793e8e8787 2021-09-13T07:09:33.692Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:09:33.693Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=42785 2021-09-13T07:09:33.693Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:09:33.711Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:09:33.711Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin963741284 network=unix timestamp=2021-09-13T07:09:33.711Z 2021-09-13T07:09:33.717Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:09:33.718Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:09:33.717Z 2021-09-13T07:09:33.726Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:09:33.730Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=42785 2021-09-13T07:09:33.730Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited reconciler_test.go:66: Error Trace: reconciler_test.go:66 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDanglingContainerRemoval === FAIL: drivers/docker TestDanglingContainerRemoval_Stopped (2.17s) 2021-09-13T07:09:38.922Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 reconciler_test.go:186: Error Trace: reconciler_test.go:186 Error: Received unexpected error: Container not running: 1b107e74e11c79587d5b4bacc0892cbcaabcca681ca883e98a615383748ced2d Test: TestDanglingContainerRemoval_Stopped === FAIL: drivers/docker TestDriver_DockerStatsCollector (1.00s) === PAUSE TestDriver_DockerStatsCollector === CONT TestDriver_DockerStatsCollector 2021-09-13T07:09:41.098Z [DEBUG] docker/coordinator.go:211: docker pull succeeded: image_ref=foo:latest 2021-09-13T07:09:41.098Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=1 2021-09-13T07:09:41.108Z [DEBUG] docker/coordinator.go:211: docker pull succeeded: image_ref=foo:latest 2021-09-13T07:09:41.109Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=2 2021-09-13T07:09:41.109Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=3 2021-09-13T07:09:41.109Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=4 2021-09-13T07:09:41.109Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=5 2021-09-13T07:09:41.109Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=6 2021-09-13T07:09:41.109Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=7 2021-09-13T07:09:41.109Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=8 2021-09-13T07:09:41.109Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=9 2021-09-13T07:09:41.109Z [DEBUG] docker/coordinator.go:250: image reference count incremented: image_name=foo image_id= references=10 === CONT TestDriver_DockerStatsCollector stats_test.go:63: Error Trace: stats_test.go:63 Error: receiving stats should not block here Test: TestDriver_DockerStatsCollector === FAIL: drivers/docker TestDocker_ExecTaskStreaming/basic:_notty:_basic (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:118 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/basic:_notty:_basic --- FAIL: TestDocker_ExecTaskStreaming/basic:_notty:_basic (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming/basic:_notty:_streaming (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:118 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/basic:_notty:_streaming --- FAIL: TestDocker_ExecTaskStreaming/basic:_notty:_streaming (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming/basic:_notty:_stty_check (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:118 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/basic:_notty:_stty_check --- FAIL: TestDocker_ExecTaskStreaming/basic:_notty:_stty_check (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming/basic:_notty:_stdin_passing (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:118 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/basic:_notty:_stdin_passing --- FAIL: TestDocker_ExecTaskStreaming/basic:_notty:_stdin_passing (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming/basic:_tty:_basic (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:118 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/basic:_tty:_basic --- FAIL: TestDocker_ExecTaskStreaming/basic:_tty:_basic (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming/basic:_tty:_streaming (0.00s) 2021-09-13T07:09:43.420Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:118 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/basic:_tty:_streaming --- FAIL: TestDocker_ExecTaskStreaming/basic:_tty:_streaming (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming/basic:_tty:_stty_check (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:118 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/basic:_tty:_stty_check --- FAIL: TestDocker_ExecTaskStreaming/basic:_tty:_stty_check (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming/basic:_tty:_stdin_passing (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:118 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/basic:_tty:_stdin_passing --- FAIL: TestDocker_ExecTaskStreaming/basic:_tty:_stdin_passing (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming/basic:_tty:_children_processes (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:118 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/basic:_tty:_children_processes --- FAIL: TestDocker_ExecTaskStreaming/basic:_tty:_children_processes (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming/isolation (0.00s) 2021-09-13T07:09:43.424Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=43290 2021-09-13T07:09:43.424Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited exec_testing.go:226: Error Trace: exec_testing.go:226 exec_testing.go:160 Error: Received unexpected error: failed to create exec object: API error (409): Container 705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c is not running Test: TestDocker_ExecTaskStreaming/isolation 2021-09-13T07:09:43.431Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1116998776 network=unix timestamp=2021-09-13T07:09:43.431Z 2021-09-13T07:09:43.431Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:09:43.432Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:09:43.432Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:09:43.432Z 2021-09-13T07:09:43.433Z [ERROR] go-plugin@v1.4.0/client.go:1028: docker.docker_logger.docker.test: log streaming ended with terminal error: @module=docker_logger error="open : no such file or directory" timestamp=2021-09-13T07:09:43.433Z --- FAIL: TestDocker_ExecTaskStreaming/isolation (0.00s) === FAIL: drivers/docker TestDocker_ExecTaskStreaming (1.37s) === PAUSE TestDocker_ExecTaskStreaming === CONT TestDocker_ExecTaskStreaming docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:09:42.114Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:09:42.125Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1974951144/alloc/logs/.nc-demo.stdout.fifo 2021-09-13T07:09:42.125Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1974951144/alloc/logs/.nc-demo.stderr.fifo 2021-09-13T07:09:42.150Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=nc-demo volumes=[/tmp/nomad_driver_harness-1974951144/alloc:/alloc, /tmp/nomad_driver_harness-1974951144/nc-demo/local:/local, /tmp/nomad_driver_harness-1974951144/nc-demo/secrets:/secrets] 2021-09-13T07:09:42.150Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=nc-demo 2021-09-13T07:09:42.150Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=nc-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:09:42.150Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=nc-demo binds="[]string{"/tmp/nomad_driver_harness-1974951144/alloc:/alloc", "/tmp/nomad_driver_harness-1974951144/nc-demo/local:/local", "/tmp/nomad_driver_harness-1974951144/nc-demo/secrets:/secrets"}" 2021-09-13T07:09:42.150Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=nc-demo 2021-09-13T07:09:42.150Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=nc-demo command="/bin/sleep 1000" 2021-09-13T07:09:42.150Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=nc-demo labels=map[com.hashicorp.nomad.alloc_id:668fe331-f008-6c74-7f13-5c9b8a2703ed] 2021-09-13T07:09:42.150Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=nc-demo container_name=nc-demo-668fe331-f008-6c74-7f13-5c9b8a2703ed 2021-09-13T07:09:42.249Z [INFO] docker/driver.go:347: docker: started container: container_id=b0b27faad5b8717f3d7a5e0356f40a06aba46b4dc4908ce221f0a180f9dd8f30 2021-09-13T07:09:42.250Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:09:42.250Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=43075 2021-09-13T07:09:42.250Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:09:42.268Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin2976550113 network=unix @module=docker_logger timestamp=2021-09-13T07:09:42.268Z 2021-09-13T07:09:42.274Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:09:42.275Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:09:42.276Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:09:42.276Z 2021-09-13T07:09:42.276Z [ERROR] go-plugin@v1.4.0/client.go:1028: docker.docker_logger.docker.test: log streaming ended with terminal error: @module=docker_logger error="open : no such file or directory" timestamp=2021-09-13T07:09:42.276Z === FAIL: drivers/docker Test_dnsConfig/dns_config (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 dns_testing.go:29 Error: Received unexpected error: failed to create exec object: API error (409): Container b0b27faad5b8717f3d7a5e0356f40a06aba46b4dc4908ce221f0a180f9dd8f30 is not running Test: Test_dnsConfig/dns_config 2021-09-13T07:09:42.282Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=nc-demo volumes=[/tmp/nomad_driver_harness-2027107827/alloc:/alloc, /tmp/nomad_driver_harness-2027107827/nc-demo/local:/local, /tmp/nomad_driver_harness-2027107827/nc-demo/secrets:/secrets] 2021-09-13T07:09:42.282Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=nc-demo 2021-09-13T07:09:42.282Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=nc-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:09:42.282Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=nc-demo binds="[]string{"/tmp/nomad_driver_harness-2027107827/alloc:/alloc", "/tmp/nomad_driver_harness-2027107827/nc-demo/local:/local", "/tmp/nomad_driver_harness-2027107827/nc-demo/secrets:/secrets"}" time="2021-09-13T07:09:42Z" level=info msg="detected 127.0.0.53 nameserver, assuming systemd-resolved, so using resolv.conf: /run/systemd/resolve/resolv.conf" 2021-09-13T07:09:42.283Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=nc-demo 2021-09-13T07:09:42.283Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=nc-demo command="/bin/sleep 1000" 2021-09-13T07:09:42.283Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=nc-demo labels=map[com.hashicorp.nomad.alloc_id:ea69cc9a-d7c4-6f95-4fe0-721760bd713b] 2021-09-13T07:09:42.283Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=nc-demo container_name=nc-demo-ea69cc9a-d7c4-6f95-4fe0-721760bd713b 2021-09-13T07:09:42.289Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:09:42.293Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=43075 2021-09-13T07:09:42.293Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:09:42.296Z [INFO] docker/driver.go:306: docker: created container: container_id=705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c 2021-09-13T07:09:42.438Z [INFO] docker/driver.go:306: docker: created container: container_id=a63a64d6f3e6ed7e24f6a46aad1a5d1de733392d5f928a5a9719f7bca46656fa 2021-09-13T07:09:43.374Z [INFO] docker/driver.go:347: docker: started container: container_id=705fb54fadb6ecd36906d78c0d04d65674ef7181d7719a1730d7a64df232e46c 2021-09-13T07:09:43.374Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:09:43.375Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=43290 2021-09-13T07:09:43.375Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:09:43.397Z [INFO] docker/driver.go:347: docker: started container: container_id=a63a64d6f3e6ed7e24f6a46aad1a5d1de733392d5f928a5a9719f7bca46656fa 2021-09-13T07:09:43.397Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:09:43.400Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=43299 2021-09-13T07:09:43.400Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:09:43.412Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1161837346 network=unix timestamp=2021-09-13T07:09:43.411Z 2021-09-13T07:09:43.412Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:09:43.413Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:09:43.414Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:09:43.414Z --- FAIL: Test_dnsConfig/dns_config (0.00s) === FAIL: drivers/docker Test_dnsConfig/dns_config#01 (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 dns_testing.go:29 Error: Received unexpected error: failed to create exec object: API error (409): Container a63a64d6f3e6ed7e24f6a46aad1a5d1de733392d5f928a5a9719f7bca46656fa is not running Test: Test_dnsConfig/dns_config#01 2021-09-13T07:09:43.446Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:09:43.453Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=nc-demo volumes=[/tmp/nomad_driver_harness-1801508883/alloc:/alloc, /tmp/nomad_driver_harness-1801508883/nc-demo/local:/local, /tmp/nomad_driver_harness-1801508883/nc-demo/secrets:/secrets] 2021-09-13T07:09:43.453Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=nc-demo 2021-09-13T07:09:43.453Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=nc-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:09:43.453Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=nc-demo binds="[]string{"/tmp/nomad_driver_harness-1801508883/alloc:/alloc", "/tmp/nomad_driver_harness-1801508883/nc-demo/local:/local", "/tmp/nomad_driver_harness-1801508883/nc-demo/secrets:/secrets"}" 2021-09-13T07:09:43.453Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=nc-demo 2021-09-13T07:09:43.453Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=nc-demo command="/bin/sleep 1000" 2021-09-13T07:09:43.453Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=nc-demo labels=map[com.hashicorp.nomad.alloc_id:5a42900d-b2b9-48e8-c3f0-1e9d42b39cd4] 2021-09-13T07:09:43.453Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=nc-demo container_name=nc-demo-5a42900d-b2b9-48e8-c3f0-1e9d42b39cd4 --- FAIL: Test_dnsConfig/dns_config#01 (0.00s) === FAIL: drivers/docker Test_dnsConfig/dns_config#02 (0.00s) exec_testing.go:226: Error Trace: exec_testing.go:226 dns_testing.go:29 Error: Received unexpected error: failed to create exec object: API error (409): Container dc890eaf8647c4f7bb5458f3fceb25ccb12af9656eee52bc3160f19a456399b1 is not running Test: Test_dnsConfig/dns_config#02 2021-09-13T07:09:44.940Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin479005426 network=unix @module=docker_logger timestamp=2021-09-13T07:09:44.940Z 2021-09-13T07:09:44.940Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:09:44.941Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:09:44.941Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:09:44.941Z --- FAIL: Test_dnsConfig/dns_config#02 (0.00s) 2021-09-13T07:09:44.957Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown === FAIL: drivers/docker Test_dnsConfig (3.78s) === PAUSE Test_dnsConfig === CONT Test_dnsConfig 2021-09-13T07:09:41.181Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:09:41.205Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:09:41.213Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=nc-demo volumes=[/tmp/nomad_driver_harness-2180649958/alloc:/alloc, /tmp/nomad_driver_harness-2180649958/nc-demo/local:/local, /tmp/nomad_driver_harness-2180649958/nc-demo/secrets:/secrets] 2021-09-13T07:09:41.213Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=nc-demo 2021-09-13T07:09:41.213Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=nc-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:09:41.213Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=nc-demo binds="[]string{"/tmp/nomad_driver_harness-2180649958/alloc:/alloc", "/tmp/nomad_driver_harness-2180649958/nc-demo/local:/local", "/tmp/nomad_driver_harness-2180649958/nc-demo/secrets:/secrets"}" 2021-09-13T07:09:41.213Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=nc-demo 2021-09-13T07:09:41.213Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=nc-demo command="/bin/sleep 1000" 2021-09-13T07:09:41.214Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=nc-demo labels=map[com.hashicorp.nomad.alloc_id:9cb5c5be-d3f9-4011-84ad-9e2cd7e57dcb] 2021-09-13T07:09:41.214Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=nc-demo container_name=nc-demo-9cb5c5be-d3f9-4011-84ad-9e2cd7e57dcb 2021-09-13T07:09:41.435Z [INFO] docker/driver.go:306: docker: created container: container_id=b0b27faad5b8717f3d7a5e0356f40a06aba46b4dc4908ce221f0a180f9dd8f30 === FAIL: drivers/docker TestDockerDriver_NetworkMode_Host (6.36s) === PAUSE TestDockerDriver_NetworkMode_Host === CONT TestDockerDriver_NetworkMode_Host === CONT TestDockerDriver_NetworkMode_Host docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:09:45.105Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins === CONT TestDockerDriver_NetworkMode_Host driver_unix_test.go:131: Error Trace: driver_unix_test.go:131 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_NetworkMode_Host === FAIL: drivers/docker TestDockerDriver_NetworkAliases_Bridge (8.84s) === PAUSE TestDockerDriver_NetworkAliases_Bridge === CONT TestDockerDriver_NetworkAliases_Bridge 2021-09-13T07:09:47.433Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:09:47.595Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:09:47.606Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-12932126/alloc/logs/.busybox.stdout.fifo 2021-09-13T07:09:47.606Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-12932126/alloc/logs/.busybox.stderr.fifo 2021-09-13T07:09:47.633Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=busybox volumes=[/tmp/nomad_driver_harness-12932126/alloc:/alloc, /tmp/nomad_driver_harness-12932126/busybox/local:/local, /tmp/nomad_driver_harness-12932126/busybox/secrets:/secrets] 2021-09-13T07:09:47.633Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=busybox 2021-09-13T07:09:47.633Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=busybox memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:09:47.633Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=busybox binds="[]string{"/tmp/nomad_driver_harness-12932126/alloc:/alloc", "/tmp/nomad_driver_harness-12932126/busybox/local:/local", "/tmp/nomad_driver_harness-12932126/busybox/secrets:/secrets"}" 2021-09-13T07:09:47.633Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=busybox command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:09:47.633Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=busybox labels=map[com.hashicorp.nomad.alloc_id:] 2021-09-13T07:09:47.633Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=busybox container_name=busybox- 2021-09-13T07:09:47.633Z [DEBUG] docker/driver.go:1188: docker: setting container network aliases: task_name=busybox network_mode=foobar network_aliases=foobar 2021-09-13T07:09:49.055Z [INFO] docker/driver.go:306: docker: created container: container_id=bf133a3d20942b0005fcc568a1e48a3f632da0427a04d76ed48126e9600e7971 2021-09-13T07:09:51.101Z [INFO] docker/driver.go:347: docker: started container: container_id=bf133a3d20942b0005fcc568a1e48a3f632da0427a04d76ed48126e9600e7971 2021-09-13T07:09:51.102Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:09:51.102Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=44053 2021-09-13T07:09:51.102Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:09:51.120Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin2670780690 network=unix timestamp=2021-09-13T07:09:51.119Z 2021-09-13T07:09:51.120Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:09:51.120Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:09:51.121Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:09:51.121Z 2021-09-13T07:09:51.129Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:09:51.131Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=44053 2021-09-13T07:09:51.131Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_NetworkAliases_Bridge driver_unix_test.go:92: Error Trace: driver_unix_test.go:92 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_NetworkAliases_Bridge === FAIL: drivers/docker TestDockerDriver_CreationIdempotent (13.16s) === PAUSE TestDockerDriver_CreationIdempotent === CONT TestDockerDriver_CreationIdempotent 2021-09-13T07:09:56.282Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:09:56.332Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:09:56.341Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2027690518/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:09:56.341Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2027690518/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:09:56.366Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-2027690518/alloc:/alloc, /tmp/nomad_driver_harness-2027690518/redis-demo/local:/local, /tmp/nomad_driver_harness-2027690518/redis-demo/secrets:/secrets] 2021-09-13T07:09:56.367Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:09:56.367Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:09:56.367Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-2027690518/alloc:/alloc", "/tmp/nomad_driver_harness-2027690518/redis-demo/local:/local", "/tmp/nomad_driver_harness-2027690518/redis-demo/secrets:/secrets"}" 2021-09-13T07:09:56.367Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:09:56.367Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9179 label=main 2021-09-13T07:09:56.367Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9179 label=main 2021-09-13T07:09:56.367Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9180 label=REDIS 2021-09-13T07:09:56.367Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9180 label=REDIS 2021-09-13T07:09:56.367Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:09:56.367Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:caebbb51-2ea6-e95c-c0e6-189d8f7cc88e] 2021-09-13T07:09:56.367Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-caebbb51-2ea6-e95c-c0e6-189d8f7cc88e 2021-09-13T07:09:56.717Z [DEBUG] docker/driver.go:435: docker: failed to create container: container_name=redis-demo-caebbb51-2ea6-e95c-c0e6-189d8f7cc88e image_name=busybox:1.29.3 image_id=busybox:1.29.3 attempt=1 error="container already exists" 2021-09-13T07:09:56.722Z [TRACE] docker/driver.go:1315: docker: listed container: names=[/redis-demo-caebbb51-2ea6-e95c-c0e6-189d8f7cc88e] 2021-09-13T07:09:56.722Z [TRACE] docker/driver.go:1318: docker: Found container: container_name=/redis-demo-caebbb51-2ea6-e95c-c0e6-189d8f7cc88e container_id=b3e6b45393de95a8efc0c9f10a46e695f411b0451434c495fbc34a7bd45b05a4 2021-09-13T07:09:56.741Z [INFO] docker/driver.go:472: docker: purged container: container_id=b3e6b45393de95a8efc0c9f10a46e695f411b0451434c495fbc34a7bd45b05a4 2021-09-13T07:09:57.375Z [DEBUG] docker/driver.go:505: docker: failed to start container: container_id=2519c83b3ed8c682a0cecf93ffa1707659580991aa5ca24f480fd3f17ae63af0 attempt=3 error="API error (500): unable to find user alice: no matching entries in passwd file" 2021-09-13T07:10:01.761Z [DEBUG] docker/driver.go:505: docker: failed to start container: container_id=2519c83b3ed8c682a0cecf93ffa1707659580991aa5ca24f480fd3f17ae63af0 attempt=4 error="API error (500): unable to find user alice: no matching entries in passwd file" driver_test.go:2823: Error Trace: driver_test.go:2823 wait.go:32 wait.go:18 driver_test.go:2811 Error: Received unexpected error: container is not running but {exited false false false false false false 0 1 2021-09-13 07:09:58.997527518 +0000 UTC 2021-09-13 07:09:58.997037905 +0000 UTC { 0 []}} Test: TestDockerDriver_CreationIdempotent 2021-09-13T07:10:09.422Z [WARN] logmon/logmon.go:292: driver_harness.logmon: timed out waiting for read-side of process output pipe to close 2021-09-13T07:10:09.422Z [WARN] logmon/logmon.go:292: driver_harness.logmon: timed out waiting for read-side of process output pipe to close === FAIL: drivers/docker TestDockerDriver_ReadonlyRootfs (7.99s) === PAUSE TestDockerDriver_ReadonlyRootfs === CONT TestDockerDriver_ReadonlyRootfs 2021-09-13T07:10:09.484Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:10:09.522Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:10:09.530Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-473915006/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:10:09.530Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-473915006/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:10:09.554Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-473915006/alloc:/alloc, /tmp/nomad_driver_harness-473915006/redis-demo/local:/local, /tmp/nomad_driver_harness-473915006/redis-demo/secrets:/secrets] 2021-09-13T07:10:09.554Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:10:09.554Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:10:09.554Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-473915006/alloc:/alloc", "/tmp/nomad_driver_harness-473915006/redis-demo/local:/local", "/tmp/nomad_driver_harness-473915006/redis-demo/secrets:/secrets"}" 2021-09-13T07:10:09.555Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:10:09.555Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9185 label=main 2021-09-13T07:10:09.555Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9185 label=main 2021-09-13T07:10:09.555Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9186 label=REDIS 2021-09-13T07:10:09.555Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9186 label=REDIS 2021-09-13T07:10:09.555Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:10:09.555Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:8100030e-547e-7a31-f789-21ac5e892835] 2021-09-13T07:10:09.555Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-8100030e-547e-7a31-f789-21ac5e892835 2021-09-13T07:10:11.012Z [INFO] docker/driver.go:306: docker: created container: container_id=243f693fc7587c14b2098a51350bf0ba93670f1b99d68f58b37c3ef1ed345e26 2021-09-13T07:10:12.273Z [INFO] docker/driver.go:347: docker: started container: container_id=243f693fc7587c14b2098a51350bf0ba93670f1b99d68f58b37c3ef1ed345e26 2021-09-13T07:10:12.285Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:10:12.285Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=45085 2021-09-13T07:10:12.285Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:10:12.316Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin1593530494 network=unix @module=docker_logger timestamp=2021-09-13T07:10:12.316Z 2021-09-13T07:10:12.316Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:10:12.317Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:10:12.318Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:10:12.318Z 2021-09-13T07:10:12.323Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:10:12.325Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=45085 2021-09-13T07:10:12.325Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:10:15.288Z [DEBUG] docker/driver.go:505: docker: failed to start container: container_id=2519c83b3ed8c682a0cecf93ffa1707659580991aa5ca24f480fd3f17ae63af0 attempt=5 error="API error (500): unable to find user alice: no matching entries in passwd file" driver_test.go:2608: Error Trace: driver_test.go:2608 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_ReadonlyRootfs === FAIL: drivers/docker TestDockerDriver_Entrypoint (7.90s) === PAUSE TestDockerDriver_Entrypoint === CONT TestDockerDriver_Entrypoint 2021-09-13T07:10:17.477Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:10:17.504Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:10:17.511Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3967534651/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:10:17.511Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3967534651/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:10:17.529Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3967534651/alloc:/alloc, /tmp/nomad_driver_harness-3967534651/redis-demo/local:/local, /tmp/nomad_driver_harness-3967534651/redis-demo/secrets:/secrets] 2021-09-13T07:10:17.529Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:10:17.529Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:10:17.529Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3967534651/alloc:/alloc", "/tmp/nomad_driver_harness-3967534651/redis-demo/local:/local", "/tmp/nomad_driver_harness-3967534651/redis-demo/secrets:/secrets"}" 2021-09-13T07:10:17.529Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:10:17.529Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9187 label=main 2021-09-13T07:10:17.530Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9187 label=main 2021-09-13T07:10:17.530Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9188 label=REDIS 2021-09-13T07:10:17.530Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9188 label=REDIS 2021-09-13T07:10:17.530Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:10:17.530Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:7ec1b58d-83c8-839c-643b-b77025bc3cd2] 2021-09-13T07:10:17.530Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-7ec1b58d-83c8-839c-643b-b77025bc3cd2 2021-09-13T07:10:18.344Z [INFO] docker/driver.go:306: docker: created container: container_id=fc766638cf789bb3b3789c1a49855ef2f8fb9fc6a51ec02ef20d473220e68ac3 2021-09-13T07:10:20.313Z [INFO] docker/driver.go:347: docker: started container: container_id=fc766638cf789bb3b3789c1a49855ef2f8fb9fc6a51ec02ef20d473220e68ac3 2021-09-13T07:10:20.313Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:10:20.316Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=45374 2021-09-13T07:10:20.317Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:10:20.341Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin4038141021 network=unix @module=docker_logger timestamp=2021-09-13T07:10:20.341Z 2021-09-13T07:10:20.341Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:10:20.342Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:10:20.343Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:10:20.343Z 2021-09-13T07:10:20.347Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:10:20.349Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=45374 2021-09-13T07:10:20.349Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited driver_test.go:2582: Error Trace: driver_test.go:2582 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Entrypoint === FAIL: drivers/docker TestDockerDriver_Device_Success (7.05s) === PAUSE TestDockerDriver_Device_Success === CONT TestDockerDriver_Device_Success 2021-09-13T07:10:25.374Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:10:25.406Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:10:25.412Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1105381256/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:10:25.412Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1105381256/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:10:25.432Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-1105381256/alloc:/alloc, /tmp/nomad_driver_harness-1105381256/redis-demo/local:/local, /tmp/nomad_driver_harness-1105381256/redis-demo/secrets:/secrets] 2021-09-13T07:10:25.432Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:10:25.432Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:10:25.432Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-1105381256/alloc:/alloc", "/tmp/nomad_driver_harness-1105381256/redis-demo/local:/local", "/tmp/nomad_driver_harness-1105381256/redis-demo/secrets:/secrets"}" 2021-09-13T07:10:25.432Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:10:25.432Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9189 label=main 2021-09-13T07:10:25.432Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9189 label=main 2021-09-13T07:10:25.432Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9190 label=REDIS 2021-09-13T07:10:25.432Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9190 label=REDIS 2021-09-13T07:10:25.432Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:10:25.432Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:0c8575b1-18a9-c189-96e8-f16f978d5c9d] 2021-09-13T07:10:25.433Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-0c8575b1-18a9-c189-96e8-f16f978d5c9d 2021-09-13T07:10:26.108Z [INFO] docker/driver.go:306: docker: created container: container_id=85f0105b8978018baf71690994273b946c7be4e8459492119b37a58e9a0db56b 2021-09-13T07:10:27.327Z [INFO] docker/driver.go:347: docker: started container: container_id=85f0105b8978018baf71690994273b946c7be4e8459492119b37a58e9a0db56b 2021-09-13T07:10:27.327Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:10:27.327Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=45602 2021-09-13T07:10:27.327Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:10:27.356Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin1899647692 network=unix @module=docker_logger timestamp=2021-09-13T07:10:27.356Z 2021-09-13T07:10:27.356Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:10:27.357Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:10:27.358Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:10:27.358Z 2021-09-13T07:10:27.362Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:10:27.364Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=45602 2021-09-13T07:10:27.364Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited driver_test.go:2555: Error Trace: driver_test.go:2555 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Device_Success 2021-09-13T07:10:32.419Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown === FAIL: drivers/docker TestDockerDriver_OOMKilled (1.64s) === PAUSE TestDockerDriver_OOMKilled === CONT TestDockerDriver_OOMKilled 2021-09-13T07:10:40.634Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:10:40.634Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:10:40.647Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:10:40.656Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-733964490/alloc/logs/.oom-killed.stdout.fifo 2021-09-13T07:10:40.657Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-733964490/alloc/logs/.oom-killed.stderr.fifo 2021-09-13T07:10:40.756Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=oom-killed volumes=[/tmp/nomad_driver_harness-733964490/alloc:/alloc, /tmp/nomad_driver_harness-733964490/oom-killed/local:/local, /tmp/nomad_driver_harness-733964490/oom-killed/secrets:/secrets] 2021-09-13T07:10:40.756Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=oom-killed 2021-09-13T07:10:40.756Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=oom-killed memory=10485760 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:10:40.756Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=oom-killed binds="[]string{"/tmp/nomad_driver_harness-733964490/alloc:/alloc", "/tmp/nomad_driver_harness-733964490/oom-killed/local:/local", "/tmp/nomad_driver_harness-733964490/oom-killed/secrets:/secrets"}" 2021-09-13T07:10:40.756Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=oom-killed 2021-09-13T07:10:40.756Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=oom-killed command="sh -c sleep 2 && x=a && while true; do x="$x$x"; done" 2021-09-13T07:10:40.756Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=oom-killed labels=map[com.hashicorp.nomad.alloc_id:b6247896-c47a-b155-94ce-244a13f12968] 2021-09-13T07:10:40.756Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=oom-killed container_name=oom-killed-b6247896-c47a-b155-94ce-244a13f12968 2021-09-13T07:10:41.514Z [INFO] docker/driver.go:306: docker: created container: container_id=dae42353ac5681f7f5de017b79f835e317307ebba497ca760de3597efb6d7afa 2021-09-13T07:10:42.237Z [INFO] docker/driver.go:347: docker: started container: container_id=dae42353ac5681f7f5de017b79f835e317307ebba497ca760de3597efb6d7afa 2021-09-13T07:10:42.237Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:10:42.237Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=45754 2021-09-13T07:10:42.237Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:10:42.249Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1505746375 network=unix timestamp=2021-09-13T07:10:42.249Z 2021-09-13T07:10:42.250Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:10:42.250Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:10:42.251Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:10:42.251Z driver_test.go:2474: not killed by OOM killer: Docker container exited with non-zero exit code: 1 2021-09-13T07:10:42.256Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:10:42.257Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=45754 2021-09-13T07:10:42.257Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === FAIL: drivers/docker TestDockerDriver_VolumesEnabled (4.19s) === PAUSE TestDockerDriver_VolumesEnabled === CONT TestDockerDriver_VolumesEnabled 2021-09-13T07:10:48.363Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:10:48.402Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:10:48.417Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-77855666/alloc/logs/.ls.stdout.fifo 2021-09-13T07:10:48.417Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-77855666/alloc/logs/.ls.stderr.fifo 2021-09-13T07:10:48.436Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=ls volumes=[/tmp/nomad_driver_harness-77855666/alloc:/alloc, /tmp/nomad_driver_harness-77855666/ls/local:/local, /tmp/nomad_driver_harness-77855666/ls/secrets:/secrets, /tmp/nomadtest_docker_volumesenabled2593180456:/mnt/vol] 2021-09-13T07:10:48.436Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=ls 2021-09-13T07:10:48.436Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=ls memory=10485760 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:10:48.436Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=ls binds="[]string{"/tmp/nomad_driver_harness-77855666/alloc:/alloc", "/tmp/nomad_driver_harness-77855666/ls/local:/local", "/tmp/nomad_driver_harness-77855666/ls/secrets:/secrets", "/tmp/nomadtest_docker_volumesenabled2593180456:/mnt/vol"}" 2021-09-13T07:10:48.436Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=ls 2021-09-13T07:10:48.436Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=ls command="touch /mnt/vol/test-7796661617864483768" 2021-09-13T07:10:48.436Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=ls labels=map[com.hashicorp.nomad.alloc_id:315c7f00-8736-681e-f454-f2a60e6a543b] 2021-09-13T07:10:48.436Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=ls container_name=ls-315c7f00-8736-681e-f454-f2a60e6a543b 2021-09-13T07:10:50.460Z [INFO] docker/driver.go:306: docker: created container: container_id=6411ea61333e5631aaf44d1f7ec0ead99d8e3cb7005ad7c36aee3edb8e61f08f 2021-09-13T07:10:52.468Z [INFO] docker/driver.go:347: docker: started container: container_id=6411ea61333e5631aaf44d1f7ec0ead99d8e3cb7005ad7c36aee3edb8e61f08f 2021-09-13T07:10:52.468Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:10:52.475Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=46105 2021-09-13T07:10:52.475Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:10:52.514Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: network=unix @module=docker_logger address=/tmp/plugin2196007610 timestamp=2021-09-13T07:10:52.513Z 2021-09-13T07:10:52.514Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:10:52.515Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:10:52.516Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:10:52.516Z driver_test.go:2245: unexpected err: &{1 0 false Docker container exited with non-zero exit code: 1} 2021-09-13T07:10:52.521Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:10:52.524Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=46105 2021-09-13T07:10:52.524Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === FAIL: drivers/docker TestDockerDriver_VolumesDisabled (6.61s) === PAUSE TestDockerDriver_VolumesDisabled === CONT TestDockerDriver_VolumesDisabled 2021-09-13T07:10:52.552Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:10:52.592Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:10:52.603Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4253948120/alloc/logs/.ls.stdout.fifo 2021-09-13T07:10:52.603Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4253948120/alloc/logs/.ls.stderr.fifo 2021-09-13T07:10:52.620Z [ERROR] docker/driver.go:289: docker: failed to create container configuration: image_name=busybox:1.29.3 image_id=sha256:59788edf1f3e78cd0ebe6ce1446e9d10788225db3dedcfd1a59f764bad2b2690 error="volumes are not enabled; cannot mount host paths: "/tmp/nomadtest_docker_volumesdisabled2167610008:/mnt/vol"" docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:10:52.636Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:10:52.642Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1645591813/alloc/logs/.ls.stdout.fifo 2021-09-13T07:10:52.642Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1645591813/alloc/logs/.ls.stderr.fifo 2021-09-13T07:10:52.661Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=ls volumes=[/tmp/nomad_driver_harness-1645591813/alloc:/alloc, /tmp/nomad_driver_harness-1645591813/ls/local:/local, /tmp/nomad_driver_harness-1645591813/ls/secrets:/secrets, /tmp/nomad_driver_harness-1645591813/ls:/mnt/vol] 2021-09-13T07:10:52.661Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=ls 2021-09-13T07:10:52.661Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=ls memory=10485760 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:10:52.661Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=ls binds="[]string{"/tmp/nomad_driver_harness-1645591813/alloc:/alloc", "/tmp/nomad_driver_harness-1645591813/ls/local:/local", "/tmp/nomad_driver_harness-1645591813/ls/secrets:/secrets", "/tmp/nomad_driver_harness-1645591813/ls:/mnt/vol"}" 2021-09-13T07:10:52.661Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=ls 2021-09-13T07:10:52.661Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=ls command="touch /mnt/vol/test-5626304432460876527" 2021-09-13T07:10:52.661Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=ls labels=map[com.hashicorp.nomad.alloc_id:b9e47052-5f2c-d80b-1647-3ace4220f644] 2021-09-13T07:10:52.661Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=ls container_name=ls-b9e47052-5f2c-d80b-1647-3ace4220f644 2021-09-13T07:10:54.371Z [INFO] docker/driver.go:306: docker: created container: container_id=4272bff5c99fcbb75e20c3f18109a2f1e9703777d3854e3315579d08f02b0620 2021-09-13T07:10:55.111Z [INFO] docker/driver.go:347: docker: started container: container_id=4272bff5c99fcbb75e20c3f18109a2f1e9703777d3854e3315579d08f02b0620 2021-09-13T07:10:55.111Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:10:55.112Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=46259 2021-09-13T07:10:55.112Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:10:55.131Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin4284728033 network=unix timestamp=2021-09-13T07:10:55.131Z 2021-09-13T07:10:55.131Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:10:55.132Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:10:55.133Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:10:55.132Z driver_test.go:2184: unexpected err: &{1 0 false Docker container exited with non-zero exit code: 1} 2021-09-13T07:10:55.146Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:10:55.147Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=46259 2021-09-13T07:10:55.147Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:10:59.162Z [WARN] logmon/logmon.go:292: driver_harness.logmon: timed out waiting for read-side of process output pipe to close 2021-09-13T07:10:59.162Z [WARN] logmon/logmon.go:292: driver_harness.logmon: timed out waiting for read-side of process output pipe to close === FAIL: drivers/docker TestDockerDriver_Stats (7.44s) === PAUSE TestDockerDriver_Stats === CONT TestDockerDriver_Stats 2021-09-13T07:10:59.164Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:10:59.164Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:10:59.204Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:10:59.211Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3608112465/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:10:59.211Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3608112465/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:10:59.233Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3608112465/alloc:/alloc, /tmp/nomad_driver_harness-3608112465/redis-demo/local:/local, /tmp/nomad_driver_harness-3608112465/redis-demo/secrets:/secrets] 2021-09-13T07:10:59.233Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:10:59.233Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:10:59.233Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3608112465/alloc:/alloc", "/tmp/nomad_driver_harness-3608112465/redis-demo/local:/local", "/tmp/nomad_driver_harness-3608112465/redis-demo/secrets:/secrets"}" 2021-09-13T07:10:59.233Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:10:59.233Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9199 label=main 2021-09-13T07:10:59.233Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9199 label=main 2021-09-13T07:10:59.233Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9101 label=REDIS 2021-09-13T07:10:59.233Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9101 label=REDIS 2021-09-13T07:10:59.233Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="sleep 1000" 2021-09-13T07:10:59.233Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:f5576228-fc5d-84c7-6ae6-50a3336aa5c2] 2021-09-13T07:10:59.233Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-f5576228-fc5d-84c7-6ae6-50a3336aa5c2 2021-09-13T07:10:59.835Z [INFO] docker/driver.go:306: docker: created container: container_id=aab025782f67e84158cd8251cc5aecdac464535029a64091379ef2975a845d76 2021-09-13T07:11:01.226Z [INFO] docker/driver.go:347: docker: started container: container_id=aab025782f67e84158cd8251cc5aecdac464535029a64091379ef2975a845d76 2021-09-13T07:11:01.227Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:01.227Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=46470 2021-09-13T07:11:01.227Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:01.266Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin2536836978 network=unix timestamp=2021-09-13T07:11:01.266Z 2021-09-13T07:11:01.266Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:01.267Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:01.267Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:01.267Z 2021-09-13T07:11:01.273Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:01.275Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=46470 2021-09-13T07:11:01.275Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited driver_test.go:2078: Error Trace: driver_test.go:2078 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Stats === FAIL: drivers/docker TestDockerDriver_CleanupContainer (3.28s) === PAUSE TestDockerDriver_CleanupContainer === CONT TestDockerDriver_CleanupContainer 2021-09-13T07:11:06.604Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:06.688Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:06.698Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1782857884/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:06.698Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1782857884/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:06.718Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-1782857884/alloc:/alloc, /tmp/nomad_driver_harness-1782857884/redis-demo/local:/local, /tmp/nomad_driver_harness-1782857884/redis-demo/secrets:/secrets] 2021-09-13T07:11:06.718Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:06.718Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:06.718Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-1782857884/alloc:/alloc", "/tmp/nomad_driver_harness-1782857884/redis-demo/local:/local", "/tmp/nomad_driver_harness-1782857884/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:06.718Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:06.718Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9102 label=main 2021-09-13T07:11:06.718Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9102 label=main 2021-09-13T07:11:06.718Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9103 label=REDIS 2021-09-13T07:11:06.718Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9103 label=REDIS 2021-09-13T07:11:06.718Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="echo hello" 2021-09-13T07:11:06.718Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:694e88ee-d32a-0b74-a230-7ab9a3cf8fe5] 2021-09-13T07:11:06.718Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-694e88ee-d32a-0b74-a230-7ab9a3cf8fe5 2021-09-13T07:11:08.611Z [DEBUG] docker/driver.go:505: docker: failed to start container: container_id=2519c83b3ed8c682a0cecf93ffa1707659580991aa5ca24f480fd3f17ae63af0 attempt=6 error="API error (500): unable to find user alice: no matching entries in passwd file" 2021-09-13T07:11:08.611Z [ERROR] docker/driver.go:314: docker: failed to start container: container_id=2519c83b3ed8c682a0cecf93ffa1707659580991aa5ca24f480fd3f17ae63af0 error="API error (500): unable to find user alice: no matching entries in passwd file" 2021-09-13T07:11:08.867Z [INFO] docker/driver.go:306: docker: created container: container_id=5bc91289c2d8fc350045efb2bd4132a1d287afbac6310cb850b6a827a94a4f9c 2021-09-13T07:11:09.828Z [INFO] docker/driver.go:347: docker: started container: container_id=5bc91289c2d8fc350045efb2bd4132a1d287afbac6310cb850b6a827a94a4f9c 2021-09-13T07:11:09.828Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:09.829Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=46783 2021-09-13T07:11:09.829Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:09.841Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin572455455 network=unix timestamp=2021-09-13T07:11:09.840Z 2021-09-13T07:11:09.841Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:09.853Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:09.854Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:09.854Z driver_test.go:1851: err: &{1 0 false Docker container exited with non-zero exit code: 1} 2021-09-13T07:11:09.860Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:09.863Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=46783 2021-09-13T07:11:09.863Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === FAIL: drivers/docker TestDockerDriver_PortsMapping (6.60s) === PAUSE TestDockerDriver_PortsMapping === CONT TestDockerDriver_PortsMapping 2021-09-13T07:11:09.907Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:09.945Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:09.948Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3416705920/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:09.948Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3416705920/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:09.968Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3416705920/alloc:/alloc, /tmp/nomad_driver_harness-3416705920/redis-demo/local:/local, /tmp/nomad_driver_harness-3416705920/redis-demo/secrets:/secrets] 2021-09-13T07:11:09.968Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:09.968Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:09.968Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3416705920/alloc:/alloc", "/tmp/nomad_driver_harness-3416705920/redis-demo/local:/local", "/tmp/nomad_driver_harness-3416705920/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:09.968Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:09.968Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9108 label=main 2021-09-13T07:11:09.968Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9108 label=main 2021-09-13T07:11:09.968Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9109 label=REDIS 2021-09-13T07:11:09.968Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9109 label=REDIS 2021-09-13T07:11:09.968Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:09.968Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:f6d1af1c-bab5-7a8f-693c-37a7b068d824] 2021-09-13T07:11:09.968Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-f6d1af1c-bab5-7a8f-693c-37a7b068d824 2021-09-13T07:11:10.299Z [INFO] docker/driver.go:306: docker: created container: container_id=d9a42220436160d2869bb1b1369cfaa4ebd0a6b058f64b7ae369069d04566b34 2021-09-13T07:11:11.373Z [INFO] docker/driver.go:347: docker: started container: container_id=d9a42220436160d2869bb1b1369cfaa4ebd0a6b058f64b7ae369069d04566b34 2021-09-13T07:11:11.373Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:11.381Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=46984 2021-09-13T07:11:11.381Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:11.396Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin3285425490 network=unix timestamp=2021-09-13T07:11:11.396Z 2021-09-13T07:11:11.396Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:11.397Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:11.401Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:11.401Z 2021-09-13T07:11:11.409Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:11.411Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=46984 2021-09-13T07:11:11.411Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:11:12.676Z [WARN] logmon/logmon.go:292: driver_harness.logmon: timed out waiting for read-side of process output pipe to close 2021-09-13T07:11:12.676Z [WARN] logmon/logmon.go:292: driver_harness.logmon: timed out waiting for read-side of process output pipe to close === CONT TestDockerDriver_PortsMapping driver_test.go:1717: Error Trace: driver_test.go:1717 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_PortsMapping === FAIL: drivers/docker TestDockerDriver_PortsNoMap (6.94s) === PAUSE TestDockerDriver_PortsNoMap === CONT TestDockerDriver_PortsNoMap docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:12.719Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:12.722Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1439788120/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:12.722Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1439788120/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:12.760Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-1439788120/alloc:/alloc, /tmp/nomad_driver_harness-1439788120/redis-demo/local:/local, /tmp/nomad_driver_harness-1439788120/redis-demo/secrets:/secrets] 2021-09-13T07:11:12.760Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:12.760Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:12.761Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-1439788120/alloc:/alloc", "/tmp/nomad_driver_harness-1439788120/redis-demo/local:/local", "/tmp/nomad_driver_harness-1439788120/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:12.761Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:12.761Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9110 label=main 2021-09-13T07:11:12.761Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9110 label=main 2021-09-13T07:11:12.761Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9111 label=REDIS 2021-09-13T07:11:12.761Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9111 label=REDIS 2021-09-13T07:11:12.761Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:12.761Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:e3a5f8d0-ca07-7356-2689-fad0352e991b] 2021-09-13T07:11:12.761Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-e3a5f8d0-ca07-7356-2689-fad0352e991b 2021-09-13T07:11:13.298Z [INFO] docker/driver.go:306: docker: created container: container_id=e6fa6ce0c3a72de9080860e6f47fff0399157eafcae3291b37cc03544ca0c2a3 2021-09-13T07:11:14.546Z [INFO] docker/driver.go:347: docker: started container: container_id=e6fa6ce0c3a72de9080860e6f47fff0399157eafcae3291b37cc03544ca0c2a3 2021-09-13T07:11:14.546Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:14.547Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=47182 2021-09-13T07:11:14.547Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:14.576Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin2519774061 network=unix @module=docker_logger timestamp=2021-09-13T07:11:14.576Z 2021-09-13T07:11:14.576Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:14.590Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:14.590Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:14.590Z 2021-09-13T07:11:14.595Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:14.602Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=47182 2021-09-13T07:11:14.602Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_PortsNoMap driver_test.go:1668: Error Trace: driver_test.go:1668 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_PortsNoMap === FAIL: drivers/docker TestDockerWorkDir (7.93s) === PAUSE TestDockerWorkDir === CONT TestDockerWorkDir 2021-09-13T07:11:16.503Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:16.535Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:16.537Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4013401683/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:16.537Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4013401683/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:16.550Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-4013401683/alloc:/alloc, /tmp/nomad_driver_harness-4013401683/redis-demo/local:/local, /tmp/nomad_driver_harness-4013401683/redis-demo/secrets:/secrets] 2021-09-13T07:11:16.550Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:16.550Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:16.550Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-4013401683/alloc:/alloc", "/tmp/nomad_driver_harness-4013401683/redis-demo/local:/local", "/tmp/nomad_driver_harness-4013401683/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:16.550Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:16.550Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9112 label=main 2021-09-13T07:11:16.550Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9112 label=main 2021-09-13T07:11:16.550Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9113 label=REDIS 2021-09-13T07:11:16.550Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9113 label=REDIS 2021-09-13T07:11:16.550Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:16.550Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:0480562d-01ea-2e84-26d7-8b419df6e492] 2021-09-13T07:11:16.550Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-0480562d-01ea-2e84-26d7-8b419df6e492 2021-09-13T07:11:17.350Z [INFO] docker/driver.go:306: docker: created container: container_id=269dbe01648c18c5c70deb716be272d316d3966103a57c7a204f763c10d57813 2021-09-13T07:11:19.316Z [INFO] docker/driver.go:347: docker: started container: container_id=269dbe01648c18c5c70deb716be272d316d3966103a57c7a204f763c10d57813 2021-09-13T07:11:19.316Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:19.321Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=47382 2021-09-13T07:11:19.321Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:19.341Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1178462076 network=unix timestamp=2021-09-13T07:11:19.341Z 2021-09-13T07:11:19.342Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:19.342Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:19.343Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:19.343Z 2021-09-13T07:11:19.348Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:19.354Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=47382 2021-09-13T07:11:19.354Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerWorkDir driver_test.go:1639: Error Trace: driver_test.go:1639 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerWorkDir === FAIL: drivers/docker TestDockerDriver_MACAddress (9.56s) === PAUSE TestDockerDriver_MACAddress === CONT TestDockerDriver_MACAddress 2021-09-13T07:11:19.623Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:19.658Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:19.666Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2719466211/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:19.666Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2719466211/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:19.683Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-2719466211/alloc:/alloc, /tmp/nomad_driver_harness-2719466211/redis-demo/local:/local, /tmp/nomad_driver_harness-2719466211/redis-demo/secrets:/secrets] 2021-09-13T07:11:19.683Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:19.683Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:19.683Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-2719466211/alloc:/alloc", "/tmp/nomad_driver_harness-2719466211/redis-demo/local:/local", "/tmp/nomad_driver_harness-2719466211/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:19.683Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:19.683Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9114 label=main 2021-09-13T07:11:19.683Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9114 label=main 2021-09-13T07:11:19.683Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9115 label=REDIS 2021-09-13T07:11:19.683Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9115 label=REDIS 2021-09-13T07:11:19.683Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:19.683Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:39dde580-8fc2-1b7e-99db-cf2837e375ee] 2021-09-13T07:11:19.683Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-39dde580-8fc2-1b7e-99db-cf2837e375ee 2021-09-13T07:11:19.683Z [DEBUG] docker/driver.go:1203: docker: setting container mac address: task_name=redis-demo mac_address=00:16:3e:00:00:00 2021-09-13T07:11:21.977Z [INFO] docker/driver.go:306: docker: created container: container_id=198d570b6947229953919af9766fbe7822ab1d08654b44a145f3896726357ffa 2021-09-13T07:11:24.055Z [INFO] docker/driver.go:347: docker: started container: container_id=198d570b6947229953919af9766fbe7822ab1d08654b44a145f3896726357ffa 2021-09-13T07:11:24.055Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:24.055Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=47609 2021-09-13T07:11:24.055Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:24.074Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:24.075Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: network=unix @module=docker_logger address=/tmp/plugin682420348 timestamp=2021-09-13T07:11:24.074Z 2021-09-13T07:11:24.076Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:24.077Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:24.077Z 2021-09-13T07:11:24.094Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:24.102Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=47609 2021-09-13T07:11:24.102Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_MACAddress driver_test.go:1618: Error Trace: driver_test.go:1618 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_MACAddress === FAIL: drivers/docker TestDockerDriver_MemoryHardLimit (6.75s) === PAUSE TestDockerDriver_MemoryHardLimit === CONT TestDockerDriver_MemoryHardLimit 2021-09-13T07:11:24.435Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:24.463Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:24.470Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1853195503/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:24.471Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1853195503/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:24.487Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-1853195503/alloc:/alloc, /tmp/nomad_driver_harness-1853195503/redis-demo/local:/local, /tmp/nomad_driver_harness-1853195503/redis-demo/secrets:/secrets] 2021-09-13T07:11:24.487Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:24.487Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=314572800 memory_reservation=268435456 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:24.487Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-1853195503/alloc:/alloc", "/tmp/nomad_driver_harness-1853195503/redis-demo/local:/local", "/tmp/nomad_driver_harness-1853195503/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:24.487Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:24.487Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9116 label=main 2021-09-13T07:11:24.488Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9116 label=main 2021-09-13T07:11:24.488Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9117 label=REDIS 2021-09-13T07:11:24.488Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9117 label=REDIS 2021-09-13T07:11:24.488Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:24.488Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:c8cc81f1-eefc-cf25-0d99-2ec6966f0405] 2021-09-13T07:11:24.488Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-c8cc81f1-eefc-cf25-0d99-2ec6966f0405 2021-09-13T07:11:24.838Z [INFO] docker/driver.go:306: docker: created container: container_id=7aa293c261008895e67ff15dabb6519a72f2408e69cede2d690800360e6393c7 2021-09-13T07:11:26.057Z [INFO] docker/driver.go:347: docker: started container: container_id=7aa293c261008895e67ff15dabb6519a72f2408e69cede2d690800360e6393c7 2021-09-13T07:11:26.057Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:26.058Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=47805 2021-09-13T07:11:26.058Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:26.072Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin3140020335 network=unix @module=docker_logger timestamp=2021-09-13T07:11:26.072Z 2021-09-13T07:11:26.072Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:26.073Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:26.074Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:26.074Z 2021-09-13T07:11:26.080Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:26.082Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=47805 2021-09-13T07:11:26.082Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_MemoryHardLimit driver_test.go:1593: Error Trace: driver_test.go:1593 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_MemoryHardLimit === FAIL: drivers/docker TestDockerDriver_DNS (7.57s) === PAUSE TestDockerDriver_DNS === CONT TestDockerDriver_DNS 2021-09-13T07:11:31.190Z [INFO] docker/driver.go:347: docker: started container: container_id=bf641d9f2d3a0a0531a8e3ea1f1b0979aa09e0808015ced855aaa257c6222f45 2021-09-13T07:11:31.190Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:31.190Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:11:31.201Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=48007 2021-09-13T07:11:31.201Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:31.227Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:31.231Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:31.231Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin530790304 network=unix timestamp=2021-09-13T07:11:31.231Z 2021-09-13T07:11:31.236Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:31.238Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2178258234/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:31.238Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2178258234/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:31.241Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:31.240Z 2021-09-13T07:11:31.252Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:31.254Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=48007 2021-09-13T07:11:31.254Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:11:31.260Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-2178258234/alloc:/alloc, /tmp/nomad_driver_harness-2178258234/redis-demo/local:/local, /tmp/nomad_driver_harness-2178258234/redis-demo/secrets:/secrets] 2021-09-13T07:11:31.260Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:31.260Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:31.260Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-2178258234/alloc:/alloc", "/tmp/nomad_driver_harness-2178258234/redis-demo/local:/local", "/tmp/nomad_driver_harness-2178258234/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:31.260Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:31.260Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9120 label=main 2021-09-13T07:11:31.260Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9120 label=main 2021-09-13T07:11:31.260Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9121 label=REDIS 2021-09-13T07:11:31.260Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9121 label=REDIS 2021-09-13T07:11:31.260Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:31.261Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:e8f90287-bb57-a4ed-eb25-7805e5bea5f8] 2021-09-13T07:11:31.261Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-e8f90287-bb57-a4ed-eb25-7805e5bea5f8 2021-09-13T07:11:31.714Z [INFO] docker/driver.go:306: docker: created container: container_id=4c0b8001414c05142650ee094ffc77b19038ebb43072d079f149ff0c8ef654f0 2021-09-13T07:11:33.419Z [INFO] docker/driver.go:347: docker: started container: container_id=4c0b8001414c05142650ee094ffc77b19038ebb43072d079f149ff0c8ef654f0 2021-09-13T07:11:33.419Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:33.420Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=48217 2021-09-13T07:11:33.420Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:33.448Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin2795051404 network=unix @module=docker_logger timestamp=2021-09-13T07:11:33.448Z 2021-09-13T07:11:33.449Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:33.450Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:33.452Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:33.452Z 2021-09-13T07:11:33.458Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:33.460Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=48217 2021-09-13T07:11:33.460Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_DNS driver_test.go:1521: Error Trace: driver_test.go:1521 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_DNS === FAIL: drivers/docker TestDockerDriver_CPUSetCPUs/Single_CPU (7.11s) docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:29.216Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:29.219Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3733203855/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:29.219Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3733203855/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:29.235Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3733203855/alloc:/alloc, /tmp/nomad_driver_harness-3733203855/redis-demo/local:/local, /tmp/nomad_driver_harness-3733203855/redis-demo/secrets:/secrets] 2021-09-13T07:11:29.235Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:29.235Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:29.235Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3733203855/alloc:/alloc", "/tmp/nomad_driver_harness-3733203855/redis-demo/local:/local", "/tmp/nomad_driver_harness-3733203855/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:29.235Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:29.235Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9118 label=main 2021-09-13T07:11:29.235Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9118 label=main 2021-09-13T07:11:29.235Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9119 label=REDIS 2021-09-13T07:11:29.235Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9119 label=REDIS 2021-09-13T07:11:29.235Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:29.235Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:80f32df6-c4b5-5056-7820-7639e49d9cb8] 2021-09-13T07:11:29.235Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-80f32df6-c4b5-5056-7820-7639e49d9cb8 2021-09-13T07:11:29.862Z [INFO] docker/driver.go:306: docker: created container: container_id=bf641d9f2d3a0a0531a8e3ea1f1b0979aa09e0808015ced855aaa257c6222f45 === CONT TestDockerDriver_CPUSetCPUs/Single_CPU driver_test.go:1566: Error Trace: driver_test.go:1566 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_CPUSetCPUs/Single_CPU --- FAIL: TestDockerDriver_CPUSetCPUs/Single_CPU (7.11s) === FAIL: drivers/docker TestDockerDriver_CPUSetCPUs/Comma_separated_list_of_CPUs (12.47s) 2021-09-13T07:11:36.308Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:36.326Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:36.329Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-353328255/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:36.329Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-353328255/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:36.345Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-353328255/alloc:/alloc, /tmp/nomad_driver_harness-353328255/redis-demo/local:/local, /tmp/nomad_driver_harness-353328255/redis-demo/secrets:/secrets] 2021-09-13T07:11:36.345Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:36.345Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:36.345Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-353328255/alloc:/alloc", "/tmp/nomad_driver_harness-353328255/redis-demo/local:/local", "/tmp/nomad_driver_harness-353328255/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:36.345Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:36.346Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9122 label=main 2021-09-13T07:11:36.346Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9122 label=main 2021-09-13T07:11:36.346Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9123 label=REDIS 2021-09-13T07:11:36.346Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9123 label=REDIS 2021-09-13T07:11:36.346Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:36.346Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:7750a72d-adfd-96fc-7dd5-86d754c86c7a] 2021-09-13T07:11:36.346Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-7750a72d-adfd-96fc-7dd5-86d754c86c7a === CONT TestDockerDriver_CPUSetCPUs/Comma_separated_list_of_CPUs driver_test.go:1566: Error Trace: driver_test.go:1566 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_CPUSetCPUs/Comma_separated_list_of_CPUs --- FAIL: TestDockerDriver_CPUSetCPUs/Comma_separated_list_of_CPUs (12.47s) === FAIL: drivers/docker TestDockerDriver_CPUSetCPUs/Range_of_CPUs (10.25s) 2021-09-13T07:11:48.778Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:48.800Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:48.816Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1151907622/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:48.816Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1151907622/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:48.838Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-1151907622/alloc:/alloc, /tmp/nomad_driver_harness-1151907622/redis-demo/local:/local, /tmp/nomad_driver_harness-1151907622/redis-demo/secrets:/secrets] 2021-09-13T07:11:48.838Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:48.838Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:48.839Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-1151907622/alloc:/alloc", "/tmp/nomad_driver_harness-1151907622/redis-demo/local:/local", "/tmp/nomad_driver_harness-1151907622/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:48.839Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:48.839Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9126 label=main 2021-09-13T07:11:48.839Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9126 label=main 2021-09-13T07:11:48.839Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9127 label=REDIS 2021-09-13T07:11:48.839Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9127 label=REDIS 2021-09-13T07:11:48.839Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:48.839Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:e2d86e91-ae54-ae5d-bda0-a1ffee50ac66] 2021-09-13T07:11:48.839Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-e2d86e91-ae54-ae5d-bda0-a1ffee50ac66 === CONT TestDockerDriver_CPUSetCPUs/Range_of_CPUs driver_test.go:1566: Error Trace: driver_test.go:1566 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_CPUSetCPUs/Range_of_CPUs --- FAIL: TestDockerDriver_CPUSetCPUs/Range_of_CPUs (10.25s) === FAIL: drivers/docker TestDockerDriver_CPUSetCPUs (29.84s) === PAUSE TestDockerDriver_CPUSetCPUs === CONT TestDockerDriver_CPUSetCPUs 2021-09-13T07:11:29.186Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 === FAIL: drivers/docker TestDockerDriver_Runtime (17.05s) === PAUSE TestDockerDriver_Runtime === CONT TestDockerDriver_Runtime 2021-09-13T07:11:59.149Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:59.182Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:59.193Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3387847700/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:59.194Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3387847700/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:59.423Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3387847700/alloc:/alloc, /tmp/nomad_driver_harness-3387847700/redis-demo/local:/local, /tmp/nomad_driver_harness-3387847700/redis-demo/secrets:/secrets] 2021-09-13T07:11:59.423Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:59.424Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:59.424Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3387847700/alloc:/alloc", "/tmp/nomad_driver_harness-3387847700/redis-demo/local:/local", "/tmp/nomad_driver_harness-3387847700/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:59.424Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:59.424Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9162 label=main 2021-09-13T07:11:59.424Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9162 label=main 2021-09-13T07:11:59.424Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9163 label=REDIS 2021-09-13T07:11:59.424Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9163 label=REDIS 2021-09-13T07:11:59.424Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:59.424Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:44814a21-c65a-91a3-4395-96f8f1abbc86] 2021-09-13T07:11:59.424Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-44814a21-c65a-91a3-4395-96f8f1abbc86 2021-09-13T07:12:00.300Z [INFO] docker/driver.go:306: docker: created container: container_id=9bef4f526c61449589973b7156ea3aa5f92457a25a33b696da03347b1cae0782 2021-09-13T07:12:05.753Z [INFO] docker/driver.go:306: docker: created container: container_id=303c06a243d190bc2005383f5d26137ff1cba32ce4efabda37a20f99318870d8 2021-09-13T07:12:06.678Z [INFO] docker/driver.go:347: docker: started container: container_id=9bef4f526c61449589973b7156ea3aa5f92457a25a33b696da03347b1cae0782 2021-09-13T07:12:06.693Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:12:06.705Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=49219 2021-09-13T07:12:06.705Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:12:06.747Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin671459521 network=unix timestamp=2021-09-13T07:12:06.747Z 2021-09-13T07:12:06.747Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:12:06.773Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:12:06.774Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:12:06.774Z 2021-09-13T07:12:06.798Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:12:06.799Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=49219 2021-09-13T07:12:06.799Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:12:10.864Z [INFO] docker/driver.go:347: docker: started container: container_id=303c06a243d190bc2005383f5d26137ff1cba32ce4efabda37a20f99318870d8 2021-09-13T07:12:10.864Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:12:10.865Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=49319 2021-09-13T07:12:10.865Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:12:10.895Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1659473999 network=unix timestamp=2021-09-13T07:12:10.895Z 2021-09-13T07:12:10.895Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:12:10.896Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:12:10.897Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:12:10.897Z 2021-09-13T07:12:10.901Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:12:10.903Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=49319 2021-09-13T07:12:10.903Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_Runtime driver_test.go:991: Error Trace: driver_test.go:991 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Runtime === FAIL: drivers/docker TestDockerDriver_SecurityOptFromFile (14.89s) === PAUSE TestDockerDriver_SecurityOptFromFile === CONT TestDockerDriver_SecurityOptFromFile 2021-09-13T07:12:16.196Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:12:16.238Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:12:16.249Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3365552759/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:12:16.249Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3365552759/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:12:16.334Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3365552759/alloc:/alloc, /tmp/nomad_driver_harness-3365552759/redis-demo/local:/local, /tmp/nomad_driver_harness-3365552759/redis-demo/secrets:/secrets] 2021-09-13T07:12:16.334Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:12:16.334Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:12:16.334Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3365552759/alloc:/alloc", "/tmp/nomad_driver_harness-3365552759/redis-demo/local:/local", "/tmp/nomad_driver_harness-3365552759/redis-demo/secrets:/secrets"}" 2021-09-13T07:12:16.338Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:12:16.338Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9166 label=main 2021-09-13T07:12:16.338Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9166 label=main 2021-09-13T07:12:16.338Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9167 label=REDIS 2021-09-13T07:12:16.338Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9167 label=REDIS 2021-09-13T07:12:16.338Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:12:16.338Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:5c61d1e4-a22d-ffa0-a479-f6b1a8b9181a] 2021-09-13T07:12:16.338Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-5c61d1e4-a22d-ffa0-a479-f6b1a8b9181a 2021-09-13T07:12:18.339Z [INFO] docker/driver.go:306: docker: created container: container_id=fcc9c8b81ccb3dad0b54a801f0117f0851be9941770f6dd2e8556b9efd545e84 2021-09-13T07:12:22.095Z [INFO] docker/driver.go:306: docker: created container: container_id=e2cfc81daea295c0c324dd3c883394739e3ce43b09a908e1663e08a234d48e27 2021-09-13T07:12:23.100Z [INFO] docker/driver.go:347: docker: started container: container_id=fcc9c8b81ccb3dad0b54a801f0117f0851be9941770f6dd2e8556b9efd545e84 2021-09-13T07:12:23.100Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:12:23.100Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=49633 2021-09-13T07:12:23.100Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:12:23.150Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin2852707566 network=unix @module=docker_logger timestamp=2021-09-13T07:12:23.149Z 2021-09-13T07:12:23.155Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:12:23.160Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:12:23.162Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:12:23.162Z 2021-09-13T07:12:23.170Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:12:23.177Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=49633 2021-09-13T07:12:23.177Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:12:25.906Z [INFO] docker/driver.go:347: docker: started container: container_id=e2cfc81daea295c0c324dd3c883394739e3ce43b09a908e1663e08a234d48e27 2021-09-13T07:12:25.906Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:12:25.906Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=49735 2021-09-13T07:12:25.906Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:12:25.932Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:12:25.932Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin278046912 network=unix timestamp=2021-09-13T07:12:25.931Z 2021-09-13T07:12:25.936Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:12:25.936Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:12:25.936Z 2021-09-13T07:12:25.945Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:12:25.954Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=49735 2021-09-13T07:12:25.954Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_SecurityOptFromFile driver_test.go:970: Error Trace: driver_test.go:970 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_SecurityOptFromFile === FAIL: drivers/docker TestDockerDriver_SecurityOptUnconfined (12.52s) === PAUSE TestDockerDriver_SecurityOptUnconfined === CONT TestDockerDriver_SecurityOptUnconfined 2021-09-13T07:12:31.085Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:12:31.143Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:12:31.146Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4082639476/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:12:31.146Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4082639476/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:12:31.280Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-4082639476/alloc:/alloc, /tmp/nomad_driver_harness-4082639476/redis-demo/local:/local, /tmp/nomad_driver_harness-4082639476/redis-demo/secrets:/secrets] 2021-09-13T07:12:31.280Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:12:31.280Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:12:31.280Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-4082639476/alloc:/alloc", "/tmp/nomad_driver_harness-4082639476/redis-demo/local:/local", "/tmp/nomad_driver_harness-4082639476/redis-demo/secrets:/secrets"}" 2021-09-13T07:12:31.280Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:12:31.280Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9170 label=main 2021-09-13T07:12:31.281Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9170 label=main 2021-09-13T07:12:31.281Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9171 label=REDIS 2021-09-13T07:12:31.281Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9171 label=REDIS 2021-09-13T07:12:31.281Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:12:31.281Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:9cf5fdb8-d0a5-2285-79a9-9cc3ffa8b96c] 2021-09-13T07:12:31.281Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-9cf5fdb8-d0a5-2285-79a9-9cc3ffa8b96c 2021-09-13T07:12:32.855Z [INFO] docker/driver.go:306: docker: created container: container_id=56b51a88412f449b0a3b2ed1c26be082d9aea63badba363e2dbdaf4b6ecc7b42 2021-09-13T07:12:35.142Z [INFO] docker/driver.go:306: docker: created container: container_id=4b38c3a396c2b71293ff583ca92a410e1f10faf5f701375bf410d0b365f86230 2021-09-13T07:12:37.632Z [INFO] docker/driver.go:347: docker: started container: container_id=56b51a88412f449b0a3b2ed1c26be082d9aea63badba363e2dbdaf4b6ecc7b42 2021-09-13T07:12:37.632Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:12:37.637Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=50167 2021-09-13T07:12:37.637Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:12:37.661Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin3176868223 network=unix @module=docker_logger timestamp=2021-09-13T07:12:37.661Z 2021-09-13T07:12:37.661Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:12:37.663Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:12:37.664Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:12:37.664Z 2021-09-13T07:12:37.667Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:12:37.669Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=50167 2021-09-13T07:12:37.669Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:12:38.519Z [INFO] docker/driver.go:347: docker: started container: container_id=4b38c3a396c2b71293ff583ca92a410e1f10faf5f701375bf410d0b365f86230 2021-09-13T07:12:38.519Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:12:38.526Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=50178 2021-09-13T07:12:38.526Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:12:38.545Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: network=unix @module=docker_logger address=/tmp/plugin188305599 timestamp=2021-09-13T07:12:38.545Z 2021-09-13T07:12:38.545Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:12:38.546Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:12:38.547Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:12:38.547Z 2021-09-13T07:12:38.559Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:12:38.568Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=50178 2021-09-13T07:12:38.568Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_SecurityOptUnconfined driver_test.go:943: Error Trace: driver_test.go:943 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_SecurityOptUnconfined 2021-09-13T07:12:43.609Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown === FAIL: drivers/docker TestDockerDriver_ForcePull_RepoDigest (4.20s) === PAUSE TestDockerDriver_ForcePull_RepoDigest === CONT TestDockerDriver_ForcePull_RepoDigest docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:12:43.647Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:12:43.650Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3956082867/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:12:43.650Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3956082867/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:12:43.659Z [DEBUG] docker/driver.go:536: docker: force pulling image instead of inspecting local: image_ref=library/busybox@sha256:58ac43b2cc92c687a32c8be6278e50a063579655fe3090125dcb2af0ff9e1a64 2021-09-13T07:12:43.660Z [TRACE] eventer/eventer.go:152: docker: emitting event: event="&{eff3f64b-413c-a9ae-caf9-ba12d9074ccf redis-demo bd608a00-1924-d2be-3e5c-e3a785aea208 2021-09-13 07:12:43.66018782 +0000 UTC m=+240.178468202 Downloading image map[image:library/busybox@sha256:58ac43b2cc92c687a32c8be6278e50a063579655fe3090125dcb2af0ff9e1a64] }" 2021-09-13T07:12:43.759Z [DEBUG] docker/coordinator.go:211: docker: docker pull succeeded: image_ref=library/busybox@sha256:58ac43b2cc92c687a32c8be6278e50a063579655fe3090125dcb2af0ff9e1a64 2021-09-13T07:12:43.760Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3956082867/alloc:/alloc, /tmp/nomad_driver_harness-3956082867/redis-demo/local:/local, /tmp/nomad_driver_harness-3956082867/redis-demo/secrets:/secrets] 2021-09-13T07:12:43.760Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:12:43.760Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:12:43.761Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3956082867/alloc:/alloc", "/tmp/nomad_driver_harness-3956082867/redis-demo/local:/local", "/tmp/nomad_driver_harness-3956082867/redis-demo/secrets:/secrets"}" 2021-09-13T07:12:43.761Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:12:43.761Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9174 label=main 2021-09-13T07:12:43.761Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9174 label=main 2021-09-13T07:12:43.761Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9177 label=REDIS 2021-09-13T07:12:43.761Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9177 label=REDIS 2021-09-13T07:12:43.761Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:12:43.761Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:bd608a00-1924-d2be-3e5c-e3a785aea208] 2021-09-13T07:12:43.761Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-bd608a00-1924-d2be-3e5c-e3a785aea208 2021-09-13T07:12:46.223Z [INFO] docker/driver.go:306: docker: created container: container_id=33a5a7bd411f52e09f1bdfc040d353dd92c2b6c63f889af986b82838d41baf2b 2021-09-13T07:12:46.963Z [INFO] docker/driver.go:347: docker: started container: container_id=33a5a7bd411f52e09f1bdfc040d353dd92c2b6c63f889af986b82838d41baf2b 2021-09-13T07:12:46.963Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:12:46.963Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=50444 2021-09-13T07:12:46.963Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:12:47.001Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin1619464423 network=unix timestamp=2021-09-13T07:12:47.001Z 2021-09-13T07:12:47.001Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:12:47.002Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:12:47.004Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:12:47.003Z driver_test.go:924: Error Trace: driver_test.go:924 Error: Not equal: expected: "sha256:8ac48589692a53a9b8c2d1ceaa6b402665aa7fe667ba51ccc03002300856d8c7" actual : "sha256:ba3a78826904c625e65a2eed1f247bbab59898f043490e7113e88907bf7c6b3b" Diff: --- Expected +++ Actual @@ -1 +1 @@ -sha256:8ac48589692a53a9b8c2d1ceaa6b402665aa7fe667ba51ccc03002300856d8c7 +sha256:ba3a78826904c625e65a2eed1f247bbab59898f043490e7113e88907bf7c6b3b Test: TestDockerDriver_ForcePull_RepoDigest 2021-09-13T07:12:47.108Z [WARN] logmon/logmon.go:292: driver_harness.logmon: timed out waiting for read-side of process output pipe to close 2021-09-13T07:12:47.108Z [WARN] logmon/logmon.go:292: driver_harness.logmon: timed out waiting for read-side of process output pipe to close === FAIL: drivers/docker TestDockerDriver_ForcePull (8.96s) === PAUSE TestDockerDriver_ForcePull === CONT TestDockerDriver_ForcePull 2021-09-13T07:12:47.805Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:12:47.808Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=50444 2021-09-13T07:12:47.808Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:12:47.850Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:12:47.862Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3162971015/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:12:47.862Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3162971015/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:12:47.882Z [DEBUG] docker/driver.go:536: docker: force pulling image instead of inspecting local: image_ref=busybox:1.29.3 2021-09-13T07:12:47.882Z [DEBUG] docker/driver.go:618: docker: loading image from disk: archive=/tmp/nomad_driver_harness-3162971015/redis-demo/local/busybox.tar 2021-09-13T07:12:48.019Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3162971015/alloc:/alloc, /tmp/nomad_driver_harness-3162971015/redis-demo/local:/local, /tmp/nomad_driver_harness-3162971015/redis-demo/secrets:/secrets] 2021-09-13T07:12:48.020Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:12:48.020Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:12:48.020Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3162971015/alloc:/alloc", "/tmp/nomad_driver_harness-3162971015/redis-demo/local:/local", "/tmp/nomad_driver_harness-3162971015/redis-demo/secrets:/secrets"}" 2021-09-13T07:12:48.020Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:12:48.020Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9180 label=main 2021-09-13T07:12:48.020Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9180 label=main 2021-09-13T07:12:48.020Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9181 label=REDIS 2021-09-13T07:12:48.020Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9181 label=REDIS 2021-09-13T07:12:48.020Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:12:48.020Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:e2f57fe0-c7a7-85d6-9ed0-abe6bee1447b] 2021-09-13T07:12:48.020Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-e2f57fe0-c7a7-85d6-9ed0-abe6bee1447b 2021-09-13T07:12:49.018Z [INFO] docker/driver.go:306: docker: created container: container_id=b1dfca0d37af62016804d70eb7123aa96cfa50fce8b6439404754271f0a414c2 2021-09-13T07:12:49.820Z [INFO] docker/driver.go:347: docker: started container: container_id=f01c56979f211e45d028697d581d8137d4564b5c91995c62a6ebcfdd1add9904 2021-09-13T07:12:49.820Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:12:49.821Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=50851 2021-09-13T07:12:49.821Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:12:49.857Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin3891246042 network=unix @module=docker_logger timestamp=2021-09-13T07:12:49.857Z 2021-09-13T07:12:49.858Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:12:49.858Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:12:49.859Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:12:49.859Z 2021-09-13T07:12:49.876Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:12:49.878Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=50851 2021-09-13T07:12:49.878Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:12:51.409Z [INFO] docker/driver.go:347: docker: started container: container_id=b1dfca0d37af62016804d70eb7123aa96cfa50fce8b6439404754271f0a414c2 2021-09-13T07:12:51.410Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:12:51.438Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=50977 2021-09-13T07:12:51.438Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:12:51.463Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin734860705 network=unix timestamp=2021-09-13T07:12:51.463Z 2021-09-13T07:12:51.463Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:12:51.464Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:12:51.494Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:12:51.493Z 2021-09-13T07:12:51.503Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:12:51.504Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=50977 2021-09-13T07:12:51.504Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_ForcePull driver_test.go:890: Error Trace: driver_test.go:890 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_ForcePull === FAIL: drivers/docker TestDockerDriver_Capabilities/default-allowlist-add-allowed (12.81s) docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:38.799Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:38.810Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2788322385/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:38.810Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2788322385/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:38.852Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-2788322385/alloc:/alloc, /tmp/nomad_driver_harness-2788322385/redis-demo/local:/local, /tmp/nomad_driver_harness-2788322385/redis-demo/secrets:/secrets] 2021-09-13T07:11:38.852Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:38.852Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:38.852Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-2788322385/alloc:/alloc", "/tmp/nomad_driver_harness-2788322385/redis-demo/local:/local", "/tmp/nomad_driver_harness-2788322385/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:38.852Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:38.852Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9124 label=main 2021-09-13T07:11:38.852Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9124 label=main 2021-09-13T07:11:38.852Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9125 label=REDIS 2021-09-13T07:11:38.852Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9125 label=REDIS 2021-09-13T07:11:38.852Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:38.852Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:f74bdace-f386-cc0a-15e0-c42fa69d908c] 2021-09-13T07:11:38.852Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-f74bdace-f386-cc0a-15e0-c42fa69d908c 2021-09-13T07:11:39.931Z [INFO] docker/driver.go:306: docker: created container: container_id=0cce144b695d0a342c43e4186e40defe833af5d05dbd69e16d8242e37b94156b 2021-09-13T07:11:43.083Z [INFO] docker/driver.go:306: docker: created container: container_id=fba76f76b7b813846f6ca1fd1ecabfcaa043d9bf9412a8007844c1262dade995 2021-09-13T07:11:43.673Z [INFO] docker/driver.go:347: docker: started container: container_id=0cce144b695d0a342c43e4186e40defe833af5d05dbd69e16d8242e37b94156b 2021-09-13T07:11:43.674Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:43.674Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=48532 2021-09-13T07:11:43.674Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:43.708Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin319642142 network=unix @module=docker_logger timestamp=2021-09-13T07:11:43.708Z 2021-09-13T07:11:43.709Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:43.710Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:43.723Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:43.723Z 2021-09-13T07:11:43.729Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:43.737Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=48532 2021-09-13T07:11:43.737Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:11:46.273Z [INFO] docker/driver.go:347: docker: started container: container_id=fba76f76b7b813846f6ca1fd1ecabfcaa043d9bf9412a8007844c1262dade995 2021-09-13T07:11:46.273Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:11:46.277Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=48629 2021-09-13T07:11:46.277Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:11:46.308Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin2305375546 network=unix timestamp=2021-09-13T07:11:46.307Z 2021-09-13T07:11:46.308Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:11:46.309Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:11:46.309Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:11:46.309Z 2021-09-13T07:11:46.315Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:11:46.317Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=48629 2021-09-13T07:11:46.318Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_Capabilities/default-allowlist-add-allowed driver_test.go:1471: Error Trace: driver_test.go:1471 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Capabilities/default-allowlist-add-allowed --- FAIL: TestDockerDriver_Capabilities/default-allowlist-add-allowed (12.81s) === FAIL: drivers/docker TestDockerDriver_Capabilities/default-allowlist-drop-existing (16.01s) 2021-09-13T07:11:55.928Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:11:55.960Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:11:55.968Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2547108722/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:11:55.968Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-2547108722/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:11:55.981Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-2547108722/alloc:/alloc, /tmp/nomad_driver_harness-2547108722/redis-demo/local:/local, /tmp/nomad_driver_harness-2547108722/redis-demo/secrets:/secrets] 2021-09-13T07:11:55.981Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:11:55.981Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:11:55.981Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-2547108722/alloc:/alloc", "/tmp/nomad_driver_harness-2547108722/redis-demo/local:/local", "/tmp/nomad_driver_harness-2547108722/redis-demo/secrets:/secrets"}" 2021-09-13T07:11:55.981Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:11:55.981Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9130 label=main 2021-09-13T07:11:55.981Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9130 label=main 2021-09-13T07:11:55.981Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9131 label=REDIS 2021-09-13T07:11:55.981Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9131 label=REDIS 2021-09-13T07:11:55.981Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:11:55.981Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:996928a2-5109-ecf5-4d43-853265569c40] 2021-09-13T07:11:55.981Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-996928a2-5109-ecf5-4d43-853265569c40 === CONT TestDockerDriver_Capabilities/default-allowlist-drop-existing driver_test.go:1471: Error Trace: driver_test.go:1471 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Capabilities/default-allowlist-drop-existing --- FAIL: TestDockerDriver_Capabilities/default-allowlist-drop-existing (16.01s) === FAIL: drivers/docker TestDockerDriver_Capabilities/restrictive-allowlist-drop-all (16.56s) 2021-09-13T07:12:11.941Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:12:11.964Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:12:11.967Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3217620611/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:12:11.967Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3217620611/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:12:11.989Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3217620611/alloc:/alloc, /tmp/nomad_driver_harness-3217620611/redis-demo/local:/local, /tmp/nomad_driver_harness-3217620611/redis-demo/secrets:/secrets] 2021-09-13T07:12:11.989Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:12:11.989Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:12:11.989Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3217620611/alloc:/alloc", "/tmp/nomad_driver_harness-3217620611/redis-demo/local:/local", "/tmp/nomad_driver_harness-3217620611/redis-demo/secrets:/secrets"}" 2021-09-13T07:12:11.989Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:12:11.989Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9164 label=main 2021-09-13T07:12:11.989Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9164 label=main 2021-09-13T07:12:11.989Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9165 label=REDIS 2021-09-13T07:12:11.989Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9165 label=REDIS 2021-09-13T07:12:11.989Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:12:11.989Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:0c23c5ac-284f-ff09-7a00-5f31e36c3805] 2021-09-13T07:12:11.990Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-0c23c5ac-284f-ff09-7a00-5f31e36c3805 === CONT TestDockerDriver_Capabilities/restrictive-allowlist-drop-all driver_test.go:1471: Error Trace: driver_test.go:1471 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Capabilities/restrictive-allowlist-drop-all 2021-09-13T07:12:28.498Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown --- FAIL: TestDockerDriver_Capabilities/restrictive-allowlist-drop-all (16.56s) === FAIL: drivers/docker TestDockerDriver_Capabilities/restrictive-allowlist-add-allowed (14.54s) docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:12:28.517Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:12:28.520Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1751241661/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:12:28.520Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1751241661/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:12:28.535Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-1751241661/alloc:/alloc, /tmp/nomad_driver_harness-1751241661/redis-demo/local:/local, /tmp/nomad_driver_harness-1751241661/redis-demo/secrets:/secrets] 2021-09-13T07:12:28.535Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:12:28.535Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:12:28.535Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-1751241661/alloc:/alloc", "/tmp/nomad_driver_harness-1751241661/redis-demo/local:/local", "/tmp/nomad_driver_harness-1751241661/redis-demo/secrets:/secrets"}" 2021-09-13T07:12:28.536Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:12:28.536Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9168 label=main 2021-09-13T07:12:28.536Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9168 label=main 2021-09-13T07:12:28.536Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9169 label=REDIS 2021-09-13T07:12:28.536Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9169 label=REDIS 2021-09-13T07:12:28.536Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:12:28.536Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:53568a2a-2a83-fd83-ddc6-f1701dbd6f96] 2021-09-13T07:12:28.536Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-53568a2a-2a83-fd83-ddc6-f1701dbd6f96 === CONT TestDockerDriver_Capabilities/restrictive-allowlist-add-allowed driver_test.go:1471: Error Trace: driver_test.go:1471 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Capabilities/restrictive-allowlist-add-allowed --- FAIL: TestDockerDriver_Capabilities/restrictive-allowlist-add-allowed (14.54s) === FAIL: drivers/docker TestDockerDriver_Capabilities/permissive-allowlist (8.21s) 2021-09-13T07:12:47.113Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:12:47.139Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:12:47.147Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3185567956/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:12:47.148Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3185567956/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:12:47.178Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3185567956/alloc:/alloc, /tmp/nomad_driver_harness-3185567956/redis-demo/local:/local, /tmp/nomad_driver_harness-3185567956/redis-demo/secrets:/secrets] 2021-09-13T07:12:47.178Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:12:47.178Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:12:47.178Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3185567956/alloc:/alloc", "/tmp/nomad_driver_harness-3185567956/redis-demo/local:/local", "/tmp/nomad_driver_harness-3185567956/redis-demo/secrets:/secrets"}" 2021-09-13T07:12:47.178Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:12:47.178Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9178 label=main 2021-09-13T07:12:47.178Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9178 label=main 2021-09-13T07:12:47.178Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9179 label=REDIS 2021-09-13T07:12:47.178Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9179 label=REDIS 2021-09-13T07:12:47.178Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:12:47.178Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:73eb3450-95a5-ff09-8df0-67617a661ff0] 2021-09-13T07:12:47.178Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-73eb3450-95a5-ff09-8df0-67617a661ff0 2021-09-13T07:12:47.744Z [INFO] docker/driver.go:306: docker: created container: container_id=f01c56979f211e45d028697d581d8137d4564b5c91995c62a6ebcfdd1add9904 2021-09-13T07:12:47.802Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" === CONT TestDockerDriver_Capabilities/permissive-allowlist driver_test.go:1471: Error Trace: driver_test.go:1471 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Capabilities/permissive-allowlist 2021-09-13T07:12:55.322Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown --- FAIL: TestDockerDriver_Capabilities/permissive-allowlist (8.21s) === FAIL: drivers/docker TestDockerDriver_Capabilities/permissive-allowlist-add-all (10.99s) docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:12:55.339Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:12:55.349Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3397296278/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:12:55.349Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3397296278/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:12:55.369Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3397296278/alloc:/alloc, /tmp/nomad_driver_harness-3397296278/redis-demo/local:/local, /tmp/nomad_driver_harness-3397296278/redis-demo/secrets:/secrets] 2021-09-13T07:12:55.369Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:12:55.369Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:12:55.369Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3397296278/alloc:/alloc", "/tmp/nomad_driver_harness-3397296278/redis-demo/local:/local", "/tmp/nomad_driver_harness-3397296278/redis-demo/secrets:/secrets"}" 2021-09-13T07:12:55.370Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:12:55.370Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9182 label=main 2021-09-13T07:12:55.370Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9182 label=main 2021-09-13T07:12:55.370Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9183 label=REDIS 2021-09-13T07:12:55.370Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9183 label=REDIS 2021-09-13T07:12:55.370Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:12:55.370Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:84a9656a-5b53-094b-239e-306d215eff58] 2021-09-13T07:12:55.370Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-84a9656a-5b53-094b-239e-306d215eff58 === CONT TestDockerDriver_Capabilities/permissive-allowlist-add-all driver_test.go:1471: Error Trace: driver_test.go:1471 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Capabilities/permissive-allowlist-add-all --- FAIL: TestDockerDriver_Capabilities/permissive-allowlist-add-all (10.99s) 2021-09-13T07:13:06.309Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown === FAIL: drivers/docker TestDockerDriver_Capabilities (87.55s) === PAUSE TestDockerDriver_Capabilities === CONT TestDockerDriver_Capabilities 2021-09-13T07:11:38.760Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 === FAIL: drivers/docker TestDockerDriver_LoggingConfiguration (10.23s) === PAUSE TestDockerDriver_LoggingConfiguration === CONT TestDockerDriver_LoggingConfiguration 2021-09-13T07:12:56.769Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:12:56.793Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:12:56.796Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4292049828/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:12:56.796Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-4292049828/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:12:56.840Z [DEBUG] docker/coordinator.go:250: docker: image reference count incremented: image_name=busybox:1.29.3 image_id=sha256:59788edf1f3e78cd0ebe6ce1446e9d10788225db3dedcfd1a59f764bad2b2690 references=1 2021-09-13T07:12:56.840Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-4292049828/alloc:/alloc, /tmp/nomad_driver_harness-4292049828/redis-demo/local:/local, /tmp/nomad_driver_harness-4292049828/redis-demo/secrets:/secrets] 2021-09-13T07:12:56.840Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:12:56.840Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:12:56.840Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-4292049828/alloc:/alloc", "/tmp/nomad_driver_harness-4292049828/redis-demo/local:/local", "/tmp/nomad_driver_harness-4292049828/redis-demo/secrets:/secrets"}" 2021-09-13T07:12:56.840Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:12:56.840Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9184 label=main 2021-09-13T07:12:56.840Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9184 label=main 2021-09-13T07:12:56.840Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9185 label=REDIS 2021-09-13T07:12:56.840Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9185 label=REDIS 2021-09-13T07:12:56.840Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:12:56.840Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels=map[com.hashicorp.nomad.alloc_id:da2f88e0-b628-a826-c6c2-d1fc9aed63c8] 2021-09-13T07:12:56.840Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-da2f88e0-b628-a826-c6c2-d1fc9aed63c8 2021-09-13T07:12:57.487Z [INFO] docker/driver.go:306: docker: created container: container_id=6e579a587de55aea7609b59856b3cf2a7d4eeeb61a00b3309c1d7012ce36e1fc 2021-09-13T07:12:59.156Z [INFO] docker/driver.go:306: docker: created container: container_id=f6c0fb97ee95356e4bde83bcdf40ae8b35f9340e4d363c32b3204b52dc08bc9b 2021-09-13T07:13:01.241Z [INFO] docker/driver.go:347: docker: started container: container_id=6e579a587de55aea7609b59856b3cf2a7d4eeeb61a00b3309c1d7012ce36e1fc 2021-09-13T07:13:01.241Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:13:01.242Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=51371 2021-09-13T07:13:01.242Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:13:01.271Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin799247273 network=unix @module=docker_logger timestamp=2021-09-13T07:13:01.271Z 2021-09-13T07:13:01.271Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:13:01.271Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:13:01.272Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:13:01.272Z 2021-09-13T07:13:01.280Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:13:01.281Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=51371 2021-09-13T07:13:01.281Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:13:01.591Z [INFO] docker/driver.go:347: docker: started container: container_id=f6c0fb97ee95356e4bde83bcdf40ae8b35f9340e4d363c32b3204b52dc08bc9b 2021-09-13T07:13:01.591Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:13:01.593Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=51379 2021-09-13T07:13:01.593Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:13:01.629Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin2290135941 network=unix @module=docker_logger timestamp=2021-09-13T07:13:01.629Z 2021-09-13T07:13:01.630Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:13:01.631Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:13:01.637Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:13:01.637Z 2021-09-13T07:13:01.641Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:13:01.643Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=51379 2021-09-13T07:13:01.643Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_LoggingConfiguration driver_test.go:866: Error Trace: driver_test.go:866 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_LoggingConfiguration 2021-09-13T07:13:06.992Z [DEBUG] docker/coordinator.go:273: docker: image id reference count decremented: image_id=sha256:59788edf1f3e78cd0ebe6ce1446e9d10788225db3dedcfd1a59f764bad2b2690 references=0 2021-09-13T07:13:06.998Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown === FAIL: drivers/docker TestDockerDriver_ExtraLabels (12.10s) === PAUSE TestDockerDriver_ExtraLabels === CONT TestDockerDriver_ExtraLabels docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:13:06.336Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:13:06.339Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1866708848/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:13:06.339Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1866708848/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:13:06.367Z [DEBUG] docker/coordinator.go:250: docker: image reference count incremented: image_name=busybox:1.29.3 image_id=sha256:59788edf1f3e78cd0ebe6ce1446e9d10788225db3dedcfd1a59f764bad2b2690 references=1 2021-09-13T07:13:06.367Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-1866708848/alloc:/alloc, /tmp/nomad_driver_harness-1866708848/redis-demo/local:/local, /tmp/nomad_driver_harness-1866708848/redis-demo/secrets:/secrets] 2021-09-13T07:13:06.367Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:13:06.367Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:13:06.367Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-1866708848/alloc:/alloc", "/tmp/nomad_driver_harness-1866708848/redis-demo/local:/local", "/tmp/nomad_driver_harness-1866708848/redis-demo/secrets:/secrets"}" 2021-09-13T07:13:06.367Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:13:06.367Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9186 label=main 2021-09-13T07:13:06.367Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9186 label=main 2021-09-13T07:13:06.367Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9187 label=REDIS 2021-09-13T07:13:06.367Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9187 label=REDIS 2021-09-13T07:13:06.367Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:13:06.367Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels="map[com.hashicorp.nomad.alloc_id:ee510313-1668-ba88-d7c3-c77dd58ea247 com.hashicorp.nomad.job_name: com.hashicorp.nomad.task_group_name: com.hashicorp.nomad.task_name:redis-demo]" 2021-09-13T07:13:06.367Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-ee510313-1668-ba88-d7c3-c77dd58ea247 === CONT TestDockerDriver_ExtraLabels driver_test.go:825: Error Trace: driver_test.go:825 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_ExtraLabels 2021-09-13T07:13:17.953Z [DEBUG] docker/coordinator.go:273: docker: image id reference count decremented: image_id=sha256:59788edf1f3e78cd0ebe6ce1446e9d10788225db3dedcfd1a59f764bad2b2690 references=0 === FAIL: drivers/docker TestDockerDriver_Labels (11.42s) === PAUSE TestDockerDriver_Labels === CONT TestDockerDriver_Labels docker.go:36: Successfully connected to docker daemon running version 20.10.7 docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:13:07.041Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:13:07.044Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3615312763/alloc/logs/.redis-demo.stdout.fifo 2021-09-13T07:13:07.044Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3615312763/alloc/logs/.redis-demo.stderr.fifo 2021-09-13T07:13:07.149Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=redis-demo volumes=[/tmp/nomad_driver_harness-3615312763/alloc:/alloc, /tmp/nomad_driver_harness-3615312763/redis-demo/local:/local, /tmp/nomad_driver_harness-3615312763/redis-demo/secrets:/secrets] 2021-09-13T07:13:07.149Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=redis-demo 2021-09-13T07:13:07.149Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=redis-demo memory=268435456 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:13:07.149Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=redis-demo binds="[]string{"/tmp/nomad_driver_harness-3615312763/alloc:/alloc", "/tmp/nomad_driver_harness-3615312763/redis-demo/local:/local", "/tmp/nomad_driver_harness-3615312763/redis-demo/secrets:/secrets"}" 2021-09-13T07:13:07.149Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=redis-demo 2021-09-13T07:13:07.149Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9188 label=main 2021-09-13T07:13:07.149Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9188 label=main 2021-09-13T07:13:07.149Z [DEBUG] docker/ports.go:61: docker: allocated static port: task_name=redis-demo ip=127.0.0.1 port=9189 label=REDIS 2021-09-13T07:13:07.149Z [DEBUG] docker/ports.go:65: docker: exposed port: task_name=redis-demo port=9189 label=REDIS 2021-09-13T07:13:07.149Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=redis-demo command="nc -l -p 3000 127.0.0.1" 2021-09-13T07:13:07.149Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=redis-demo labels="map[com.hashicorp.nomad.alloc_id:0b59891a-a91a-7615-702c-88e0facc5dac label1:value1 label2:value2]" 2021-09-13T07:13:07.149Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=redis-demo container_name=redis-demo-0b59891a-a91a-7615-702c-88e0facc5dac 2021-09-13T07:13:09.728Z [INFO] docker/driver.go:306: docker: created container: container_id=ab8bf699133de40d4f4244a3621e07959e59315674f057440cc07a2b8fac97f9 2021-09-13T07:13:09.731Z [INFO] docker/driver.go:306: docker: created container: container_id=e4954906ecf55b5d79ae17d412f5a5512c11e5f15672d431e37b2304cedfc6f9 2021-09-13T07:13:12.728Z [INFO] docker/driver.go:347: docker: started container: container_id=e4954906ecf55b5d79ae17d412f5a5512c11e5f15672d431e37b2304cedfc6f9 2021-09-13T07:13:12.728Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:13:12.728Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=51791 2021-09-13T07:13:12.729Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:13:12.730Z [INFO] docker/driver.go:347: docker: started container: container_id=ab8bf699133de40d4f4244a3621e07959e59315674f057440cc07a2b8fac97f9 2021-09-13T07:13:12.730Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:13:12.746Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=51797 2021-09-13T07:13:12.746Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:13:12.751Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:13:12.751Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin741032517 network=unix timestamp=2021-09-13T07:13:12.751Z 2021-09-13T07:13:12.753Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:13:12.755Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:13:12.754Z 2021-09-13T07:13:12.762Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:13:12.765Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin2073696683 network=unix timestamp=2021-09-13T07:13:12.765Z 2021-09-13T07:13:12.766Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:13:12.766Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:13:12.767Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:13:12.767Z 2021-09-13T07:13:12.772Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:13:12.773Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=51797 2021-09-13T07:13:12.773Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited 2021-09-13T07:13:12.774Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=51791 2021-09-13T07:13:12.774Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited driver_test.go:795: Error Trace: driver_test.go:795 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_Labels === FAIL: drivers/docker TestDockerDriver_StartNVersions (14.38s) === PAUSE TestDockerDriver_StartNVersions === CONT TestDockerDriver_StartNVersions 2021-09-13T07:13:18.411Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown === CONT TestDockerDriver_StartNVersions docker.go:36: Successfully connected to docker daemon running version 20.10.7 driver_test.go:740: Starting 3 tasks 2021-09-13T07:13:18.444Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins === CONT TestDockerDriver_StartNVersions driver_test.go:753: Error Trace: driver_test.go:753 Error: Received unexpected error: task never transitioned to running, currently 'unknown' Test: TestDockerDriver_StartNVersions 2021-09-13T07:13:32.794Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown === FAIL: drivers/docker TestDockerDriver_Start_Wait_AllocDir (5.57s) === PAUSE TestDockerDriver_Start_Wait_AllocDir === CONT TestDockerDriver_Start_Wait_AllocDir 2021-09-13T07:13:43.651Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:13:43.671Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:13:43.677Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1173946957/alloc/logs/.busybox-demo.stdout.fifo 2021-09-13T07:13:43.677Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-1173946957/alloc/logs/.busybox-demo.stderr.fifo 2021-09-13T07:13:43.760Z [TRACE] docker/driver.go:797: docker: binding volumes: task_name=busybox-demo volumes=[/tmp/nomad_driver_harness-1173946957/alloc:/alloc, /tmp/nomad_driver_harness-1173946957/busybox-demo/local:/local, /tmp/nomad_driver_harness-1173946957/busybox-demo/secrets:/secrets] 2021-09-13T07:13:43.760Z [TRACE] docker/driver.go:896: docker: no docker log driver provided, defaulting to plugin config: task_name=busybox-demo 2021-09-13T07:13:43.760Z [DEBUG] docker/driver.go:901: docker: configured resources: task_name=busybox-demo memory=10485760 memory_reservation=0 cpu_shares=512 cpu_quota=0 cpu_period=0 2021-09-13T07:13:43.760Z [DEBUG] docker/driver.go:906: docker: binding directories: task_name=busybox-demo binds="[]string{"/tmp/nomad_driver_harness-1173946957/alloc:/alloc", "/tmp/nomad_driver_harness-1173946957/busybox-demo/local:/local", "/tmp/nomad_driver_harness-1173946957/busybox-demo/secrets:/secrets"}" 2021-09-13T07:13:43.760Z [DEBUG] docker/driver.go:1073: docker: networking mode not specified; using default: task_name=busybox-demo 2021-09-13T07:13:43.760Z [DEBUG] docker/driver.go:1127: docker: setting container startup command: task_name=busybox-demo command="sh -c sleep 1; echo -n win > $NOMAD_ALLOC_DIR/output.txt" 2021-09-13T07:13:43.760Z [DEBUG] docker/driver.go:1170: docker: applied labels on the container: task_name=busybox-demo labels=map[com.hashicorp.nomad.alloc_id:2961857b-0117-ec24-c144-3c4d7ed34e0b] 2021-09-13T07:13:43.760Z [DEBUG] docker/driver.go:1175: docker: setting container name: task_name=busybox-demo container_name=busybox-demo-2961857b-0117-ec24-c144-3c4d7ed34e0b 2021-09-13T07:13:46.387Z [INFO] docker/driver.go:306: docker: created container: container_id=5eb31d0b4da6d5248f530aeeb26f47dfcc600c5fc659e2eb2af07b88c87c4e27 2021-09-13T07:13:47.014Z [INFO] docker/driver.go:306: docker: created container: container_id=46b65acbed5e0489dacd927a01e8ebf8ac32c3ac3e77d4c4cec560c09264a665 2021-09-13T07:13:49.134Z [INFO] docker/driver.go:347: docker: started container: container_id=46b65acbed5e0489dacd927a01e8ebf8ac32c3ac3e77d4c4cec560c09264a665 2021-09-13T07:13:49.134Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:13:49.135Z [INFO] docker/driver.go:347: docker: started container: container_id=5eb31d0b4da6d5248f530aeeb26f47dfcc600c5fc659e2eb2af07b88c87c4e27 2021-09-13T07:13:49.135Z [DEBUG] go-plugin@v1.4.0/client.go:574: docker.docker_logger: starting plugin: path=/tmp/go-build124451399/b1077/docker.test args=[/tmp/go-build124451399/b1077/docker.test, docker_logger] 2021-09-13T07:13:49.138Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=52970 2021-09-13T07:13:49.138Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:13:49.139Z [DEBUG] go-plugin@v1.4.0/client.go:582: docker.docker_logger: plugin started: path=/tmp/go-build124451399/b1077/docker.test pid=52971 2021-09-13T07:13:49.139Z [DEBUG] go-plugin@v1.4.0/client.go:675: docker.docker_logger: waiting for RPC address: path=/tmp/go-build124451399/b1077/docker.test 2021-09-13T07:13:49.157Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: address=/tmp/plugin930505171 network=unix @module=docker_logger timestamp=2021-09-13T07:13:49.157Z 2021-09-13T07:13:49.157Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:13:49.158Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:13:49.159Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:13:49.159Z 2021-09-13T07:13:49.167Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:13:49.174Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=52971 2021-09-13T07:13:49.174Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === CONT TestDockerDriver_Start_Wait_AllocDir driver_test.go:533: Error Trace: driver_test.go:533 Error: ExitResult should be successful: &{1 0 false Docker container exited with non-zero exit code: 1} Test: TestDockerDriver_Start_Wait_AllocDir === FAIL: drivers/docker TestDockerCoordinator_Pull_Remove (panic) === PAUSE TestDockerCoordinator_Pull_Remove === FAIL: drivers/docker TestDockerDriver_Start_WaitFinish (panic) === PAUSE TestDockerDriver_Start_WaitFinish === FAIL: drivers/docker TestDockerDriver_Start_Wait (panic) === PAUSE TestDockerDriver_Start_Wait === FAIL: drivers/docker TestDockerCoordinator_No_Cleanup (panic) === PAUSE TestDockerCoordinator_No_Cleanup === FAIL: drivers/docker TestDockerCoordinator_Remove_Cancel (panic) === PAUSE TestDockerCoordinator_Remove_Cancel === FAIL: drivers/docker TestDockerDriver_Start_BadPull_Recoverable (panic) === PAUSE TestDockerDriver_Start_BadPull_Recoverable === CONT TestDockerDriver_Start_BadPull_Recoverable 2021-09-13T07:13:49.186Z [DEBUG] go-plugin@v1.4.0/client.go:723: docker.docker_logger: using plugin: version=2 2021-09-13T07:13:49.186Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: plugin address: @module=docker_logger address=/tmp/plugin2842575508 network=unix timestamp=2021-09-13T07:13:49.185Z 2021-09-13T07:13:49.186Z [TRACE] eventer/eventer.go:68: docker: task event loop shutdown 2021-09-13T07:13:49.192Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: docker.docker_logger.stdio: waiting for stdio data 2021-09-13T07:13:49.193Z [DEBUG] go-plugin@v1.4.0/client.go:1022: docker.docker_logger.docker.test: using client connection initialized from environment: @module=docker_logger timestamp=2021-09-13T07:13:49.193Z === CONT TestDockerDriver_Start_BadPull_Recoverable docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:13:49.206Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:13:49.207Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: docker.docker_logger.stdio: received EOF, stopping recv loop: err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:13:49.210Z [DEBUG] go-plugin@v1.4.0/client.go:635: docker.docker_logger: plugin process exited: path=/tmp/go-build124451399/b1077/docker.test pid=52970 2021-09-13T07:13:49.210Z [DEBUG] go-plugin@v1.4.0/client.go:454: docker.docker_logger: plugin exited === FAIL: drivers/docker TestDockerDriver_Start_StoppedContainer (panic) === PAUSE TestDockerDriver_Start_StoppedContainer === FAIL: drivers/docker TestDockerDriver_PidsLimit (panic) === PAUSE TestDockerDriver_PidsLimit === FAIL: drivers/docker TestDockerDriver_Start_LoadImage (panic) === PAUSE TestDockerDriver_Start_LoadImage === CONT TestDockerDriver_Start_LoadImage docker.go:36: Successfully connected to docker daemon running version 20.10.7 2021-09-13T07:13:49.258Z [WARN] loader/init.go:224: plugin_loader: skipping external plugins since plugin_dir doesn't exist: plugin_dir=./plugins 2021-09-13T07:13:49.261Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3993143505/alloc/logs/.busybox-demo.stdout.fifo 2021-09-13T07:13:49.261Z [INFO] logmon/logmon.go:198: driver_harness.logmon: opening fifo: path=/tmp/nomad_driver_harness-3993143505/alloc/logs/.busybox-demo.stderr.fifo panic: Fail in goroutine after TestDockerDriver_Start_Kill_Wait has completed goroutine 15072 [running]: testing.(*common).Fail(0x40001f1a00) /usr/local/go/src/testing/testing.go:710 +0x100 testing.(*common).Errorf(0x40001f1a00, {0x105b191, 0x3}, {0x40016e4b80, 0x1, 0x1}) /usr/local/go/src/testing/testing.go:818 +0x78 github.com/stretchr/testify/assert.Fail({0x11d9c00, 0x40001f1a00}, {0x40008a91f0, 0x63}, {0x0, 0x0, 0x0}) /home/ubuntu/go/pkg/mod/github.com/stretchr/testify@v1.7.0/assert/assertions.go:262 +0x3a0 github.com/stretchr/testify/assert.NoError({0x11d9c00, 0x40001f1a00}, {0x11d8b80, 0x400008b140}, {0x0, 0x0, 0x0}) /home/ubuntu/go/pkg/mod/github.com/stretchr/testify@v1.7.0/assert/assertions.go:1332 +0xe0 github.com/stretchr/testify/require.NoError({0x11e5798, 0x40001f1a00}, {0x11d8b80, 0x400008b140}, {0x0, 0x0, 0x0}) /home/ubuntu/go/pkg/mod/github.com/stretchr/testify@v1.7.0/require/require.go:1231 +0xb0 github.com/hashicorp/nomad/drivers/docker.TestDockerDriver_Start_Kill_Wait.func1(0x40015a65f0, 0x400083a780, 0x40001f1a00) /home/ubuntu/go/src/github.com/hashicorp/nomad/drivers/docker/driver_test.go:582 +0xa8 created by github.com/hashicorp/nomad/drivers/docker.TestDockerDriver_Start_Kill_Wait /home/ubuntu/go/src/github.com/hashicorp/nomad/drivers/docker/driver_test.go:576 +0x3b4 === FAIL: drivers/docker/docklog TestDockerLogger_LoggingNotSupported (10.93s) docker.go:36: Successfully connected to docker daemon running version 20.10.7 === PAUSE TestDockerLogger_LoggingNotSupported === CONT TestDockerLogger_LoggingNotSupported 2021-09-13T07:08:55.844Z [DEBUG] docklog/docker_logger.go:230: using client connection initialized from environment 2021-09-13T07:08:55.871Z [DEBUG] docklog/docker_logger.go:230: using client connection initialized from environment === CONT TestDockerLogger_LoggingNotSupported docker_logger_test.go:293: Error Trace: docker_logger_test.go:293 Error: timedout while waiting for docker_logging to terminate Test: TestDockerLogger_LoggingNotSupported === FAIL: drivers/exec TestExec_ExecTaskStreaming/basic:_tty:_children_processes (3.06s) exec_testing.go:340: received stdout: from main 2021-09-13T07:12:49.738Z [DEBUG] go-plugin@v1.4.0/grpc_stdio.go:139: exec.executor.stdio: received EOF, stopping recv loop: alloc_id= task_name=sleep err="rpc error: code = Unavailable desc = transport is closing" 2021-09-13T07:12:49.740Z [DEBUG] go-plugin@v1.4.0/client.go:635: exec.executor: plugin process exited: alloc_id= task_name=sleep path=/tmp/go-build124451399/b1084/exec.test pid=49720 2021-09-13T07:12:49.740Z [DEBUG] go-plugin@v1.4.0/client.go:454: exec.executor: plugin exited: alloc_id= task_name=sleep exec_testing.go:340: received stdout: from background exec_testing.go:124: Error Trace: exec_testing.go:124 Error: Not equal: expected: "from main\r\n" actual : "from main\r\nfrom background\r\n" Diff: --- Expected +++ Actual @@ -1,2 +1,3 @@ from main +from background Test: TestExec_ExecTaskStreaming/basic:_tty:_children_processes --- FAIL: TestExec_ExecTaskStreaming/basic:_tty:_children_processes (3.06s) === FAIL: drivers/exec TestExec_ExecTaskStreaming (56.71s) === PAUSE TestExec_ExecTaskStreaming === CONT TestExec_ExecTaskStreaming 2021-09-13T07:12:01.603Z [TRACE] eventer/eventer.go:68: exec: task event loop shutdown 2021-09-13T07:12:24.713Z [INFO] exec/driver.go:445: exec: starting task: driver_cfg="{Command:/bin/sleep Args:[9000] ModePID: ModeIPC: CapAdd:[] CapDrop:[]}" 2021-09-13T07:12:24.713Z [DEBUG] go-plugin@v1.4.0/client.go:574: exec.executor: starting plugin: alloc_id= task_name=sleep path=/tmp/go-build124451399/b1084/exec.test args=[/tmp/go-build124451399/b1084/exec.test, executor, {"LogFile":"/tmp/nomad_driver_harness-1329972392/sleep/executor.out","LogLevel":"debug","FSIsolation":true}] 2021-09-13T07:12:24.713Z [DEBUG] go-plugin@v1.4.0/client.go:582: exec.executor: plugin started: alloc_id= task_name=sleep path=/tmp/go-build124451399/b1084/exec.test pid=49720 2021-09-13T07:12:24.713Z [DEBUG] go-plugin@v1.4.0/client.go:675: exec.executor: waiting for RPC address: alloc_id= task_name=sleep path=/tmp/go-build124451399/b1084/exec.test 2021-09-13T07:12:25.300Z [DEBUG] go-plugin@v1.4.0/client.go:723: exec.executor: using plugin: alloc_id= task_name=sleep version=2 time="2021-09-13T07:12:25Z" level=info msg="detected 127.0.0.53 nameserver, assuming systemd-resolved, so using resolv.conf: /run/systemd/resolve/resolv.conf" 2021-09-13T07:12:25.304Z [DEBUG] exec/driver.go:482: exec: task capabilities: capabilities=[] 2021-09-13T07:12:25.304Z [TRACE] go-plugin@v1.4.0/grpc_stdio.go:131: exec.executor.stdio: waiting for stdio data: alloc_id= task_name=sleep === FAIL: drivers/exec TestExecDriver_Stats (50.47s) === PAUSE TestExecDriver_Stats === CONT TestExecDriver_Stats 2021-09-13T07:14:55.406Z [TRACE] eventer/eventer.go:68: exec: task event loop shutdown === CONT TestExecDriver_Stats driver_test.go:428: Error Trace: driver_test.go:428 Error: Should not be zero, but was 0 Test: TestExecDriver_Stats 2021-09-13T07:15:45.878Z [TRACE] eventer/eventer.go:68: exec: task event loop shutdown === FAIL: drivers/shared/executor TestExecutor_Start_Wait/LibcontainerExecutor (5.35s) 2021-09-13T07:17:11.269Z [TRACE] executor/executor_linux.go:89: isolated_executor: preparing to launch command: command=/bin/echo args="hello world" 2021-09-13T07:17:11.269Z [DEBUG] executor/executor_linux.go:160: isolated_executor: launching: command=/bin/echo args="hello world" 2021-09-13T07:17:16.204Z [TRACE] executor/executor_linux.go:89: isolated_executor: preparing to launch command: command=/bin/ls args="-F / /etc/" 2021-09-13T07:17:16.204Z [DEBUG] executor/executor_linux.go:160: isolated_executor: launching: command=/bin/ls args="-F / /etc/" time="2021-09-13T07:17:16Z" level=warning msg="cannot toggle freezer: cgroups not configured for container" time="2021-09-13T07:17:16Z" level=warning msg="cannot toggle freezer: cgroups not configured for container" time="2021-09-13T07:17:16Z" level=warning msg="lstat : no such file or directory" executor_test.go:208: Error Trace: executor_test.go:208 wait.go:32 wait.go:18 executor_test.go:201 Error: Received unexpected error: expected: 'hello world' actual: '' Test: TestExecutor_Start_Wait/LibcontainerExecutor time="2021-09-13T07:17:16Z" level=warning msg="cannot toggle freezer: cgroups not configured for container" time="2021-09-13T07:17:16Z" level=warning msg="cannot toggle freezer: cgroups not configured for container" time="2021-09-13T07:17:16Z" level=warning msg="lstat : no such file or directory" --- FAIL: TestExecutor_Start_Wait/LibcontainerExecutor (5.35s) === FAIL: drivers/shared/executor TestExecutor_Start_Wait (5.38s) === PAUSE TestExecutor_Start_Wait === CONT TestExecutor_Start_Wait === FAIL: drivers/shared/executor TestExecutor_Start_NonExecutableBinaries/LibcontainerExecutor (5.46s) 2021-09-13T07:17:32.892Z [TRACE] executor/executor_linux.go:89: isolated_executor: preparing to launch command: command=/tmp/nomad-executor-tests1267024149/nonexecutablefile args= 2021-09-13T07:17:32.892Z [DEBUG] executor/executor_linux.go:160: isolated_executor: launching: command=/tmp/nomad-executor-tests1267024149/nonexecutablefile args= 2021-09-13T07:17:34.471Z [TRACE] executor/executor_linux.go:89: isolated_executor: preparing to launch command: command=/bin/ls args="-F / /etc/" 2021-09-13T07:17:34.471Z [DEBUG] executor/executor_linux.go:160: isolated_executor: launching: command=/bin/ls args="-F / /etc/" executor_test.go:640: stderr: standard_init_linux.go:219: exec user process caused: no such file or directory 2021-09-13T07:17:32.978Z [ERROR] failed to initialize libcontainer-shim: error="standard_init_linux.go:219: exec user process caused: no such file or directory" executor_test.go:641: Error Trace: executor_test.go:641 wait.go:32 wait.go:18 executor_test.go:632 Error: Received unexpected error: expected: 'hello world' actual: '' Test: TestExecutor_Start_NonExecutableBinaries/LibcontainerExecutor time="2021-09-13T07:17:38Z" level=warning msg="cannot toggle freezer: cgroups not configured for container" time="2021-09-13T07:17:38Z" level=warning msg="cannot toggle freezer: cgroups not configured for container" time="2021-09-13T07:17:38Z" level=warning msg="lstat : no such file or directory" --- FAIL: TestExecutor_Start_NonExecutableBinaries/LibcontainerExecutor (5.46s) === FAIL: drivers/shared/executor TestExecutor_Start_NonExecutableBinaries (5.48s) === PAUSE TestExecutor_Start_NonExecutableBinaries === CONT TestExecutor_Start_NonExecutableBinaries === FAIL: drivers/shared/executor TestExecutor_IsolationAndConstraints (7.41s) === PAUSE TestExecutor_IsolationAndConstraints === CONT TestExecutor_IsolationAndConstraints === CONT TestExecutor_IsolationAndConstraints executor_linux_test.go:250: Command output incorrectly: want /: alloc/ bin/ dev/ etc/ lib/ lib64/ local/ proc/ secrets/ sys/ tmp/ usr/ /etc/: ld.so.cache ld.so.conf ld.so.conf.d/ passwd; got /: alloc/ bin/ dev/ etc/ lib/ local/ proc/ secrets/ sys/ tmp/ usr/ /etc/: ld.so.cache ld.so.conf ld.so.conf.d/ passwd === FAIL: drivers/shared/executor TestExecutor_Start_Kill (panic) === PAUSE TestExecutor_Start_Kill === CONT TestExecutor_Start_Kill === FAIL: drivers/shared/executor TestExecutor_WaitExitSignal/UniversalExecutor (panic) 2021-09-13T07:17:40.114Z [TRACE] executor/executor.go:283: executor: preparing to launch command: command=/bin/sleep args=10000 panic: Fail in goroutine after TestExecutor_WaitExitSignal/LibcontainerExecutor has completed goroutine 204 [running]: testing.(*common).Fail(0x40004f2680) /usr/local/go/src/testing/testing.go:710 +0x100 testing.(*common).Errorf(0x40004f2680, {0xdf3420, 0x3}, {0x400053f730, 0x1, 0x1}) /usr/local/go/src/testing/testing.go:818 +0x78 github.com/stretchr/testify/assert.Fail({0xf19720, 0x40004f2680}, {0x40003cb110, 0x22}, {0x4000240e98, 0x1, 0x1}) /home/ubuntu/go/pkg/mod/github.com/stretchr/testify@v1.7.0/assert/assertions.go:262 +0x3a0 github.com/stretchr/testify/assert.NotEmpty({0xf19720, 0x40004f2680}, {0xcd4f40, 0x4000446ba0}, {0x4000240e98, 0x1, 0x1}) /home/ubuntu/go/pkg/mod/github.com/stretchr/testify@v1.7.0/assert/assertions.go:611 +0xf0 github.com/hashicorp/nomad/drivers/shared/executor.TestExecutor_WaitExitSignal.func1.1.1() /home/ubuntu/go/src/github.com/hashicorp/nomad/drivers/shared/executor/executor_test.go:282 +0x194 github.com/hashicorp/nomad/testutil.WaitForResultRetries(0x1f4, 0x4000240f80, 0x4000240f60) /home/ubuntu/go/src/github.com/hashicorp/nomad/testutil/wait.go:26 +0x50 github.com/hashicorp/nomad/testutil.WaitForResult(0x40002daf80, 0x40002daf60) /home/ubuntu/go/src/github.com/hashicorp/nomad/testutil/wait.go:18 +0x68 github.com/hashicorp/nomad/drivers/shared/executor.TestExecutor_WaitExitSignal.func1.1({0xf40680, 0x400021ab60}, 0x40004f2680, 0x40004b01d8) /home/ubuntu/go/src/github.com/hashicorp/nomad/drivers/shared/executor/executor_test.go:273 +0x88 created by github.com/hashicorp/nomad/drivers/shared/executor.TestExecutor_WaitExitSignal.func1 /home/ubuntu/go/src/github.com/hashicorp/nomad/drivers/shared/executor/executor_test.go:272 +0x290 === FAIL: drivers/shared/executor TestExecutor_Start_Kill/UniversalExecutor (panic) 2021-09-13T07:17:38.453Z [TRACE] executor/executor.go:283: executor: preparing to launch command: command=/bin/sleep args=10 2021-09-13T07:17:38.457Z [DEBUG] executor/executor.go:511: executor: shutdown requested: signal=SIGINT grace_period_ms=100ms === FAIL: drivers/shared/executor TestExecutor_Start_Wait_Children (panic) === PAUSE TestExecutor_Start_Wait_Children === FAIL: drivers/shared/executor TestExecutor_WaitExitSignal (panic) === PAUSE TestExecutor_WaitExitSignal === CONT TestExecutor_WaitExitSignal === FAIL: drivers/shared/executor TestExecutor_WaitExitSignal/LibcontainerExecutor (panic) 2021-09-13T07:17:40.081Z [TRACE] executor/executor_linux.go:89: isolated_executor: preparing to launch command: command=/bin/sleep args=10000 2021-09-13T07:17:40.081Z [DEBUG] executor/executor_linux.go:160: isolated_executor: launching: command=/bin/sleep args=10000 executor_test.go:303: Error Trace: executor_test.go:303 Error: Not equal: expected: 0 actual : 9 Test: TestExecutor_WaitExitSignal/LibcontainerExecutor DONE 5965 tests, 44 skipped, 105 failures in 794.943s make: *** [GNUmakefile:280: test-nomad] Error 1