Why would Nomad be throwing the errors for a task if there’s plenty of memory available?
* Resources exhausted on 1 nodes
* Dimension "memory" exhausted on 1 nodes
Here’s the response of http://localhost:4646/v1/node/ID if it helps
{
"Attributes": {
"consul.connect": "true",
"consul.datacenter": "dc1",
"consul.ft.namespaces": "false",
"consul.grpc": "8502",
"consul.revision": "bbcbb733b",
"consul.server": "true",
"consul.sku": "oss",
"consul.version": "1.9.6",
"cpu.arch": "amd64",
"cpu.frequency": "2199",
"cpu.modelname": "DO-Regular",
"cpu.numcores": "2",
"cpu.totalcompute": "4398",
"driver.docker": "1",
"driver.docker.bridge_ip": "172.17.0.1",
"driver.docker.os_type": "linux",
"driver.docker.privileged.enabled": "true",
"driver.docker.runtimes": "io.containerd.runc.v2,io.containerd.runtime.v1.linux,runc",
"driver.docker.version": "20.10.7",
"driver.docker.volumes.enabled": "true",
"driver.exec": "1",
"driver.raw_exec": "1",
"kernel.name": "linux",
"kernel.version": "5.4.0-91-generic",
"memory.totalbytes": "4127391744",
"nomad.advertise.address": "10.46.0.7:4646",
"nomad.revision": "0e11f8d5173e78dce23433903d086e3f9dfea7ea",
"nomad.version": "1.2.0",
"os.name": "ubuntu",
"os.signals": "SIGILL,SIGIOT,SIGUSR2,SIGSTOP,SIGTRAP,SIGURG,SIGXCPU,SIGABRT,SIGQUIT,SIGSEGV,SIGTTIN,SIGCHLD,SIGFPE,SIGSYS,SIGUSR1,SIGINT,SIGIO,SIGPIPE,SIGALRM,SIGKILL,SIGXFSZ,SIGPROF,SIGTSTP,SIGWINCH,SIGHUP,SIGTTOU,SIGBUS,SIGCONT,SIGTERM",
"os.version": "20.04",
"unique.cgroup.mountpoint": "/sys/fs/cgroup/systemd",
"unique.consul.name": "ubuntu-s-1vcpu-2gb-sfo2-01",
"unique.hostname": "ubuntu-s-1vcpu-2gb-sfo2-01",
"unique.network.ip-address": "x.x.x.x",
"unique.storage.bytesfree": "11104468992",
"unique.storage.bytestotal": "51848359936",
"unique.storage.volume": "/dev/vda1",
"vault.accessible": "true",
"vault.cluster_id": "51d941b1-6c5c-7229-d4ab-ac1e00d1747d",
"vault.cluster_name": "vault-cluster-ef071555",
"vault.version": "1.7.3"
},
"CSIControllerPlugins": null,
"CSINodePlugins": null,
"ComputedClass": "v1:6027288472736180174",
"CreateIndex": 7,
"Datacenter": "dc1",
"Drain": false,
"DrainStrategy": null,
"Drivers": {
"docker": {
"Attributes": {
"driver.docker": "true",
"driver.docker.bridge_ip": "172.17.0.1",
"driver.docker.os_type": "linux",
"driver.docker.privileged.enabled": "true",
"driver.docker.runtimes": "io.containerd.runc.v2,io.containerd.runtime.v1.linux,runc",
"driver.docker.version": "20.10.7",
"driver.docker.volumes.enabled": "true"
},
"Detected": true,
"HealthDescription": "Healthy",
"Healthy": true,
"UpdateTime": "2021-12-29T16:45:51.668084692-08:00"
},
"exec": {
"Attributes": {
"driver.exec": "true"
},
"Detected": true,
"HealthDescription": "Healthy",
"Healthy": true,
"UpdateTime": "2021-12-29T16:45:51.641966723-08:00"
},
"java": {
"Attributes": null,
"Detected": false,
"HealthDescription": "",
"Healthy": false,
"UpdateTime": "2021-12-29T16:45:51.642161489-08:00"
},
"qemu": {
"Attributes": null,
"Detected": false,
"HealthDescription": "",
"Healthy": false,
"UpdateTime": "2021-12-29T16:45:51.642039394-08:00"
},
"raw_exec": {
"Attributes": {
"driver.raw_exec": "true"
},
"Detected": true,
"HealthDescription": "Healthy",
"Healthy": true,
"UpdateTime": "2021-12-29T16:45:51.642383855-08:00"
}
},
"Events": [
{
"CreateIndex": 0,
"Details": null,
"Message": "Node registered",
"Subsystem": "Cluster",
"Timestamp": "2021-12-28T01:23:06-08:00"
}
],
"HTTPAddr": "10.46.0.7:4646",
"HostNetworks": null,
"HostVolumes": null,
"ID": "3ea1f8e5-acaa-9c3d-be65-e6ccc798f6e3",
"LastDrain": null,
"Links": {
"consul": "dc1.ubuntu-s-1vcpu-2gb-sfo2-01"
},
"Meta": {
"connect.gateway_image": "envoyproxy/envoy:v${NOMAD_envoy_version}",
"connect.log_level": "info",
"connect.proxy_concurrency": "1",
"connect.sidecar_image": "envoyproxy/envoy:v${NOMAD_envoy_version}",
"node": "main",
"volumes_joined": "alpha"
},
"ModifyIndex": 4797,
"Name": "ubuntu-s-1vcpu-2gb-sfo2-01",
"NodeClass": "",
"NodeResources": {
"Cpu": {
"CpuShares": 4398,
"ReservableCpuCores": [
0,
1
],
"TotalCpuCores": 2
},
"Devices": null,
"Disk": {
"DiskMB": 10590
},
"MaxDynamicPort": 32000,
"Memory": {
"MemoryMB": 3936
},
"MinDynamicPort": 20000,
"Networks": [
{
"CIDR": "",
"DNS": null,
"Device": "",
"DynamicPorts": null,
"Hostname": "",
"IP": "",
"MBits": 0,
"Mode": "bridge",
"ReservedPorts": null
},
{
"CIDR": "x.x.x.x/32",
"DNS": null,
"Device": "eth0",
"DynamicPorts": null,
"Hostname": "",
"IP": "x.x.x.x",
"MBits": 1000,
"Mode": "host",
"ReservedPorts": null
},
{
"CIDR": "10.46.0.7/32",
"DNS": null,
"Device": "eth0",
"DynamicPorts": null,
"Hostname": "",
"IP": "10.46.0.7",
"MBits": 1000,
"Mode": "host",
"ReservedPorts": null
}
],
"NodeNetworks": [
{
"Addresses": null,
"Device": "",
"MacAddress": "",
"Mode": "bridge",
"Speed": 0
},
{
"Addresses": [
{
"Address": "x.x.x.x",
"Alias": "default",
"Family": "ipv4",
"Gateway": "",
"ReservedPorts": ""
},
{
"Address": "10.46.0.7",
"Alias": "default",
"Family": "ipv4",
"Gateway": "",
"ReservedPorts": ""
}
],
"Device": "eth0",
"MacAddress": "1a:aa:c9:5d:02:c0",
"Mode": "host",
"Speed": 1000
}
]
},
"Reserved": {
"CPU": 0,
"Cores": 0,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 0,
"MemoryMaxMB": 0,
"Networks": null
},
"ReservedResources": {
"Cpu": {
"CpuShares": 0,
"ReservedCpuCores": null
},
"Disk": {
"DiskMB": 0
},
"Memory": {
"MemoryMB": 0
},
"Networks": {
"ReservedHostPorts": ""
}
},
"Resources": {
"CPU": 4398,
"Cores": 0,
"Devices": null,
"DiskMB": 10590,
"IOPS": 0,
"MemoryMB": 3936,
"MemoryMaxMB": 0,
"Networks": [
{
"CIDR": "x.x.x.x/32",
"DNS": null,
"Device": "eth0",
"DynamicPorts": null,
"Hostname": "",
"IP": "x.x.x.x",
"MBits": 1000,
"Mode": "host",
"ReservedPorts": null
},
{
"CIDR": "10.46.0.7/32",
"DNS": null,
"Device": "eth0",
"DynamicPorts": null,
"Hostname": "",
"IP": "10.46.0.7",
"MBits": 1000,
"Mode": "host",
"ReservedPorts": null
}
]
},
"SchedulingEligibility": "eligible",
"SecretID": "",
"Status": "ready",
"StatusDescription": "",
"StatusUpdatedAt": 1640825175,
"TLSEnabled": false
}