Nomad not being able to allocate tasks even though there's plenty of memory on node

Why would Nomad be throwing the errors for a task if there’s plenty of memory available?

  * Resources exhausted on 1 nodes
  * Dimension "memory" exhausted on 1 nodes

Here’s the response of http://localhost:4646/v1/node/ID if it helps

{
  "Attributes": {
    "consul.connect": "true",
    "consul.datacenter": "dc1",
    "consul.ft.namespaces": "false",
    "consul.grpc": "8502",
    "consul.revision": "bbcbb733b",
    "consul.server": "true",
    "consul.sku": "oss",
    "consul.version": "1.9.6",
    "cpu.arch": "amd64",
    "cpu.frequency": "2199",
    "cpu.modelname": "DO-Regular",
    "cpu.numcores": "2",
    "cpu.totalcompute": "4398",
    "driver.docker": "1",
    "driver.docker.bridge_ip": "172.17.0.1",
    "driver.docker.os_type": "linux",
    "driver.docker.privileged.enabled": "true",
    "driver.docker.runtimes": "io.containerd.runc.v2,io.containerd.runtime.v1.linux,runc",
    "driver.docker.version": "20.10.7",
    "driver.docker.volumes.enabled": "true",
    "driver.exec": "1",
    "driver.raw_exec": "1",
    "kernel.name": "linux",
    "kernel.version": "5.4.0-91-generic",
    "memory.totalbytes": "4127391744",
    "nomad.advertise.address": "10.46.0.7:4646",
    "nomad.revision": "0e11f8d5173e78dce23433903d086e3f9dfea7ea",
    "nomad.version": "1.2.0",
    "os.name": "ubuntu",
    "os.signals": "SIGILL,SIGIOT,SIGUSR2,SIGSTOP,SIGTRAP,SIGURG,SIGXCPU,SIGABRT,SIGQUIT,SIGSEGV,SIGTTIN,SIGCHLD,SIGFPE,SIGSYS,SIGUSR1,SIGINT,SIGIO,SIGPIPE,SIGALRM,SIGKILL,SIGXFSZ,SIGPROF,SIGTSTP,SIGWINCH,SIGHUP,SIGTTOU,SIGBUS,SIGCONT,SIGTERM",
    "os.version": "20.04",
    "unique.cgroup.mountpoint": "/sys/fs/cgroup/systemd",
    "unique.consul.name": "ubuntu-s-1vcpu-2gb-sfo2-01",
    "unique.hostname": "ubuntu-s-1vcpu-2gb-sfo2-01",
    "unique.network.ip-address": "x.x.x.x",
    "unique.storage.bytesfree": "11104468992",
    "unique.storage.bytestotal": "51848359936",
    "unique.storage.volume": "/dev/vda1",
    "vault.accessible": "true",
    "vault.cluster_id": "51d941b1-6c5c-7229-d4ab-ac1e00d1747d",
    "vault.cluster_name": "vault-cluster-ef071555",
    "vault.version": "1.7.3"
  },
  "CSIControllerPlugins": null,
  "CSINodePlugins": null,
  "ComputedClass": "v1:6027288472736180174",
  "CreateIndex": 7,
  "Datacenter": "dc1",
  "Drain": false,
  "DrainStrategy": null,
  "Drivers": {
    "docker": {
      "Attributes": {
        "driver.docker": "true",
        "driver.docker.bridge_ip": "172.17.0.1",
        "driver.docker.os_type": "linux",
        "driver.docker.privileged.enabled": "true",
        "driver.docker.runtimes": "io.containerd.runc.v2,io.containerd.runtime.v1.linux,runc",
        "driver.docker.version": "20.10.7",
        "driver.docker.volumes.enabled": "true"
      },
      "Detected": true,
      "HealthDescription": "Healthy",
      "Healthy": true,
      "UpdateTime": "2021-12-29T16:45:51.668084692-08:00"
    },
    "exec": {
      "Attributes": {
        "driver.exec": "true"
      },
      "Detected": true,
      "HealthDescription": "Healthy",
      "Healthy": true,
      "UpdateTime": "2021-12-29T16:45:51.641966723-08:00"
    },
    "java": {
      "Attributes": null,
      "Detected": false,
      "HealthDescription": "",
      "Healthy": false,
      "UpdateTime": "2021-12-29T16:45:51.642161489-08:00"
    },
    "qemu": {
      "Attributes": null,
      "Detected": false,
      "HealthDescription": "",
      "Healthy": false,
      "UpdateTime": "2021-12-29T16:45:51.642039394-08:00"
    },
    "raw_exec": {
      "Attributes": {
        "driver.raw_exec": "true"
      },
      "Detected": true,
      "HealthDescription": "Healthy",
      "Healthy": true,
      "UpdateTime": "2021-12-29T16:45:51.642383855-08:00"
    }
  },
  "Events": [
    {
      "CreateIndex": 0,
      "Details": null,
      "Message": "Node registered",
      "Subsystem": "Cluster",
      "Timestamp": "2021-12-28T01:23:06-08:00"
    }
  ],
  "HTTPAddr": "10.46.0.7:4646",
  "HostNetworks": null,
  "HostVolumes": null,
  "ID": "3ea1f8e5-acaa-9c3d-be65-e6ccc798f6e3",
  "LastDrain": null,
  "Links": {
    "consul": "dc1.ubuntu-s-1vcpu-2gb-sfo2-01"
  },
  "Meta": {
    "connect.gateway_image": "envoyproxy/envoy:v${NOMAD_envoy_version}",
    "connect.log_level": "info",
    "connect.proxy_concurrency": "1",
    "connect.sidecar_image": "envoyproxy/envoy:v${NOMAD_envoy_version}",
    "node": "main",
    "volumes_joined": "alpha"
  },
  "ModifyIndex": 4797,
  "Name": "ubuntu-s-1vcpu-2gb-sfo2-01",
  "NodeClass": "",
  "NodeResources": {
    "Cpu": {
      "CpuShares": 4398,
      "ReservableCpuCores": [
        0,
        1
      ],
      "TotalCpuCores": 2
    },
    "Devices": null,
    "Disk": {
      "DiskMB": 10590
    },
    "MaxDynamicPort": 32000,
    "Memory": {
      "MemoryMB": 3936
    },
    "MinDynamicPort": 20000,
    "Networks": [
      {
        "CIDR": "",
        "DNS": null,
        "Device": "",
        "DynamicPorts": null,
        "Hostname": "",
        "IP": "",
        "MBits": 0,
        "Mode": "bridge",
        "ReservedPorts": null
      },
      {
        "CIDR": "x.x.x.x/32",
        "DNS": null,
        "Device": "eth0",
        "DynamicPorts": null,
        "Hostname": "",
        "IP": "x.x.x.x",
        "MBits": 1000,
        "Mode": "host",
        "ReservedPorts": null
      },
      {
        "CIDR": "10.46.0.7/32",
        "DNS": null,
        "Device": "eth0",
        "DynamicPorts": null,
        "Hostname": "",
        "IP": "10.46.0.7",
        "MBits": 1000,
        "Mode": "host",
        "ReservedPorts": null
      }
    ],
    "NodeNetworks": [
      {
        "Addresses": null,
        "Device": "",
        "MacAddress": "",
        "Mode": "bridge",
        "Speed": 0
      },
      {
        "Addresses": [
          {
            "Address": "x.x.x.x",
            "Alias": "default",
            "Family": "ipv4",
            "Gateway": "",
            "ReservedPorts": ""
          },
          {
            "Address": "10.46.0.7",
            "Alias": "default",
            "Family": "ipv4",
            "Gateway": "",
            "ReservedPorts": ""
          }
        ],
        "Device": "eth0",
        "MacAddress": "1a:aa:c9:5d:02:c0",
        "Mode": "host",
        "Speed": 1000
      }
    ]
  },
  "Reserved": {
    "CPU": 0,
    "Cores": 0,
    "Devices": null,
    "DiskMB": 0,
    "IOPS": 0,
    "MemoryMB": 0,
    "MemoryMaxMB": 0,
    "Networks": null
  },
  "ReservedResources": {
    "Cpu": {
      "CpuShares": 0,
      "ReservedCpuCores": null
    },
    "Disk": {
      "DiskMB": 0
    },
    "Memory": {
      "MemoryMB": 0
    },
    "Networks": {
      "ReservedHostPorts": ""
    }
  },
  "Resources": {
    "CPU": 4398,
    "Cores": 0,
    "Devices": null,
    "DiskMB": 10590,
    "IOPS": 0,
    "MemoryMB": 3936,
    "MemoryMaxMB": 0,
    "Networks": [
      {
        "CIDR": "x.x.x.x/32",
        "DNS": null,
        "Device": "eth0",
        "DynamicPorts": null,
        "Hostname": "",
        "IP": "x.x.x.x",
        "MBits": 1000,
        "Mode": "host",
        "ReservedPorts": null
      },
      {
        "CIDR": "10.46.0.7/32",
        "DNS": null,
        "Device": "eth0",
        "DynamicPorts": null,
        "Hostname": "",
        "IP": "10.46.0.7",
        "MBits": 1000,
        "Mode": "host",
        "ReservedPorts": null
      }
    ]
  },
  "SchedulingEligibility": "eligible",
  "SecretID": "",
  "Status": "ready",
  "StatusDescription": "",
  "StatusUpdatedAt": 1640825175,
  "TLSEnabled": false
}

What is the jobspec? Do you have specific resources stanzas for the task?

Hi @axsuul.

Without additional information it’s hard to determine exactly what is happening. Specifically if you could provide the job specification which you are trying to register, as well as the output of the node allocations API call, that would be great.

Thanks,
jrasell and the Nomad team