Consul exec shows 0/0 node(s) completed / acknowledged

Consul
1

When making a consul exec call I am getting:

0 / 0 node(s) completed / acknowledged

I have set disable_remote_exec to false and configured the proper ACL rules as outlined in the docs:

  resource "consul_acl_policy" "exec" {
  name        = "exec"
  description = "managed by ops/terraform"

  rules = <<-RULE
    agent "" {policy = "read"}
    session "" {policy = "write"}
    key "_rexec" {policy = "write"}
    event "_rexec" {policy = "write"}
  RULE
}

Both client and server consul versions are v1.8.4

client config:

{
  "acl": {
    "enable_token_persistence": true,
    "enabled": true,
    "tokens": {
      "agent": "123",
      "default": "123"
    }
  },
  "bind_addr": "...",
  "ca_file": "...",
  "cert_file": "...",
  "connect": {
    "enabled": true
  },
  "data_dir": "...",
  "datacenter": "...",
  "disable_remote_exec": false,
  "encrypt": "...",
  "key_file": "...",
  "log_level": "INFO",
  "ports": {
    "grpc": ...,
    "https": ...
  },
  "primary_datacenter": "...",
  "retry_join": [
    "...",
    "...",
    "..."
  ],
  "server": false,
  "ui": false,
  "verify_incoming": false,
  "verify_outgoing": true,
  "verify_server_hostname": true
}

server config:

{
  "acl": {
    "default_policy": "deny",
    "enable_token_persistence": true,
    "enabled": true,
    "tokens": {
      "agent": "..."
    }
  },
  "bind_addr": "...",
  "bootstrap_expect": 3,
  "ca_file": "...",
  "cert_file": "...",
  "client_addr": "...",
  "connect": {
    "enabled": true
  },
  "data_dir": "...",
  "datacenter": "...",
  "disable_remote_exec": false,
  "encrypt": "...",
  "key_file": "...",
  "log_level": "INFO",
  "ports": {
    "grpc": ...,
    "https": ...
  },
  "primary_datacenter": "...",
  "retry_join": [
    "...",
    "...",
    "..."
  ],
  "retry_join_wan": [
    "...",
    "...",
    "..."
  ],
  "server": true,
  "ui": true,
  "verify_incoming": false,
  "verify_outgoing": true,
  "verify_server_hostname": true
}
2

Hi @benschinn,

Could you check the policy associated with your agent token to see whether it has write under _rexec key prefix?

key_prefix "_rexec" {
    policy = "write"
}

Every agent should have the permission to read the command from the KV store, and write the result back to the KV store. The nodes uses the agentToken for this.

Looks like the documentation is not explicitly stating this requirement unfortunately.