Troubleshooting MinIO, Redis + Sidecar Combination

Nomad
,
1

Hi,

I’m trying to troubleshoot a simple Nomad job that spins up a Redis Docker container, a MinIO container, and a downstream (Ubuntu) client container that uses the former two as upstreams. I’m able to access the MinIO service locally from the client. However I’m not able to work with Redis.

My job definition follows.

# A minimal job to troubleshoot Redis + sidecar combination
job "example" {
  region      = "global"
  # NOTE: Local cluster is called `dc-local`
  datacenters = ["dc-local"]
  type        = "service"

  # Three groups: Redis, MinIO, and Ubuntu client
  group "redis" {
    count = 1

    network {
      mode = "bridge" # Required for sidecar_service
      # Dynamically allocate port
      port "redis" {}
    }

    service {
      name = "redis"
      port = "redis" # Defined above, in `network` stanza
      connect {
        sidecar_service {}
      }
    }

    task "redis" {
      driver = "docker"
      env {
        REDIS_PASSWORD = "topSecret"
      }

      config {
        image = "docker.io/bitnami/redis:6.2"
        ports = ["redis"] # Defined above, in `network` stanza
      }

      resources {
        cpu    = 500
        memory = 512
      }

    }

  }

  group "minio" {
    count = 1

    network {
      mode = "bridge" # Required for sidecar_service
      port "api" {}
      port "console" {}
    }

    service {
      name = "minio"
      port = "api" # Defined above, in `network` stanza
      connect {
        sidecar_service {}
      }
    }

    service {
      name = "minio-console"
      port = "console" # Defined above, in `network` stanza
    }

    task "minio" {
      driver = "docker"

      env {
        MINIO_ROOT_USER     = "admin"
        MINIO_ROOT_PASSWORD = "topSecret"
      }

      config {
        image   = "quay.io/minio/minio"
        ports   = ["api", "console"] # Defined above, in `network` stanza
        command = "server"
        args    = [
          "/tmp/",
          "--address", ":${NOMAD_PORT_api}",
          "--console-address", ":${NOMAD_PORT_console}",
        ]
      }

      resources {
        cpu    = 500
        memory = 512
      }

    }

  }

  group "ubuntu" {
    count = 1

    network {
      mode = "bridge" # Required for sidecar_service
    }

    service {
      name = "client"
      connect {
        sidecar_service {
          proxy {
            upstreams {
              destination_name = "minio"
              local_bind_port  = 7200
            }

            upstreams {
              destination_name = "redis"
              local_bind_port  = 6379
            }
          }
        }
      }
    }

    task "client" {
      driver = "docker"

      config {
        image = "phusion/baseimage:0.11"
      }

      resources {
        cpu    = 500
        memory = 512
      }

    }

  }

}

After the job starts I’m able to connect to MinIO from the client container using localhost. The proxy seems to be working as expected:

vagrant@client2:~$ docker exec -it client-e73d3f42-62bc-8afa-0a44-1df437d46cca /bin/bash
root@b1fda67e0409:/# curl localhost:7200
<?xml version="1.0" encoding="UTF-8"?>
<Error><Code>AccessDenied</Code><Message>Access Denied.</Message><Resource>/</Resource><RequestId>171BCBBECAE728E5</RequestId><HostId>51ee9090-de27-4eef-95b1-f376983dd721</HostId></Error>root@b1fda67e0409:/#

However I’m unable to connect to Redis using redis-cli.

root@b1fda67e0409:/# redis-cli -h localhost -p 6379 -a topSecret
localhost:6379> ping
Error: Connection reset by peer
root@b1fda67e0409:/#

I tried using the address/port of the proxy directly. This revealed a different error message.

root@b1fda67e0409:/# redis-cli -h  172.20.20.21 -p 24991 -a topSecret
172.20.20.21:24991> ping
Error: Protocol error, got "\x15" as reply type byte

My Consul server configuration follows:

$ cat /etc/consul.d/consul.hcl
# NOTE: `ansible_eth1` because `eth1` is the name of the interface created via Vagrant
bind_addr            = "172.20.20.10"
data_dir             = "/var/consul"
datacenter           = "dc-local"
enable_script_checks = true
enable_syslog        = true
leave_on_terminate   = true
log_level            = "DEBUG"
node_name            = "server"
# // @formatter:off
retry_join           = ["172.20.20.10", "172.20.20.21", "172.20.20.22"]
# // @formatter:on

# https://learn.hashicorp.com/tutorials/consul/service-mesh-with-envoy-proxy?in=consul/developer-mesh#enable-connect-and-grpc
# // @formatter:off
ports {
    grpc = 8502
}

connect {
    enabled = true
}
# // @formatter:on

# // @formatter:off
bootstrap_expect    = 1
client_addr         = "0.0.0.0"
server              = true
ui                  = true
# // @formatter:on

What am I missing? Let me know if you need more information.

2

I am able to work around this problem for now by replacing the dynamic port for Redis with a fixed one, as mentioned in this thread. Not sure whether (a) Redis doesn’t like dynamic ports - MinIO seems to work fine, or (b) I’m missing something else.

3

Hi @egmanoj,

It seems like you’re not instructing Redis to listen on the dynamically generated port which is causing this issue unlike the configuration you have for MinIO. The task config for Redis could look something like below, to use a custom port listener:

config {
  image = "docker.io/bitnami/redis:6.2"
  args  = ["--port", "${NOMAD_PORT_redis}"]
  ports = ["redis"]
}

Thanks,
jrasell and the Nomad team

1 Like
4

@jrasell You were right. I pointed the REDIS_PORT_NUMBER environment variable to the dynamic port and everything is working as expected. Thanks!