Static ports not reliably accessible

Hey there,

I have a rather simple mysql-galera replication setup with a bootstrap and a replica node. Both come with a static port on 3306 on a private network 192.168.100.x

No what surprises me is that the static port 3306 comes up green for the bootstrap node but not for the replica (which syncs up just fine).

I checked that

  • the docker container is synced
  • the port is open in the container

However, the port is not accessible from the client that runs the allocation, nor from anywhere else on the network.

At first I thought it may be related to firewalling, but its not. What other aspects can I research to get the static port 3306 working reliably across allocations?

Hi @xeroc,

Thanks for using Nomad!

Do you mind sharing your job, server, and agent config files (after removing secrets)? It would be really helpful in trying to understand where the problem may lie.

Thanks!
@DerekStrickland and the Nomad team

Sure:

client

client {
    enabled = true
    node_class = ""
    no_host_uuid = false
    max_kill_timeout = "30s"
    network_interface = "enp4s0.4000"
    network_speed = 0
    cpu_total_compute = 0
    gc_interval = "1m"
    gc_disk_usage_threshold = 80
    gc_inode_usage_threshold = 70
    gc_parallel_destroys = 2
    reserved {
        cpu = 0
        memory = 0
        disk = 0
    }
    options = {
        "driver.raw_exec.enable" = "1"
    }
    }
plugin "docker" {
    config {
        allow_privileged = true
        volumes {
            enabled = true

        }

    }

}

server

server {
    enabled = true
    bootstrap_expect = 1
    rejoin_after_leave = false
    enabled_schedulers = ["service","batch","system"]
    num_schedulers = 12
    node_gc_threshold = "24h"
    eval_gc_threshold = "1h"
    job_gc_threshold = "4h"
    deployment_gc_threshold = "1h"
    encrypt = ""
    raft_protocol = 2
}

job

  # Bootstrapping Node #################################################################
  group "mysql-bootstrap" {
    count = 1

    restart {
      interval = "5m"
      attempts = 10
      delay = "25s"
      mode = "delay"
    }

    network {
      mode = "bridge"
      port "gcomm" {
        static = 4567
      }
      port "ist" {}
      port "sst" {}
      port "mysql" {
         to     = 3306
         static = 3306
      }
    }

    volume "mysql" {
      type      = "host"
      source    = "mysql"
      read_only = false
    }

    service {
      name = "mysql-bootstrap-gcomm"
      tags = ["db"]
      port = "gcomm"

      check {
        name     = "${NOMAD_GROUP_NAME} service port alive"
        type = "tcp"
        interval = "10s"
        timeout = "2s"
        #check_restart {
        #  limit = 2
        #  grace = "60s"
        #  ignore_warnings = false
        #}
      }
    }

    service {
      name = "mysql"
      tags = ["db"]
      port = "mysql"

      check {
        name     = "${NOMAD_GROUP_NAME} service port alive"
        type = "tcp"
        interval = "20s"
        timeout = "5s"
        #check_restart {
        #  limit = 10
        #  grace = "60s"
        #  ignore_warnings = true
        #}
      }
    }

    task "mariadb" {
      driver = "docker"

       volume_mount {
         volume      = "mysql"
         destination = "/bitnami/mariadb"
         read_only   = false
       }

      config {
        image       = "bitnami/mariadb-galera:10.1"
        volumes     = [
          "local/exporter-support.sql:/docker-entrypoint-initdb.d/exporter.sql"
        ]
      }

      env {
        MARIADB_EXTRA_FLAGS = "--wsrep-provider-options=ist.recv_addr=${NOMAD_HOST_ADDR_ist};ist.recv_bind=0.0.0.0:${NOMAD_HOST_PORT_ist} --wsrep-sst-receive-address=${NOMAD_HOST_ADDR_sst}"

        MARIADB_CHARACTER_SET = "utf8"
        MARIADB_COLLATE = "utf8_general_ci"
        MARIADB_ROOT_USER = "root"

        # Cluster configuration
        MARIADB_GALERA_NODE_NAME = "${NOMAD_GROUP_NAME}-${NOMAD_ALLOC_ID}"
        MARIADB_GALERA_NODE_ADDRESS = "${NOMAD_HOST_ADDR_gcomm}"

        MARIADB_GALERA_CLUSTER_BOOTSTRAP = "no"
        MARIADB_GALERA_CLUSTER_NAME = "${var.galera_cluster_name}"
        MARIADB_GALERA_MARIABACKUP_USER = "${var.galera_backup_user}"
        MARIADB_GALERA_MARIABACKUP_PASSWORD = "${var.galera_backup_password}"
        MARIADB_REPLICATION_USER = "${var.galera_replication_user}"
        MARIADB_REPLICATION_PASSWORD = "${var.galera_replication_password}"

        # this is needed only in the case we bootstrap without external nodes!
        #MARIADB_GALERA_CLUSTER_BOOTSTRAP = "yes"
        #MARIADB_GALERA_FORCE_SAFETOBOOTSTRAP = "yes"

        # From original everett cluster!
        MARIADB_GALERA_CLUSTER_ADDRESS = "gcomm://${ join(",", var.galera_external_bootstrap_nodes)},0.0.0.0:${NOMAD_HOST_PORT_gcomm}"
      }

      vault {
        policies = ["mysql-root"]
        change_mode = "noop"
      }

      template {
        data        = <<-EOF
          {{with secret "secrets/data/mysql/root"}}
          MARIADB_ROOT_PASSWORD = "{{ .Data.data.password }}"
          {{end}}
          EOF
        destination = "secrets/config.env"
        env         = true
      }

      template {
        data = <<-EOF
          CREATE USER 'exporter'@'%' IDENTIFIED BY 'secret-exporter-password';
          GRANT PROCESS, SUPER, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'exporter'@'%';
          GRANT SELECT ON performance_schema.* TO 'exporter'@'%';
          EOF
        destination = "local/exporter-support.sql"
      }

      resources {
        cpu    = 1024
        memory = 512
      }
    }
  }

  # Replica Node(s) ####################################################################
  group "mysql-replica" {
    count = "${var.galera_num_replicas}"

    restart {
      interval = "5m"
      attempts = 10
      delay = "25s"
      mode = "delay"
    }

    network {
      mode = "bridge"
      port "gcomm" {}
      port "ist" {}
      port "sst" {}
      port "mysql" {
         to     = 3306
         static = 3306
      }
    }

    ephemeral_disk {
      migrate = true
      size    = 5000  # 5G
      sticky  = true
    }

    service {
      name = "mysql"
      tags = ["db"]
      port = "mysql"

      check {
        name     = "${NOMAD_GROUP_NAME} service port alive"
        type = "tcp"
        interval = "20s"
        timeout = "5s"
        #check_restart {
        #  limit = 10
        #  grace = "60s"
        #  ignore_warnings = true
        #}
      }
    }

    task "mariadb" {
      driver = "docker"

      # We update the entrypoint as we need to create a folder with proper ownership
      template {
        data = <<-EOF
          #!/bin/bash
          if [[ ! -L "/bitnami/mariadb/data" ]]; then
            rm -rf /bitnami/mariadb/data
            ln -s $NOMAD_ALLOC_DIR/data /bitnami/mariadb/
          fi
          exec /opt/bitnami/scripts/mariadb-galera/entrypoint.sh /opt/bitnami/scripts/mariadb-galera/run.sh
          EOF
        destination = "local/entrypoint.sh"
        perms = "755"
      }

      config {
        image = "bitnami/mariadb-galera:10.1"
        entrypoint = ["/local/entrypoint.sh"]
      }

      env {
        FOO                 =" BAR"
        MARIADB_EXTRA_FLAGS = "--wsrep-provider-options=ist.recv_addr=${NOMAD_HOST_ADDR_ist};ist.recv_bind=0.0.0.0:${NOMAD_HOST_PORT_ist} --wsrep-sst-receive-address=${NOMAD_HOST_ADDR_sst}"

        MARIADB_CHARACTER_SET = "utf8"
        MARIADB_COLLATE = "utf8_general_ci"
        MARIADB_ROOT_USER = "root"

        # Cluster configuration
        MARIADB_GALERA_NODE_NAME = "${NOMAD_GROUP_NAME}-${NOMAD_ALLOC_ID}"
        MARIADB_GALERA_NODE_ADDRESS = "${NOMAD_HOST_ADDR_gcomm}"

        MARIADB_GALERA_CLUSTER_BOOTSTRAP = "no"
        MARIADB_GALERA_CLUSTER_NAME = "${var.galera_cluster_name}"
        MARIADB_GALERA_MARIABACKUP_USER = "${var.galera_backup_user}"
        MARIADB_GALERA_MARIABACKUP_PASSWORD = "${var.galera_backup_password}"
        MARIADB_REPLICATION_USER = "${var.galera_replication_user}"
        MARIADB_REPLICATION_PASSWORD = "${var.galera_replication_password}"

        MARIADB_GALERA_CLUSTER_ADDRESS = "gcomm://${join(",", var.galera_external_bootstrap_nodes)},mysql-bootstrap-gcomm.service.consul:4567,0.0.0.0:${NOMAD_HOST_PORT_gcomm}"
      }

      vault {
        policies = ["mysql-root"]
        change_mode = "noop"
      }

      template {
        data        = <<-EOF
          {{with secret "secrets/data/mysql/root"}}
          MARIADB_ROOT_PASSWORD = "{{ .Data.data.password }}"
          {{end}}
          EOF
        destination = "secrets/config.env"
        env         = true
      }

      resources {
        cpu    = 1024
        memory = 512
      }
    }
  }
}

It’s worth noting that:

  • I use network = bridge
  • It all works when i use static = 3307 instead of static = 3306.

How can I even debug this?