xeroc
August 27, 2021, 10:10am
1
Hey there,
I have a rather simple mysql-galera replication setup with a bootstrap and a replica node. Both come with a static port on 3306 on a private network 192.168.100.x
No what surprises me is that the static port 3306 comes up green for the bootstrap node but not for the replica (which syncs up just fine).
I checked that
the docker container is synced
the port is open in the container
However, the port is not accessible from the client that runs the allocation, nor from anywhere else on the network.
At first I thought it may be related to firewalling, but its not. What other aspects can I research to get the static port 3306 working reliably across allocations?
Hi @xeroc ,
Thanks for using Nomad!
Do you mind sharing your job, server, and agent config files (after removing secrets)? It would be really helpful in trying to understand where the problem may lie.
Thanks!@DerekStrickland and the Nomad team
xeroc
August 27, 2021, 11:32am
3
Sure:
client {
enabled = true
node_class = ""
no_host_uuid = false
max_kill_timeout = "30s"
network_interface = "enp4s0.4000"
network_speed = 0
cpu_total_compute = 0
gc_interval = "1m"
gc_disk_usage_threshold = 80
gc_inode_usage_threshold = 70
gc_parallel_destroys = 2
reserved {
cpu = 0
memory = 0
disk = 0
}
options = {
"driver.raw_exec.enable" = "1"
}
}
plugin "docker" {
config {
allow_privileged = true
volumes {
enabled = true
}
}
}
server {
enabled = true
bootstrap_expect = 1
rejoin_after_leave = false
enabled_schedulers = ["service","batch","system"]
num_schedulers = 12
node_gc_threshold = "24h"
eval_gc_threshold = "1h"
job_gc_threshold = "4h"
deployment_gc_threshold = "1h"
encrypt = ""
raft_protocol = 2
}
# Bootstrapping Node #################################################################
group "mysql-bootstrap" {
count = 1
restart {
interval = "5m"
attempts = 10
delay = "25s"
mode = "delay"
}
network {
mode = "bridge"
port "gcomm" {
static = 4567
}
port "ist" {}
port "sst" {}
port "mysql" {
to = 3306
static = 3306
}
}
volume "mysql" {
type = "host"
source = "mysql"
read_only = false
}
service {
name = "mysql-bootstrap-gcomm"
tags = ["db"]
port = "gcomm"
check {
name = "${NOMAD_GROUP_NAME} service port alive"
type = "tcp"
interval = "10s"
timeout = "2s"
#check_restart {
# limit = 2
# grace = "60s"
# ignore_warnings = false
#}
}
}
service {
name = "mysql"
tags = ["db"]
port = "mysql"
check {
name = "${NOMAD_GROUP_NAME} service port alive"
type = "tcp"
interval = "20s"
timeout = "5s"
#check_restart {
# limit = 10
# grace = "60s"
# ignore_warnings = true
#}
}
}
task "mariadb" {
driver = "docker"
volume_mount {
volume = "mysql"
destination = "/bitnami/mariadb"
read_only = false
}
config {
image = "bitnami/mariadb-galera:10.1"
volumes = [
"local/exporter-support.sql:/docker-entrypoint-initdb.d/exporter.sql"
]
}
env {
MARIADB_EXTRA_FLAGS = "--wsrep-provider-options=ist.recv_addr=${NOMAD_HOST_ADDR_ist};ist.recv_bind=0.0.0.0:${NOMAD_HOST_PORT_ist} --wsrep-sst-receive-address=${NOMAD_HOST_ADDR_sst}"
MARIADB_CHARACTER_SET = "utf8"
MARIADB_COLLATE = "utf8_general_ci"
MARIADB_ROOT_USER = "root"
# Cluster configuration
MARIADB_GALERA_NODE_NAME = "${NOMAD_GROUP_NAME}-${NOMAD_ALLOC_ID}"
MARIADB_GALERA_NODE_ADDRESS = "${NOMAD_HOST_ADDR_gcomm}"
MARIADB_GALERA_CLUSTER_BOOTSTRAP = "no"
MARIADB_GALERA_CLUSTER_NAME = "${var.galera_cluster_name}"
MARIADB_GALERA_MARIABACKUP_USER = "${var.galera_backup_user}"
MARIADB_GALERA_MARIABACKUP_PASSWORD = "${var.galera_backup_password}"
MARIADB_REPLICATION_USER = "${var.galera_replication_user}"
MARIADB_REPLICATION_PASSWORD = "${var.galera_replication_password}"
# this is needed only in the case we bootstrap without external nodes!
#MARIADB_GALERA_CLUSTER_BOOTSTRAP = "yes"
#MARIADB_GALERA_FORCE_SAFETOBOOTSTRAP = "yes"
# From original everett cluster!
MARIADB_GALERA_CLUSTER_ADDRESS = "gcomm://${ join(",", var.galera_external_bootstrap_nodes)},0.0.0.0:${NOMAD_HOST_PORT_gcomm}"
}
vault {
policies = ["mysql-root"]
change_mode = "noop"
}
template {
data = <<-EOF
{{with secret "secrets/data/mysql/root"}}
MARIADB_ROOT_PASSWORD = "{{ .Data.data.password }}"
{{end}}
EOF
destination = "secrets/config.env"
env = true
}
template {
data = <<-EOF
CREATE USER 'exporter'@'%' IDENTIFIED BY 'secret-exporter-password';
GRANT PROCESS, SUPER, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'exporter'@'%';
GRANT SELECT ON performance_schema.* TO 'exporter'@'%';
EOF
destination = "local/exporter-support.sql"
}
resources {
cpu = 1024
memory = 512
}
}
}
# Replica Node(s) ####################################################################
group "mysql-replica" {
count = "${var.galera_num_replicas}"
restart {
interval = "5m"
attempts = 10
delay = "25s"
mode = "delay"
}
network {
mode = "bridge"
port "gcomm" {}
port "ist" {}
port "sst" {}
port "mysql" {
to = 3306
static = 3306
}
}
ephemeral_disk {
migrate = true
size = 5000 # 5G
sticky = true
}
service {
name = "mysql"
tags = ["db"]
port = "mysql"
check {
name = "${NOMAD_GROUP_NAME} service port alive"
type = "tcp"
interval = "20s"
timeout = "5s"
#check_restart {
# limit = 10
# grace = "60s"
# ignore_warnings = true
#}
}
}
task "mariadb" {
driver = "docker"
# We update the entrypoint as we need to create a folder with proper ownership
template {
data = <<-EOF
#!/bin/bash
if [[ ! -L "/bitnami/mariadb/data" ]]; then
rm -rf /bitnami/mariadb/data
ln -s $NOMAD_ALLOC_DIR/data /bitnami/mariadb/
fi
exec /opt/bitnami/scripts/mariadb-galera/entrypoint.sh /opt/bitnami/scripts/mariadb-galera/run.sh
EOF
destination = "local/entrypoint.sh"
perms = "755"
}
config {
image = "bitnami/mariadb-galera:10.1"
entrypoint = ["/local/entrypoint.sh"]
}
env {
FOO =" BAR"
MARIADB_EXTRA_FLAGS = "--wsrep-provider-options=ist.recv_addr=${NOMAD_HOST_ADDR_ist};ist.recv_bind=0.0.0.0:${NOMAD_HOST_PORT_ist} --wsrep-sst-receive-address=${NOMAD_HOST_ADDR_sst}"
MARIADB_CHARACTER_SET = "utf8"
MARIADB_COLLATE = "utf8_general_ci"
MARIADB_ROOT_USER = "root"
# Cluster configuration
MARIADB_GALERA_NODE_NAME = "${NOMAD_GROUP_NAME}-${NOMAD_ALLOC_ID}"
MARIADB_GALERA_NODE_ADDRESS = "${NOMAD_HOST_ADDR_gcomm}"
MARIADB_GALERA_CLUSTER_BOOTSTRAP = "no"
MARIADB_GALERA_CLUSTER_NAME = "${var.galera_cluster_name}"
MARIADB_GALERA_MARIABACKUP_USER = "${var.galera_backup_user}"
MARIADB_GALERA_MARIABACKUP_PASSWORD = "${var.galera_backup_password}"
MARIADB_REPLICATION_USER = "${var.galera_replication_user}"
MARIADB_REPLICATION_PASSWORD = "${var.galera_replication_password}"
MARIADB_GALERA_CLUSTER_ADDRESS = "gcomm://${join(",", var.galera_external_bootstrap_nodes)},mysql-bootstrap-gcomm.service.consul:4567,0.0.0.0:${NOMAD_HOST_PORT_gcomm}"
}
vault {
policies = ["mysql-root"]
change_mode = "noop"
}
template {
data = <<-EOF
{{with secret "secrets/data/mysql/root"}}
MARIADB_ROOT_PASSWORD = "{{ .Data.data.password }}"
{{end}}
EOF
destination = "secrets/config.env"
env = true
}
resources {
cpu = 1024
memory = 512
}
}
}
}
It’s worth noting that:
I use network = bridge
It all works when i use static = 3307 instead of static = 3306.
How can I even debug this?
1 Like
Hello @xeroc , did you figure out why this happens or how to prevent this?