I was able to figure it out. primary_gateways was set to the Pod’s service IP which was not routable to the other cluster. I had to set the wanAddress to a static value which reflected the FQDN of the wan interface. Only then did the clusters actually sync completely. When running a consul members -wan, all server nodes now report “alive” status.
Working helm values
DC1:
global:
datacenter: dc1
name: consul
domain: consul
tls:
enabled: true
enableAutoEncrypt: true
serverAdditionalDNSSANs:
- "consul-server.consul.svc.cluster.local"
federation:
enabled: true
createFederationSecret: true
acls:
manageSystemACLs: true
createReplicationToken: true
gossipEncryption:
autoGenerate: true
logJSON: true
connectInject:
enabled: true
default: false
controller:
enabled: true
meshGateway:
enabled: true
replicas: 1
service:
enabled: true
type: NodePort
nodePort: 30085
wanAddress:
enabled: true
source: "Static"
static: "dc1.example.com"
port: 30085
syncCatalog:
enabled: true
default: true
toConsul: true
toK8S: true
metrics:
enabled: true
prometheus:
enabled: true
ui:
enabled: true
service:
type: NodePort
nodePort:
https: 30084
server:
replicas: 3
securityContext:
runAsNonRoot: false
runAsUser: 0
service:
type: NodePort
client:
securityContext:
runAsNonRoot: false
runAsUser: 0
DC2:
global:
datacenter: dc2
name: consul
domain: consul
tls:
enabled: true
enableAutoEncrypt: true
serverAdditionalDNSSANs:
- "consul-server.consul.svc.cluster.local"
caCert:
secretName: consul-federation
secretKey: caCert
caKey:
secretName: consul-federation
secretKey: caKey
acls:
manageSystemACLs: true
replicationToken:
secretName: consul-federation
secretKey: replicationToken
federation:
enabled: true
gossipEncryption:
secretName: consul-federation
secretKey: gossipEncryptionKey
logJSON: true
connectInject:
enabled: true
default: false
controller:
enabled: true
meshGateway:
enabled: true
replicas: 1
service:
enabled: true
type: NodePort
nodePort: 30085
wanAddress:
enabled: true
source: "Static"
static: "dc2.example.com"
port: 30085
syncCatalog:
enabled: true
default: true
toConsul: true
toK8S: true
metrics:
enabled: true
prometheus:
enabled: true
ui:
enabled: true
service:
type: NodePort
nodePort:
https: 30084
server:
replicas: 1
securityContext:
runAsNonRoot: false
runAsUser: 0
extraVolumes:
- type: secret
name: consul-federation
items:
- key: serverConfigJSON
path: config.json
load: true
client:
securityContext:
runAsNonRoot: false
runAsUser: 0