Mesh Gateway federation woes!

Consul
,
6

@karl-cardenas-coding

I was able to figure it out. primary_gateways was set to the Pod’s service IP which was not routable to the other cluster. I had to set the wanAddress to a static value which reflected the FQDN of the wan interface. Only then did the clusters actually sync completely. When running a consul members -wan, all server nodes now report “alive” status.

Working helm values

DC1:

global:
    datacenter: dc1
    name: consul
    domain: consul
    tls:
        enabled: true
        enableAutoEncrypt: true
        serverAdditionalDNSSANs:
            - "consul-server.consul.svc.cluster.local"
    federation:
        enabled: true
        createFederationSecret: true
    acls:
        manageSystemACLs: true
        createReplicationToken: true
    gossipEncryption:
        autoGenerate: true
    logJSON: true
connectInject:
    enabled: true
    default: false
controller:
    enabled: true
meshGateway:
    enabled: true
    replicas: 1
    service:
        enabled: true
        type: NodePort
        nodePort: 30085
    wanAddress:
        enabled: true
        source: "Static"
        static: "dc1.example.com"
        port: 30085
syncCatalog:
    enabled: true
    default: true
    toConsul: true
    toK8S: true
metrics:
    enabled: true
prometheus:
    enabled: true
ui:
    enabled: true
    service:
        type: NodePort
        nodePort:
            https: 30084
server:
    replicas: 3
    securityContext:
        runAsNonRoot: false
        runAsUser: 0
    service:
        type: NodePort
client:
    securityContext:
        runAsNonRoot: false
        runAsUser: 0

DC2:

global:
    datacenter: dc2
    name: consul
    domain: consul
    tls:
        enabled: true
        enableAutoEncrypt: true
        serverAdditionalDNSSANs:
            - "consul-server.consul.svc.cluster.local"
        caCert:
            secretName: consul-federation
            secretKey: caCert
        caKey:
            secretName: consul-federation
            secretKey: caKey
    acls:
        manageSystemACLs: true
        replicationToken:
            secretName: consul-federation
            secretKey: replicationToken
    federation:
        enabled: true
    gossipEncryption:
        secretName: consul-federation
        secretKey: gossipEncryptionKey
    logJSON: true
connectInject:
    enabled: true
    default: false
controller:
    enabled: true
meshGateway:
    enabled: true
    replicas: 1
    service:
        enabled: true
        type: NodePort
        nodePort: 30085
    wanAddress:
        enabled: true
        source: "Static"
        static: "dc2.example.com"
        port: 30085
syncCatalog:
    enabled: true
    default: true
    toConsul: true
    toK8S: true
metrics:
    enabled: true
prometheus:
    enabled: true
ui:
    enabled: true
    service:
        type: NodePort
        nodePort:
            https: 30084
server:
    replicas: 1
    securityContext:
        runAsNonRoot: false
        runAsUser: 0
    extraVolumes:
        - type: secret
          name: consul-federation
          items:
              - key: serverConfigJSON
                path: config.json
          load: true
client:
    securityContext:
        runAsNonRoot: false
        runAsUser: 0