Issue setting up traefik -> whoami with sidecar and ACL

Nomad
1

Hello, I am trying to get treafik with a sidecar working with ACL. The example works if am setting the intention count-dashboard -> count-api. But for traefik -> whoami this intention is not working. I need to set All -> whoami. The main difference I could find is that for whoami in consul no topology is shown. I was unable to set it via upstreams. I think this is the issue that consul is not allowing a connection from the proxy to the job. Does anybody know how to fix this?

traefik config

job "traefik" {
  region      = "global"
  datacenters = ["dc1"]
  type        = "service"

  group "traefik" {
    count = 1
    network {
      mode = "bridge"
      port "http-priv" {
        static = 80 
        to = 80
        host_network = "private"
      }
    }
    service {
      name = "traefik" 
      port = "http-priv"
      connect{
        native = true
      }
      tags = [
        "traefik.enable=true",
        "traefik.http.routers.dashboard.rule=Host(`traefik.test`)",
        "traefik.http.routers.dashboard.service=api@internal",
      ]
    }
    task "traefik" {
      driver = "docker"
      config {
        image = "traefik:latest" 
        ports = ["http-priv"]
        volumes = [
          "local/traefik.yml:/etc/traefik/traefik.yml",
          "/home/test/consul/tls/consul-agent-ca.pem:/etc/traefik/consul-agent-ca.pem",
          "/home/test/consul/tls/homelab-client-consul-3-key.pem:/etc/traefik/homelab-client-consul-3-key.pem",
          "/home/test/consul/tls/homelab-client-consul-3.pem:/etc/traefik/homelab-client-consul-3.pem",
        ]
      }


      template {
        data        = <<EOF
entryPoints:
  http-priv:
    address: ':80'
log:
  level: DEBUG
api:
  dashboard: true
providers:
  consulCatalog:
    connectAware: true
    prefix: traefik
    exposedByDefault: false
    endpoint:
      address: 172.17.0.1:8500
      scheme: http
      token: {{key "nomad/namd-token"}}
      tls:
        caOptional: false
        ca: /etc/traefik/consul-agent-ca.pem
        cert: /etc/traefik/homelab-client-consul-3.pem
        key: /etc/traefik/homelab-client-consul-3-key.pem
EOF
        destination = "local/traefik.yml"
      }

    }
  }
}

whoami config

job "whoami" {
  datacenters = ["dc1"]

  group "whoami" {
    count = 1

    network {
      mode = "bridge"
      port "web" {
        host_network = "private"
      }
      port "connect-proxy-whoami" {
        host_network = "private"
      }
    }

    service {
      name = "whoami"
      port = "web"
      connect {
        sidecar_service {
        }
      }

      tags = [
        "traefik.enable=true",
        "traefik.consulcatalog.connect=true",
        "traefik.http.routers.whoami.rule=Path(`/whoami`)",
      ]

      check {
        type     = "http"
        path     = "/health"
        port     = "web"
        interval = "10s"
        timeout  = "2s"
      }
    }

    task "whoami" {
      driver = "docker"
      config {
        image = "traefik/whoami"
        ports = ["web"]
        args  = ["--port", "${NOMAD_PORT_web}"]
      }

      resources {
        cpu    = 100
        memory = 128
      }
    }
  }
}
1 Like
2

Hi @setime :wave:

I don’t have a lot of experience with Traefik’s Connect Native integration, but maybe you are missing the connectByDefault option in your configuration?

I tried adding it and the intention traefik -block-> whoami started working for me.

3

I have the same problem.

My traefik config:

job "traefik" {
  datacenters = ["dc1"]
  type        = "system"

  group "traefik" {
    service {
      name = "traefik-http"
      port = "http"
      connect {
        sidecar_service {
          proxy {
            transparent_proxy {}
          }
        }
      }
      #connect {
      #  native = true
      #}
    }
    
    service {
      name = "traefik-api"
      port = "api"
      check {
        type     = "tcp"
        interval = "3s"
        timeout  = "1s"
      }
    }
    
    network {
      mode = "bridge"
      port "api" {
        static = 1936
        to = 1936
      }
      port "http" {
        static = 80
        to = 80
      }
      port "https" {
        static = 443
        to = 443
      }
    }

    task "traefik" {
      driver = "docker"

      config {
        image = "traefik:3.4.4"
        ports = ["api", "http", "https"]
        volumes = [
          "local/traefik.toml:/etc/traefik/traefik.toml",
          "local/ca.pem:/etc/traefik/ca.pem",
          "local/cert.pem:/etc/traefik/cert.pem",
          "local/key.pem:/etc/traefik/key.pem",
        ]
        #network_mode = "host"
      }
      
      template {
        data = "{{ key \"config/certs/ca.pem\" }}"
        destination = "local/ca.pem"
      }
      
      template {
        data = "{{ key \"config/certs/cert.pem\" }}"
        destination = "local/cert.pem"
      }
      
      template {
        data = "{{ key \"config/certs/key.pem\" }}"
        destination = "local/key.pem"
      }      
      
      template {
        left_delimiter = "[[["
        right_delimiter = "]]]"
        data = <<EOF
[entryPoints]
  [entryPoints.http]
    address = ":[[[ env "NOMAD_PORT_http" ]]]"
  [entryPoints.https]
    address = ":[[[ env "NOMAD_PORT_https" ]]]"
  [entryPoints.traefik]
    address = ":[[[ env "NOMAD_PORT_api" ]]]"

#[tls.stores]
#  [tls.stores.default]
#    [tls.stores.default.defaultCertificate]
#      certFile = "/etc/traefik/cert.pem"
#      keyFile = "/etc/traefik/key.pem"
        
[api]
  dashboard = true
  insecure = true
        
[log]
  level = "DEBUG"

[providers]
  [providers.consulCatalog]
    prefix           = "traefik"
    exposedByDefault = true
    connectAware = true
    connectByDefault = true
    defaultRule = "Host(`{{ normalize .Name }}.service.consul`)"
    #watch = true

    [providers.consulCatalog.endpoint]
      address = "[[[ env "NOMAD_IP_http" ]]]:8501"
      scheme  = "https"        
      [providers.consulCatalog.endpoint.tls]
        ca = "/etc/traefik/ca.pem"
        cert = "/etc/traefik/cert.pem"
        key = "/etc/traefik/key.pem"
        
  #[providers.consul]
   # endpoints = ["[[[ env "NOMAD_IP_http" ]]]:8501"]
    #[providers.consul.tls]
     # ca = "/etc/traefik/ca.pem"
      #cert = "/etc/traefik/cert.pem"
      #key = "/etc/traefik/key.pem"
        
EOF
        destination = "local/traefik.toml"
      }
      
      env {
        #CONSUL_HTTP_SSL = false
        #CONSUL_HTTP_SSL_VERIFY = false
      }

      resources {
        cpu    = 256
        memory = 256
      }
    }
  }
}

It correctly routes to the proxy-sidecar address (mTLS, hence https):

image
image823×495 17.8 KB

And I’m able to browse to it:

image
image901×442 40.3 KB

But ONLY if I allow all services to access all other services:

image
image949×381 17.5 KB

As soon as I switch that off:

image
image946×384 17.5 KB

I get 502: Bad Gateway

image
image550×147 2.31 KB

Is it possible that traefik has a hidden service with which it accesses the mesh, but doesn’t register itself with Consul, such that intentions don’t work?

4

OK, I figured it out. It’s really dumb…

[providers]
  [providers.consulCatalog]
    prefix           = "traefik"
    exposedByDefault = true
    connectAware = true
    connectByDefault = true
    defaultRule = "Host(`{{ normalize .Name }}.service.consul`)"
    servicename = "traefik-http" # <-- this needs to match with how the service is registered inside consul!
    #watch = true

servicename inside your consul catalog definition MUST MATCH with how the service is registered in Consul.

E.g. mine is registered as traefik-http. So, I need to configure servicename = "traefik-http".

Now, when setting default-deny, but allowing traefik-http, I’m able to browse!

image
image940×360 17.7 KB

image
image935×465 52 KB