Consul pods are failing to run

Consul
1

config.yml

global:
  name: consul
  datacenter: dc1
  image: hashicorp/consul:1.10.0
  imageK8S: hashicorp/consul-k8s:0.26.0
  tls:
    enabled: false
    enableAutoEncrypt: false
    verify: false
  # metrics:
  #   enabled: true
  #   enableGatewayMetrics: true
server:
  replicas: 1
  # securityContext:
  #   runAsNonRoot: true
#  bootstrapExpect: 1
#  extraConfig: |
#    {
#      "telemetry": {
#        "prometheus_retention_time": "8h",
#        "disable_hostname": true
#      }
#    }
client:
  enabled: true
  # securityContext:
  #   runAsNonRoot: true

#  extraConfig: |
#    {
#      "telemetry": {
#        "prometheus_retention_time": "1m",
#        "disable_hostname": true
#      }
#    }
controller:
  enabled: true
  default: true
syncCatalog:
  enabled: true
  toConsul: true
  toK8S: true
  default: true
connectInject:
  enabled: true
  default: true
ui:
  enabled: true
  service:
    enabled: true
    type: ClusterIP
#  metrics:
#    enabled: true
#    provider: "prometheus"
#    baseURL: http://prometheus-server
ingressGateways:
  enabled: false
  # securityContext:
  #   runAsNonRoot: true
  defaults:
    replicas: 1
    service:
      type: LoadBalancer
      ports:
        - port: 80
        - port: 443
meshGateway:
  enabled: false
  # replicas: 1

helm version
version.BuildInfo{Version:"v3.7.2", GitCommit:"663a896f4a815053445eec4153677ddc24a0a361", GitTreeState:"clean", GoVersion:"go1.16.10"}

helm repo add hashicorp https://helm.releases.hashicorp.com
helm repo update
helm install -f config.yaml --version "0.32.1" consul hashicorp/consul -n consul

kubectl describe pod consul-webhook-cert-manager-56cdbb7648-jpj4h -n consul
Name:         consul-webhook-cert-manager-56cdbb7648-jpj4h
Namespace:    consul
Priority:     0
Node:         aks-systempool-33755211-vmss000000/10.240.0.4
Start Time:   Fri, 07 Jan 2022 19:30:52 +0000
Labels:       app=consul
              chart=consul-helm
              component=webhook-cert-manager
              heritage=Helm
              pod-template-hash=56cdbb7648
              release=consul
Annotations:  consul.hashicorp.com/config-checksum: 44f20d3c49318074ca5a4aef932fc051358ba926a51ae01fd5b2fc9ea9cd5769
              consul.hashicorp.com/connect-inject: false
Status:       Running
IP:           10.240.0.11
IPs:
  IP:           10.240.0.11
Controlled By:  ReplicaSet/consul-webhook-cert-manager-56cdbb7648
Containers:
  webhook-cert-manager:
    Container ID:  containerd://69fab4eca5ee19ead5cb737fcbc5cff9e65988a13f40cb6f4f66e3392690df15
    Image:         hashicorp/consul-k8s:0.26.0
    Image ID:      docker.io/hashicorp/consul-k8s@sha256:16c8066aeb1d85b1b3e72e7a3a2c19f3f9b2c2742201d97a668ffb2657efd32f
    Port:          <none>
    Host Port:     <none>
    Command:
      /bin/sh
      -ec
      consul-k8s webhook-cert-manager \
        -config-file=/bootstrap/config/webhook-config.json \
        -deployment-name=consul-webhook-cert-manager \
        -deployment-namespace=consul
      
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    1
      Started:      Fri, 07 Jan 2022 19:42:00 +0000
      Finished:     Fri, 07 Jan 2022 19:42:00 +0000
    Ready:          False
    Restart Count:  7
    Limits:
      cpu:     100m
      memory:  50Mi
    Requests:
      cpu:        100m
      memory:     50Mi
    Environment:  <none>
    Mounts:
      /bootstrap/config from config (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-lbx7f (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  config:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      consul-webhook-cert-manager-config
    Optional:  false
  kube-api-access-lbx7f:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   Guaranteed
Node-Selectors:              <none>
Tolerations:                 node.kubernetes.io/memory-pressure:NoSchedule op=Exists
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason     Age                   From               Message
  ----     ------     ----                  ----               -------
  Normal   Scheduled  13m                   default-scheduler  Successfully assigned consul/consul-webhook-cert-manager-56cdbb7648-jpj4h to aks-systempool-33755211-vmss00
0000
  Normal   Pulling    13m                   kubelet            Pulling image "hashicorp/consul-k8s:0.26.0"
  Normal   Pulled     13m                   kubelet            Successfully pulled image "hashicorp/consul-k8s:0.26.0" in 2.353216911s
  Normal   Created    12m (x5 over 13m)     kubelet            Created container webhook-cert-manager
  Normal   Started    12m (x5 over 13m)     kubelet            Started container webhook-cert-manager
  Normal   Pulled     12m (x4 over 13m)     kubelet            Container image "hashicorp/consul-k8s:0.26.0" already present on machine
  Warning  BackOff    3m33s (x47 over 13m)  kubelet            Back-off restarting failed container

PS C:\mart_deployment> helm version
version.BuildInfo{Version:"v3.7.2", GitCommit:"663a896f4a815053445eec4153677ddc24a0a361", GitTreeState:"clean", GoVersion:"go1.16.10"}

kubectl get po -n consul
NAME                                                          READY   STATUS              RESTARTS       AGE
consul-connect-injector-webhook-deployment-5d6b98587c-q4k6p   0/1     ContainerCreating   0              18m
consul-connect-injector-webhook-deployment-5d6b98587c-ss5cm   0/1     ContainerCreating   0              18m
consul-controller-dff49c9f4-tppd7                             0/1     ContainerCreating   0              18m
consul-scb7t                                                  1/1     Running             0              18m
consul-server-0                                               1/1     Running             0              18m
consul-sync-catalog-78998c5f4-b84x2                           1/1     Running             0              18m
consul-webhook-cert-manager-56cdbb7648-jpj4h                  0/1     CrashLoopBackOff    8 (117s ago)   18m
2

I’m not a kubernetes expert, but from a Consul point of view. I think if you don’t tell Consul to boostrap and only have 1 node, it’ll fail to start.

3

can you suggest what need to done.
Is it in consul config file?

4

bootstrapExpect, among others.

5

Hi @ukreddy-erwin could you post the output of the following command

kubectl logs -n consul consul-connect-injector-webhook-deployment-5d6b98587c-q4k6p

That is assuming the above pod is still running under that name. Otherwise any injector pod that you find when running kubectl pods -n consul would work

Thank you!

1 Like
6

Also kubectl describe on the pod that’s crash looping.