Hi, so yesterday i finally enabled telemetry to gather metrics from vault.
I have 2 clusters in my k8s setup, one for autounsealing divided by namespace, here everything works. I can get most of the metrics i’m interested in.
However in the second cluster which has identical configuration in terms of metrics/telemetry config. I get response only from one of the pods. The other ones end the request with message:
curl -k https://vault.services-dev.svc:8200/v1/sys/metrics\?format=prometheus
prometheus is not enabled
Can’t really figure out what is happening.
Configs from both vault clusters are here
Not working cluster:
config: |
ui = true
listener "tcp" {
tls_disable = 0
address = "0.0.0.0:8200"
cluster_address = "0.0.0.0:8300"
tls_cert_file = "/vault/userconfig/tls-server/tls.crt"
tls_key_file = "/vault/userconfig/tls-server/tls.key"
tls_min_version = "tls12"
telemetry {
unauthenticated_metrics_access = "true"
}
}
seal "transit" {
address = "https://vault-auth.vault-auth.svc:8200"
# please add correct token to the config otherwise it will not work!!
# this needs to be the unwrapped one !!
token = "unique_token"
disable_renewal = "false"
key_name = "autounseal"
mount_path = "transit/"
tls_skip_verify = "true"
}
storage "postgresql" {
connection_url = "postgres://user:pass@vault-db-postgresql-ha-pgpool.services-dev.svc:5432/vault?sslmode=disable"
table = "vault_kv_store"
ha_enabled = "true"
ha_table = "vault_ha_locks"
}
telemetry {
disable_hostname = true
prometheus_retention_time = "72h"
}
service_registration "kubernetes" {}
disable_mlock = true
Working cluster:
config: |
ui = true
listener "tcp" {
tls_disable = 0
address = "0.0.0.0:8200"
cluster_address = "0.0.0.0:8300"
tls_cert_file = "/vault/userconfig/tls-server/tls.crt"
tls_key_file = "/vault/userconfig/tls-server/tls.key"
tls_min_version = "tls12"
telemetry {
unauthenticated_metrics_access = "true"
}
}
storage "postgresql" {
connection_url = "postgres://user:pass@vault-db-postgresql-ha-pgpool.services-dev.svc:5432/vault_auth?sslmode=disable"
table = "vault_kv_store"
ha_enabled = "true"
ha_table = "vault_ha_locks"
}
telemetry {
disable_hostname = true
prometheus_retention_time = "72h"
}