In my stack we use eck operator
And Elasticsearch CRDS which are applied with kubernetes_manifest terraform resource:
https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest
The problem is that any change in the resource causes PVC recreation. Offcource i tried to set volumeClaimDeletePolicy but this works only for yaml’s, it doesnt for terraform which inteprets changes separately by using statefile.
I tried to use ignore_changes. But it do not (and will not) work because (as far i understand) volumeClaimTemplate section is not the CAUSE of changes, it is a RESULT of changes. The cause lies in manifest.spec.nodeSets.0.podTemplate.spec.containers.0 where im trying to add securityContext.
So is it possible selectively exclude somehow resource built with VolumClaimTemplates even though the changes which caused the re-creation are somewhere else?
Recource definition:
resource "kubernetes_manifest" "elasticsearch" {
manifest = {
"apiVersion" = "elasticsearch.k8s.elastic.co/v1"
"kind" = "Elasticsearch"
"metadata" = {
"name" = "elasticsearch"
"namespace" = kubernetes_namespace_v1.elasticsearch.id
}
"spec" = {
# "version" = "8.1.2"
# "version" = "7.17.1"
"version" = "8.2.2"
# https://www.elastic.co/guide/en/cloud-on-k8s/master/k8s-volume-claim-templates.html#k8s_controlling_volume_claim_deletion
"volumeClaimDeletePolicy" = "DeleteOnScaledownOnly"
"auth" = {
"fileRealm" = [{
"secretName" = kubernetes_secret.elasticsearch_users.metadata.0.name
}]
}
"http" = {
"tls" = {
"selfSignedCertificate" = {
"disabled" = true
}
}
}
"nodeSets" = [
{
"name" = "masters"
"count" = 1
"config" = {
"node.roles" = [
"master",
"data"
]
"node.attr.attr_name" = "attr_value"
"node.store.allow_mmap" = false
}
"podTemplate" = {
"metadata" = {
# "creationTimestamp" = "20220420"
"labels" = {
"env" = "stg"
}
}
"spec" = {
"containers" = [
{
"name" = "elasticsearch"
"resources" = {
"limits" = {
"cpu" = "2"
"memory" = "2Gi"
}
"requests" = {
"cpu" = "1"
"memory" = "2Gi"
}
}
# TODO: [SECURITY] uncomment whe kibana get volume for its settings
"securityContext" = {
"runAsNonRoot" = true
"runAsUser" = 1000
}
}
]
}
}
"volumeClaimTemplates" = [
{
"metadata" = {
"name" = "elasticsearch-data"
}
"spec" = {
"accessModes" = [
"ReadWriteOnce"
]
"resources" = {
"requests" = {
"storage" = var.elk_disk_size[var.env]
}
}
"storageClassName" = "csi-disk"
}
}
]
}
]
}
}
field_manager {
# name = ".spec.nodeSets"
force_conflicts = true
}
lifecycle {
# ignore_changes = all
# ignore_changes = [
# # manifest.metadata.0.annotations["pv.kubernetes.io/bind-completed"],
# # manifest.metadata.0.annotations["pv.kubernetes.io/bound-by-controller"],
# # manifest.metadata.0.annotations["volume.beta.kubernetes.io/storage-provisioner"],
# manifest.spec.nodeSets.0.volumeClaimTemplates.0,
# # object.spec.nodeSets.0.volumeClaimTemplates.0
# ]
# prevent_destroy = [manifest.spec.nodeSets.0.volumeClaimTemplates]
}
depends_on = [helm_release.eck_operator]
}
PVC which is created with it
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
volume.beta.kubernetes.io/storage-provisioner: everest-csi-provisioner
creationTimestamp: "2022-09-05T16:40:39Z"
finalizers:
- kubernetes.io/pvc-protection
labels:
common.k8s.elastic.co/type: elasticsearch
elasticsearch.k8s.elastic.co/cluster-name: elasticsearch
elasticsearch.k8s.elastic.co/statefulset-name: elasticsearch-es-masters
name: elasticsearch-data-elasticsearch-es-masters-0
namespace: elasticsearch
ownerReferences:
- apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
name: elasticsearch
uid: 7def015f-14e3-4360-8cb6-87c954e13f10
resourceVersion: "55823421"
uid: 5b3f7e60-d368-493e-bd17-590de81e141b
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 20Gi
storageClassName: csi-disk
volumeMode: Filesystem
volumeName: pvc-5b3f7e60-d368-493e-bd17-590de81e141b
status:
accessModes:
- ReadWriteOnce
capacity:
storage: 20Gi
phase: Bound
Plan when im trying to add securityContect section
# kubernetes_manifest.elasticsearch must be replaced
-/+ resource "kubernetes_manifest" "elasticsearch" {
~ manifest = {
~ spec = {
~ nodeSets = [
~ {
name = "masters"
~ podTemplate = {
~ spec = {
~ containers = [
~ {
name = "elasticsearch"
~ securityContext = {
+ runAsNonRoot = true
+ runAsUser = 1000
}
# (1 unchanged element hidden)
},
]
}
# (1 unchanged element hidden)
}
# (3 unchanged elements hidden)
},
]
# (3 unchanged elements hidden)
}
# (3 unchanged elements hidden)
}
~ object = {
~ metadata = {
~ annotations = {
- "eck.k8s.elastic.co/orchestration-hints" = jsonencode(
{
- no_transient_settings = true
}
)
- "elasticsearch.k8s.elastic.co/cluster-uuid" = "HmK2-WuWQEiSc2vcxrGjKQ"
} -> (known after apply)
~ clusterName = null -> (known after apply)
~ creationTimestamp = null -> (known after apply)
~ deletionGracePeriodSeconds = null -> (known after apply)
~ deletionTimestamp = null -> (known after apply)
~ finalizers = null -> (known after apply)
~ generateName = null -> (known after apply)
~ generation = null -> (known after apply)
~ labels = null -> (known after apply)
~ managedFields = null -> (known after apply)
name = "elasticsearch"
~ ownerReferences = null -> (known after apply)
~ resourceVersion = null -> (known after apply)
~ selfLink = null -> (known after apply)
~ uid = null -> (known after apply)
# (1 unchanged element hidden)
}
~ spec = {
~ auth = {
~ roles = null -> (known after apply)
# (1 unchanged element hidden)
}
~ http = {
~ service = {
~ metadata = {
~ annotations = null -> (known after apply)
~ finalizers = null -> (known after apply)
~ labels = null -> (known after apply)
~ name = null -> (known after apply)
~ namespace = null -> (known after apply)
}
~ spec = {
~ allocateLoadBalancerNodePorts = null -> (known after apply)
~ clusterIP = null -> (known after apply)
~ clusterIPs = null -> (known after apply)
~ externalIPs = null -> (known after apply)
~ externalName = null -> (known after apply)
~ externalTrafficPolicy = null -> (known after apply)
~ healthCheckNodePort = null -> (known after apply)
~ internalTrafficPolicy = null -> (known after apply)
~ ipFamilies = null -> (known after apply)
~ ipFamilyPolicy = null -> (known after apply)
~ loadBalancerClass = null -> (known after apply)
~ loadBalancerIP = null -> (known after apply)
~ loadBalancerSourceRanges = null -> (known after apply)
~ ports = null -> (known after apply)
~ publishNotReadyAddresses = null -> (known after apply)
~ selector = null -> (known after apply)
~ sessionAffinity = null -> (known after apply)
~ sessionAffinityConfig = {
~ clientIP = {
~ timeoutSeconds = null -> (known after apply)
}
}
~ type = null -> (known after apply)
}
}
~ tls = {
~ certificate = {
~ secretName = null -> (known after apply)
}
~ selfSignedCertificate = {
~ subjectAltNames = null -> (known after apply)
# (1 unchanged element hidden)
}
}
}
~ image = null -> (known after apply)
~ monitoring = {
~ logs = {
~ elasticsearchRefs = null -> (known after apply)
}
~ metrics = {
~ elasticsearchRefs = null -> (known after apply)
}
}
~ nodeSets = [
~ {
name = "masters"
~ podTemplate = {
~ metadata = {
- creationTimestamp = null -> null
# (1 unchanged element hidden)
}
~ spec = {
~ containers = [
~ {
name = "elasticsearch"
~ securityContext = {
+ runAsNonRoot = true
+ runAsUser = 1000
}
# (1 unchanged element hidden)
},
]
}
}
~ volumeClaimTemplates = [
~ {
~ apiVersion = null -> (known after apply)
~ kind = null -> (known after apply)
~ metadata = {
~ annotations = null -> (known after apply)
~ finalizers = null -> (known after apply)
~ labels = null -> (known after apply)
name = "elasticsearch-data"
~ namespace = null -> (known after apply)
}
~ spec = {
~ dataSource = {
~ apiGroup = null -> (known after apply)
~ kind = null -> (known after apply)
~ name = null -> (known after apply)
}
~ dataSourceRef = {
~ apiGroup = null -> (known after apply)
~ kind = null -> (known after apply)
~ name = null -> (known after apply)
}
~ resources = {
~ limits = null -> (known after apply)
# (1 unchanged element hidden)
}
~ selector = {
~ matchExpressions = null -> (known after apply)
~ matchLabels = null -> (known after apply)
}
~ volumeMode = null -> (known after apply)
~ volumeName = null -> (known after apply)
# (2 unchanged elements hidden)
}
~ status = {
~ accessModes = null -> (known after apply)
~ allocatedResources = null -> (known after apply)
~ capacity = null -> (known after apply)
~ conditions = null -> (known after apply)
~ phase = null -> (known after apply)
~ resizeStatus = null -> (known after apply)
}
},
]
# (2 unchanged elements hidden)
},
]
~ podDisruptionBudget = {
~ metadata = {
~ annotations = null -> (known after apply)
~ finalizers = null -> (known after apply)
~ labels = null -> (known after apply)
~ name = null -> (known after apply)
~ namespace = null -> (known after apply)
}
~ spec = {
~ maxUnavailable = null -> (known after apply)
~ minAvailable = null -> (known after apply)
~ selector = {
~ matchExpressions = null -> (known after apply)
~ matchLabels = null -> (known after apply)
}
}
}
~ remoteClusters = null -> (known after apply)
~ secureSettings = null -> (known after apply)
~ serviceAccountName = null -> (known after apply)
~ transport = {
~ service = {
~ metadata = {
~ annotations = null -> (known after apply)
~ finalizers = null -> (known after apply)
~ labels = null -> (known after apply)
~ name = null -> (known after apply)
~ namespace = null -> (known after apply)
}
~ spec = {
~ allocateLoadBalancerNodePorts = null -> (known after apply)
~ clusterIP = null -> (known after apply)
~ clusterIPs = null -> (known after apply)
~ externalIPs = null -> (known after apply)
~ externalName = null -> (known after apply)
~ externalTrafficPolicy = null -> (known after apply)
~ healthCheckNodePort = null -> (known after apply)
~ internalTrafficPolicy = null -> (known after apply)
~ ipFamilies = null -> (known after apply)
~ ipFamilyPolicy = null -> (known after apply)
~ loadBalancerClass = null -> (known after apply)
~ loadBalancerIP = null -> (known after apply)
~ loadBalancerSourceRanges = null -> (known after apply)
~ ports = null -> (known after apply)
~ publishNotReadyAddresses = null -> (known after apply)
~ selector = null -> (known after apply)
~ sessionAffinity = null -> (known after apply)
~ sessionAffinityConfig = {
~ clientIP = {
~ timeoutSeconds = null -> (known after apply)
}
}
~ type = null -> (known after apply)
}
}
~ tls = {
~ certificate = {
~ secretName = null -> (known after apply)
}
~ otherNameSuffix = null -> (known after apply)
~ subjectAltNames = null -> (known after apply)
}
}
~ updateStrategy = {
~ changeBudget = {
~ maxSurge = null -> (known after apply)
~ maxUnavailable = null -> (known after apply)
}
}
~ volumeClaimDeletePolicy = null -> (known after apply)
# (1 unchanged element hidden)
}
# (2 unchanged elements hidden)
}
# (1 unchanged block hidden)
}
Im thinking of using some external PVC instead of using VolumeClaimTemplate but im not sure if its even possible.