Terraform want to recreate an aws_ecs_service after a scale down/up script despite config having not changed

Terraform
1

Hi everyone,

Context: I tried before holiday an aws lambda script that basically:
-Save the current scale parameters (min, max, desired) of an ECS services in the service tags
-Scale the services (on fargate and EC2) to 0
-Then when needed scale them back to the previous config

The problem is that when I tried to terraform plan after thatn I’m having an issue with Terraform where it plans to destroy and recreate an ECS service, even though the load balancer configuration in the Terraform state and the AWS CLI results appear identical.

Here is the extract of the terraform plan:

 # module.fargate.module.backend_fargate.aws_ecs_service.fargate_ecs_service["my-service-name"] must be replaced
-/+ resource "aws_ecs_service" "fargate_ecs_service" {
- health_check_grace_period_seconds  = 0 -> null
~ iam_role                           = "aws-service-role" -> (known after apply)
~ id                                 = "arn:aws:ecs:eu-west-1:<My Aws AccountID>:service/backend-fargate/development-my-service-name" -> (known after apply)
name                               = "development-my-service-name"
~ platform_version                   = "LATEST" -> (known after apply)
~ tags                               = {
"Terraformed"          = "true"
- "original_scaling_max" = "0" -> null
- "original_scaling_min" = "0" -> null
}
~ tags_all                           = {
- "original_scaling_max" = "0" -> null
- "original_scaling_min" = "0" -> null
# (1 unchanged element hidden)
}
~ task_definition                    = "development-my-service-name:35" -> "dummy-development-my-service-name"
# (10 unchanged attributes hidden)

- deployment_circuit_breaker {
- enable   = false -> null
- rollback = false -> null
}

- deployment_controller {
- type = "ECS" -> null
}

- load_balancer { # forces replacement
- container_name   = "development-my-service-name" -> null
- container_port   = <my_port_number> -> null
- target_group_arn = "arn:aws:elasticloadbalancing:eu-west-1:<My Aws AccountID>:targetgroup/tg-my-service-name/64493473f8861be2" -> null
}

# (1 unchanged block hidden)
}

And here is the state show:

# module.fargate.module.backend_fargate.aws_ecs_service.fargate_ecs_service["my-service-name"]:
resource "aws_ecs_service" "fargate_ecs_service" {
cluster                            = "arn:aws:ecs:eu-west-1:<my_accountid>:cluster/backend-fargate"
deployment_maximum_percent         = 200
deployment_minimum_healthy_percent = 50
desired_count                      = 2
enable_ecs_managed_tags            = true
enable_execute_command             = false
health_check_grace_period_seconds  = 0
iam_role                           = "aws-service-role"
id                                 = "arn:aws:ecs:eu-west-1:<my_accountid>:service/backend-fargate/development-my-service-name"
launch_type                        = "FARGATE"
name                               = "development-my-service-name"
platform_version                   = "LATEST"
propagate_tags                     = "SERVICE"
scheduling_strategy                = "REPLICA"
tags                               = {
"Terraformed" = "true"
}
tags_all                           = {
"Terraformed" = "true"
}
task_definition                    = "development-my-service-name:31"
wait_for_steady_state              = false

deployment_circuit_breaker {
enable   = false
rollback = false
}

deployment_controller {
type = "ECS"
}

load_balancer {
container_name   = "development-my-service-name"
container_port   = <my_port_number>
target_group_arn = "arn:aws:elasticloadbalancing:eu-west-1:<my_accountid>:targetgroup/tg-my-service-name/64493473f8861be2"
}

network_configuration {
assign_public_ip = false
security_groups  = [
"security-group-3",
"security-group-1",
"security-group-2",
]
subnets          = [
"subnet-1",
"subnet-2",
"subnet-3",
]
}
}

And finally the ecs_describe:


{
"services": [
{
"serviceArn": "arn:aws:ecs:eu-west-1:<my_accountid>:service/backend-fargate/development-my-service-name",
"serviceName": "development-my-service-name",
"clusterArn": "arn:aws:ecs:eu-west-1:<my_accountid>:cluster/backend-fargate",
"loadBalancers": [
{
"targetGroupArn": "arn:aws:elasticloadbalancing:eu-west-1:<my_accountid>:targetgroup/tg-my-service-name/64493473f8861be2",
"containerName": "development-my-service-name",
"containerPort": <my_port_number>
}
],
"serviceRegistries": [],
"status": "ACTIVE",
"desiredCount": 1,
"runningCount": 1,
"pendingCount": 0,
"launchType": "FARGATE",
"platformVersion": "LATEST",
"platformFamily": "Linux",
"taskDefinition": "arn:aws:ecs:eu-west-1:<my_accountid>:task-definition/development-my-service-name:35",
"deploymentConfiguration": {
"deploymentCircuitBreaker": {
"enable": false,
"rollback": false
},
"maximumPercent": 200,
"minimumHealthyPercent": 50
},
"deployments": [
{
"id": "ecs-svc/<ecs_svc_ID>",
"status": "PRIMARY",
"taskDefinition": "arn:aws:ecs:eu-west-1:<my_accountid>:task-definition/development-my-service-name:35",
"desiredCount": 1,
"pendingCount": 0,
"runningCount": 1,
"failedTasks": 0,
"createdAt": 1736327715.5,
"updatedAt": 1736327896.151,
"launchType": "FARGATE",
"platformVersion": "1.4.0",
"platformFamily": "Linux",
"networkConfiguration": {
"awsvpcConfiguration": {
"subnets": [
"subnet-1",
"subnet-3",
"subnet-2"
],
"securityGroups": [
"security-group-1",
"security-group-2",
"security-group-3"
],
"assignPublicIp": "DISABLED"
}
},
"rolloutState": "COMPLETED",
"rolloutStateReason": "ECS deployment ecs-svc/<ecs_svc_ID> completed."
}
],
"roleArn": "arn:aws:iam::<my_accountid>:role/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS",
"events": [
<some_events>
],
"createdAt": 1714753283.411,
"placementConstraints": [],
"placementStrategy": [],
"networkConfiguration": {
"awsvpcConfiguration": {
"subnets": [
"subnet-1",
"subnet-3",
"subnet-2"
],
"securityGroups": [
"security-group-1",
"security-group-2",
"security-group-3"
],
"assignPublicIp": "DISABLED"
}
},
"healthCheckGracePeriodSeconds": 0,
"schedulingStrategy": "REPLICA",
"deploymentController": {
"type": "ECS"
},
"createdBy": "arn:aws:iam::<my_accountid>:role/<my_role>",
"enableECSManagedTags": true,
"propagateTags": "SERVICE",
"enableExecuteCommand": false
}
],
"failures": []
}

As you can see the config are similar appart from the task definition which change because since my last apply some more version got deployed but this won’t be an issue because I have a:

lifecycle {
    ignore_changes = [
      desired_count,
      task_definition,
      tags,
      tags_all
    ]
  }

And I’m very confused as it really seems it’s the loadbalancer config that causes the replacement, as when I add it in the ignore_changes list, it solves the issue.
But that’s a dirty way to.

I want to understand why terraform detects changes where there isn’t and even more why it wants to destroy and recreate my aws_ecs_service resource.

I’m running short on idea and I would love some help with this

Thank you in advance for your time :pray:

2

Hi @GLimantour,

You didn’t show the current configuration, so we can’t say what exactly might be happening. Whatever sets the value for load_balancer has changed in some way though, so we ned to figure out why that block has changed.

3

Hi @jbardin,

Thank you for the feedback, here is the home made module I’m calling for creating my ecs fargate services, I’m showing only the aws_ecs_task_definition and aws_ecs_service but I can provide with others if needed:

resource "aws_ecs_task_definition" "fargate_ecs_task_definition" {
  for_each = var.services
  family   = replace("dummy-${local.env_suffix[var.env]}-${each.value["service_name"]}", "_", "-")
  cpu      = 256
  memory   = 512
  #  execution_role_arn       = aws_iam_role.ecs_role.arn
  task_role_arn            = aws_iam_role.fargate_ecs_task_role.arn
  requires_compatibilities = ["FARGATE"]
  container_definitions = jsonencode(
    [
      {
        name  = replace("${local.env_suffix[var.env]}-${each.value["service_name"]}", "_", "-")
        image = "containous/whoami"
        portMappings = [
          {
            containerPort = each.value["service_port"]
          },
          {
            hostPort      = 8126
            protocol      = "udp"
            containerPort = 8126
          }
        ]
        cpu = 64
        environment = []
        mountPoints = []
        dockerLabels = {
        }
        links      = []
        privileged = false
        volumes    = []
        service_load_balancers = [
          {
            target_group_arn = aws_alb_target_group.fargate_ecs_service_target_group[each.key].arn
            container_name   = replace("${local.env_suffix[var.env]}-${each.value["service_name"]}", "_", "-")
            container_port   = each.value["service_port"]
          }
        ]
      }
    ]
  )

  network_mode = "awsvpc"

  lifecycle {
    ignore_changes = [
      container_definitions # if template file changed, do nothing, believe that human's changes are source of truth
    ]
  }

  tags = {
    Terraformed = true
  }
}

resource "aws_ecs_service" "fargate_ecs_service" {
  for_each                           = var.services
  name                               = replace("${local.env_suffix[var.env]}-${each.value["service_name"]}", "_", "-")
  cluster                            = aws_ecs_cluster.fargate_ecs_cluster.id
  desired_count                      = each.value["service_min_size"]
  enable_ecs_managed_tags            = true
  propagate_tags                     = "SERVICE"
  deployment_minimum_healthy_percent = "50"
  launch_type                        = "FARGATE"
  network_configuration {
    subnets          = var.private_subnet_ids
    security_groups  = var.security_group_ids
    assign_public_ip = false
  }
  task_definition = aws_ecs_task_definition.fargate_ecs_task_definition[each.key].family



  lifecycle {
    ignore_changes = [
      desired_count,
      task_definition,
      tags,
      tags_all,
#      load_balancer
    ]
  }

  tags = {
    Terraformed = true
  }
}

And I don’t think anything wants to change the value of my loadbalancer considering the state and console are showing the same IDs?

4

The doesn’t have load_balancer set, which corresponds to the plan showing that the attributes are being changed to null. As to how those values were originally set I’m not sure, but this seems like a bug in the provider where it should be able to keep or ignore the defaults somehow.

I am guessing that the provider expects the user to always configure that if it’s in use, so the fix would probably be to add that missing portion of the aws_ecs_service to your configuration.