Templatefile() and cloud-init yaml issues

Terraform
1

Receiving the following error when trying to deploy an AWS BIGIP instance using a Terraform templatefile() function to pass variables to cloud-init template file,

tmos_do_only_wait - DEBUG - f5-declarative-onboarding error 422 - {u’message’: u’request failed with null exception’, u’code’: 422, u’referer’: u’Unknown’, u’restOperationId’: 6674184, u’kind’: u’:resterrorresponse’}

When I use static values along with the file() function in the user_data argument the BIGIP has no issues accepting the configuration. I’m just not sure if the issue is with the F5 or with Terraform modifying the structure of the file with the introduction of the templatefile() function. The template file is structured in YAML.

variables.tf

variable "name_servers" { default = "" }
variable "ltm_eth0_sg_id" { default = "" }
variable "ltm_eth0_subnet_id" { default = "" }
variable "ltm_eth0_subnet_cidr_block" { default = "" }
variable "ltm_eth1_sg_id" { default = "" }
variable "ltm_eth1_subnet_id" { default = "" }
variable "ltm_eth1_subnet_cidr_block" { default = "" }
variable "ltm_eth2_sg_id" { default = "" }
variable "ltm_eth2_subnet_id" { default = "" }
variable "ltm_eth2_subnet_cidr_block" { default = "" }
variable "ltm_eth3_sg_id" { default = "" }
variable "ltm_eth3_subnet_id" { default = "" }
variable "ltm_eth3_subnet_cidr_block" { default = "" }
variable "ltm_eth4_sg_id" { default = "" }
variable "ltm_eth4_subnet_id" { default = "" }
variable "ltm_eth4_subnet_cidr_block" { default = "" }
variable "ltm_eth5_sg_id" { default = "" }
variable "ltm_eth5_subnet_id" { default = "" }
variable "ltm_eth5_subnet_cidr_block" { default = "" }
variable "ltm_eth6_sg_id" { default = "" }
variable "ltm_eth6_subnet_id" { default = "" }
variable "ltm_eth6_subnet_cidr_block" { default = "" }
variable "ltm_eth7_sg_id" { default = "" }
variable "ltm_eth7_subnet_id" { default = "" }
variable "ltm_eth7_subnet_cidr_block" { default = "" }
variable "ltm2_eth0_intf_ip" { default = "" }
variable "ltm2_eth1_intf_ip" { default = "" }
variable "ltm2_eth2_intf_ip" { default = "" }
variable "ltm2_eth3_intf_ip" { default = "" }
variable "ltm2_eth4_intf_ip" { default = "" }
variable "ltm2_eth5_intf_ip" { default = "" }
variable "ltm2_eth6_intf_ip" { default = "" }
variable "ltm2_eth7_intf_ip" { default = "" }

main.tf

  module "ltm" {
  name_servers                   = ["169.254.169.253", "8.8.8.8"]
  ltm_eth0_sg_id                 = module.security_groups.ltm_mgmt_security_group_id
  ltm_eth0_subnet_id             = module.subnet.ltm_gtm_mgmt_subnet_id
  ltm_eth0_subnet_cidr_block     = module.subnet.ltm_gtm_mgmt_subnet_cidr_block
  ltm_eth1_sg_id                 = module.security_groups.ltm_routing_security_group_id
  ltm_eth1_subnet_id             = module.subnet.ltm_gtm_routing_subnet_id
  ltm_eth1_subnet_cidr_block     = module.subnet.ltm_gtm_routing_subnet_cidr_block
  ltm_eth2_sg_id                 = module.security_groups.ltm_external_security_group_id
  ltm_eth2_subnet_id             = module.subnet.ltm_ext_vip1_subnet_id
  ltm_eth2_subnet_cidr_block     = module.subnet.ltm_ext_vip1_subnet_cidr_block
  ltm_eth3_sg_id                 = module.security_groups.ltm_external_security_group_id
  ltm_eth3_subnet_id             = module.subnet.ltm_ext_vip2_subnet_id
  ltm_eth3_subnet_cidr_block     = module.subnet.ltm_ext_vip2_subnet_cidr_block
  ltm_eth4_sg_id                 = module.security_groups.ltm_external_security_group_id
  ltm_eth4_subnet_id             = module.subnet.ltm_ext_vip3_subnet_id
  ltm_eth4_subnet_cidr_block     = module.subnet.ltm_ext_vip3_subnet_cidr_block
  ltm_eth5_sg_id                 = module.security_groups.ltm_internal_security_group_id
  ltm_eth5_subnet_id             = module.subnet.ltm_int_vip1_subnet_id
  ltm_eth5_subnet_cidr_block     = module.subnet.ltm_int_vip1_subnet_cidr_block
  ltm_eth6_sg_id                 = module.security_groups.ltm_internal_security_group_id
  ltm_eth6_subnet_id             = module.subnet.ltm_int_vip2_subnet_id
  ltm_eth6_subnet_cidr_block     = module.subnet.ltm_int_vip2_subnet_cidr_block
  ltm_eth7_sg_id                 = module.security_groups.ltm_internal_security_group_id
  ltm_eth7_subnet_id             = module.subnet.ltm_int_vip3_subnet_id
  ltm_eth7_subnet_cidr_block     = module.subnet.ltm_int_vip3_subnet_cidr_block
  ltm2_eth0_intf_ip              = ["10.169.96.7"]
  ltm2_eth1_intf_ip              = ["10.169.96.72"]
  ltm2_eth2_intf_ip              = ["10.169.97.40"]
  ltm2_eth3_intf_ip              = ["10.169.98.40"]
  ltm2_eth4_intf_ip              = ["10.169.99.40"]
  ltm2_eth5_intf_ip              = ["10.169.100.40"]
  ltm2_eth6_intf_ip              = ["10.169.101.40"]
  ltm2_eth7_intf_ip              = ["10.169.102.40"]
  }

~ltm2_main.tf~
resource "aws_network_interface" "ltm2_eth0_intf" {
  subnet_id         = var.ltm_eth0_subnet_id
  private_ips       = var.ltm2_eth0_intf_ip
  security_groups   = [var.ltm_eth0_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH0"
  }
}
resource "aws_network_interface" "ltm2_eth1_intf" {
  subnet_id         = var.ltm_eth1_subnet_id
  private_ips       = var.ltm2_eth1_intf_ip
  security_groups   = [var.ltm_eth1_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH1"
  }
}
resource "aws_network_interface" "ltm2_eth2_intf" {
  subnet_id         = var.ltm_eth2_subnet_id
  private_ips       = var.ltm2_eth2_intf_ip
  security_groups   = [var.ltm_eth2_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH2"
  }
}

resource "aws_network_interface" "ltm2_eth3_intf" {
  subnet_id         = var.ltm_eth3_subnet_id
  private_ips       = var.ltm2_eth3_intf_ip
  security_groups   = [var.ltm_eth3_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH3"
  }
}
resource "aws_network_interface" "ltm2_eth4_intf" {
  subnet_id         = var.ltm_eth4_subnet_id
  private_ips       = var.ltm2_eth4_intf_ip
  security_groups   = [var.ltm_eth4_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH4"
  }
}
resource "aws_network_interface" "ltm2_eth5_intf" {
  subnet_id         = var.ltm_eth5_subnet_id
  private_ips       = var.ltm2_eth5_intf_ip
  security_groups   = [var.ltm_eth5_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH5"
  }
}
resource "aws_network_interface" "ltm2_eth6_intf" {
  subnet_id         = var.ltm_eth6_subnet_id
  private_ips       = var.ltm2_eth6_intf_ip
  security_groups   = [var.ltm_eth6_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH6"
  }
}
resource "aws_network_interface" "ltm2_eth7_intf" {
  subnet_id         = var.ltm_eth7_subnet_id
  private_ips       = var.ltm2_eth7_intf_ip
  security_groups   = [var.ltm_eth7_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH7"
  }
}
resource "aws_instance" "ltm2" {
  ami           = "ami-07cda73ac7991dfc8"    
  instance_type = "c4.8xlarge"
  key_name      = "ltm2-key"
  disable_api_termination = "false"
  ebs_optimized           = "false"
  root_block_device {
    #device_name = "/dev/xvda"
    volume_type = "standard"
    volume_size = "142"
  }
  ebs_block_device {
    device_name = "/dev/xvdb"
    volume_type = "standard"
    volume_size = "20"
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth0_intf[0].id
    device_index         = 0
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth1_intf[0].id
    device_index         = 1
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth2_intf[0].id
    device_index         = 2
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth3_intf[0].id
    device_index         = 3
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth4_intf[0].id
    device_index         = 4
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth5_intf[0].id
    device_index         = 5
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth6_intf[0].id
    device_index         = 6
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth7_intf[0].id
    device_index         = 7
  }
  user_data = templatefile("${path.module}/ltm.tftpl", {
    hostname         = "ltm2-lab"
    name_server1     = var.name_servers[0]
    name_server2     = var.name_servers[1]
    ltm_ext_vips1_ip = var.ltm2_eth2_intf_ip[0]
    ltm_ext_vips2_ip = var.ltm2_eth3_intf_ip[0]
    ltm_ext_vips3_ip = var.ltm2_eth4_intf_ip[0]
    ltm_int_vips1_ip = var.ltm2_eth5_intf_ip[0]
    ltm_int_vips2_ip = var.ltm2_eth6_intf_ip[0]
    ltm_int_vips3_ip = var.ltm2_eth7_intf_ip[0]
    ltm_traffic_ip   = var.ltm2_eth1_intf_ip[0]
    admin_pwd        = "Password1234"
  })

  tags = {
    Name = "ltm2-lab"
  }
}

ltm.tftpl

#cloud-config
tmos_declared:
  enabled: true
  icontrollx_trusted_sources: false
  icontrollx_package_urls:
    - "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.37.0/f5-declarative-onboarding-1.37.0-3.noarch.rpm"
    - "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.44.0/f5-appsvcs-3.44.0-3.noarch.rpm"
  do_declaration:
    schemaVersion: 1.0.0
    class: Device
    async: true
    label: Cloudinit Onboarding
    Common:
      class: Tenant
      provisioningLevels:
        class: Provision
        ltm: dedicated
      System:
        class: System
        hostname: ${hostname}
        mgmtDhcpEnabled: false
        autoCheck: false
        autoPhonehome: true
      dnsServers:
        class: DNS
        nameServers:
          - ${name_server1}
          - ${name_server2}
        search:
          - ops.sfdc.net
      ntpServers:
        class: NTP
        servers:
          - ${name_server1}
          - ${name_server2}
      EXT-VIPS-1-VLAN:
        class: VLAN
        tag: 102
        interfaces:
          - name: "1.2"
            tagged: false
      SelfIp_EXT-VIPS-1:
        class: SelfIp
        address: ${ltm_ext_vips1_ip}/24
        vlan: EXT-VIPS-1-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      EXT-VIPS-2-VLAN:
        class: VLAN
        tag: 103
        interfaces:
          - name: "1.3"
            tagged: false
      SelfIp_EXT-VIPS-2:
        class: SelfIp
        address: ${ltm_ext_vips2_ip}/24
        vlan: EXT-VIPS-2-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      EXT-VIPS-3-VLAN:
        class: VLAN
        tag: 104
        interfaces:
          - name: "1.4"
            tagged: false
      SelfIp_EXT-VIPS-3:
        class: SelfIp
        address: ${ltm_ext_vips3_ip}/24
        vlan: EXT-VIPS-3-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      INT-VIPS-1-VLAN:
        class: VLAN
        tag: 105
        interfaces:
          - name: "1.5"
            tagged: false
      SelfIp_INT-VIPS-1:
        class: SelfIp
        address: ${ltm_int_vips1_ip}/24
        vlan: INT-VIPS-1-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      INT-VIPS-2-VLAN:
        class: VLAN
        tag: 106
        interfaces:
          - name: "1.6"
            tagged: false
      SelfIp_INT-VIPS-2:
        class: SelfIp
        address: ${ltm_int_vips2_ip}/24
        vlan: INT-VIPS-2-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      INT-VIPS-3-VLAN:
        class: VLAN
        tag: 107
        interfaces:
          - name: "1.7"
            tagged: false
      SelfIp_INT-VIPS-3:
        class: SelfIp
        address: ${ltm_int_vips3_ip}/24
        vlan: INT-VIPS-3-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      TRAFFIC-VLAN:
        class: VLAN
        tag: 101
        interfaces:
          - name: "1.1"
            tagged: false
      SelfIp_TRAFFIC:
        class: SelfIp
        address: ${ltm_traffic_ip}/27
        vlan: TRAFFIC-VLAN
        trafficGroup: traffic-group-local-only
        allowService: default
chpasswd:
  list: |
    admin:${admin_pwd}
  expire: False

ltm.tfpl- rendered output

#cloud-config
tmos_declared:
  enabled: true
  icontrollx_trusted_sources: false
  icontrollx_package_urls:
    - "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.37.0/f5-declarative-onboarding-1.37.0-3.noarch.rpm"
    - "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.44.0/f5-appsvcs-3.44.0-3.noarch.rpm"
  do_declaration:
    schemaVersion: 1.0.0
    class: Device
    async: true
    label: Cloudinit Onboarding
    Common:
      class: Tenant
      provisioningLevels:
        class: Provision
        ltm: dedicated
      System:
        class: System
        hostname: dpl2-labnetsec
        mgmtDhcpEnabled: false
        autoCheck: false
        autoPhonehome: true
      dnsServers:
        class: DNS
        nameServers:
          - 169.254.169.253
          - 8.8.8.8
        search:
          - ops.sfdc.net
      ntpServers:
        class: NTP
        servers:
          - 169.254.169.253
          - 8.8.8.8
      EXT-VIPS-1-VLAN:
        class: VLAN
        tag: 102
        interfaces:
          - name: "1.2"
            tagged: false
      SelfIp_EXT-VIPS-1:
        class: SelfIp
        address: 10.169.97.40/24
        vlan: EXT-VIPS-1-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      EXT-VIPS-2-VLAN:
        class: VLAN
        tag: 103
        interfaces:
          - name: "1.3"
            tagged: false
      SelfIp_EXT-VIPS-2:
        class: SelfIp
        address: 10.169.98.40/24
        vlan: EXT-VIPS-2-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      EXT-VIPS-3-VLAN:
        class: VLAN
        tag: 104
        interfaces:
          - name: "1.4"
            tagged: false
      SelfIp_EXT-VIPS-3:
        class: SelfIp
        address: 10.169.99.40/24
        vlan: EXT-VIPS-3-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      INT-VIPS-1-VLAN:
        class: VLAN
        tag: 105
        interfaces:
          - name: "1.5"
            tagged: false
      SelfIp_INT-VIPS-1:
        class: SelfIp
        address: 10.169.100.40/24
        vlan: INT-VIPS-1-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      INT-VIPS-2-VLAN:
        class: VLAN
        tag: 106
        interfaces:
          - name: "1.6"
            tagged: false
      SelfIp_INT-VIPS-2:
        class: SelfIp
        address: 10.169.101.40/24
        vlan: INT-VIPS-2-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      INT-VIPS-3-VLAN:
        class: VLAN
        tag: 107
        interfaces:
          - name: "1.7"
            tagged: false
      SelfIp_INT-VIPS-3:
        class: SelfIp
        address: 10.169.102.40/24
        vlan: INT-VIPS-3-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      TRAFFIC-VLAN:
        class: VLAN
        tag: 101
        interfaces:
          - name: "1.1"
            tagged: false
      SelfIp_TRAFFIC:
        class: SelfIp
        address: 10.169.96.72/27
        vlan: TRAFFIC-VLAN
        trafficGroup: traffic-group-local-only
        allowService: default
chpasswd:
  list: |
    admin:eni-abcd1234
  expire: False
2

In that case, and since you have already obtained the rendered output to show to us in your post, couldn’t you just use a diff tool to compare the working static input with the rendered output, to see what is actually different?

1 Like