Nested loop with foreach

Terraform
1

Hi all,

I have been following you for a long time and you have helped me many times… thank you very much!

So… i have little bit problem… and i don’t find out the solution

I need to deploy multiple VMs on Vsphere with multiple disks on multiple datastore…
I have compiled this module with this tfvars file, all is ok but i cannot get datastore.id by data.tf… :frowning:

when i run terraform plan, my result is it ok but the datastore id is not resolved by terraform but only take the “clear” data on tfvars… (will be not deploy inside it)… below the bold datastore.id

terraform plan --var site=US --var-file=prod.tfvars -out=prod.plan

# module.vm.vsphere_virtual_machine.vm["server1"] will be created
  + resource "vsphere_virtual_machine" "vm" {
      + annotation                              = "VM1 PRod"
      + boot_retry_delay                        = 10000
      + change_version                          = (known after apply)
      + cpu_hot_add_enabled                     = true
      + cpu_limit                               = -1
      + cpu_share_count                         = (known after apply)
      + cpu_share_level                         = "normal"
      + datastore_id                            = "datastore-1179"
      + default_ip_address                      = (known after apply)
      + ept_rvi_mode                            = "automatic"
      + extra_config_reboot_required            = true
      + firmware                                = "efi"
      + force_power_off                         = true
      + guest_id                                = "sles15_64Guest"
      + guest_ip_addresses                      = (known after apply)
      + hardware_version                        = (known after apply)
      + host_system_id                          = "host-1175"
      + hv_mode                                 = "hvAuto"
      + id                                      = (known after apply)
      + ide_controller_count                    = 2
      + imported                                = (known after apply)
      + latency_sensitivity                     = "normal"
      + memory                                  = 2048
      + memory_hot_add_enabled                  = true
      + memory_limit                            = -1
      + memory_share_count                      = (known after apply)
      + memory_share_level                      = "normal"
      + migrate_wait_timeout                    = 30
      + moid                                    = (known after apply)
      + name                                    = "server1"
      + num_cores_per_socket                    = 1
      + num_cpus                                = 8
      + power_state                             = (known after apply)
      + poweron_timeout                         = 300
      + reboot_required                         = (known after apply)
      + resource_pool_id                        = "resgroup-1177"
      + run_tools_scripts_after_power_on        = true
      + run_tools_scripts_after_resume          = true
      + run_tools_scripts_before_guest_shutdown = true
      + run_tools_scripts_before_guest_standby  = true
      + sata_controller_count                   = 0
      + scsi_bus_sharing                        = "noSharing"
      + scsi_controller_count                   = 4
      + scsi_type                               = "pvscsi"
      + shutdown_wait_timeout                   = 3
      + storage_policy_id                       = (known after apply)
      + swap_placement_policy                   = "inherit"
      + sync_time_with_host                     = true
      + tools_upgrade_policy                    = "manual"
      + uuid                                    = (known after apply)
      + vapp_transport                          = (known after apply)
      + vmware_tools_status                     = (known after apply)
      + vmx_path                                = (known after apply)
      + wait_for_guest_ip_timeout               = 0
      + wait_for_guest_net_routable             = true
      + wait_for_guest_net_timeout              = 0

      + clone {
          + template_uuid = "423deb75-fdf1-7553-b080-bd617bb1281d"
          + timeout       = 30

          + customize {
              + dns_server_list = [
                  + "10.237.216.7",
                  + "10.237.208.5",
                  + "10.237.111.8",
                ]
              + ipv4_gateway    = "10.237.113.1"
              + timeout         = 10

              + linux_options {
                  + domain       = "adgr.net"
                  + host_name    = "server1"
                  + hw_clock_utc = true
                }

              + network_interface {
                  + ipv4_address = "10.237.113.186"
                  + ipv4_netmask = 24
                }
            }
        }

      + disk {
          + attach            = false
          + controller_type   = "scsi"
          + datastore_id      = "<computed>"
          + device_address    = (known after apply)
          + disk_mode         = "persistent"
          + disk_sharing      = "sharingNone"
          + eagerly_scrub     = false
          + io_limit          = -1
          + io_reservation    = 0
          + io_share_count    = 0
          + io_share_level    = "normal"
          + keep_on_remove    = false
          + key               = 0
          + label             = "disk0"
          + path              = (known after apply)
          + size              = 80
          + storage_policy_id = (known after apply)
          + thin_provisioned  = true
          + unit_number       = 0
          + uuid              = (known after apply)
          + write_through     = false
        }
      + disk {
          + attach            = false
          + controller_type   = "scsi"
          + datastore_id      = "datastore1"
          + device_address    = (known after apply)
          + disk_mode         = "persistent"
          + disk_sharing      = "sharingNone"
          + eagerly_scrub     = false
          + io_limit          = -1
          + io_reservation    = 0
          + io_share_count    = 0
          + io_share_level    = "normal"
          + keep_on_remove    = false
          + key               = 0
          + label             = "0"
          + path              = (known after apply)
          + size              = 20
          + storage_policy_id = (known after apply)
          + thin_provisioned  = true
          + unit_number       = 10
          + uuid              = (known after apply)
          + write_through     = false
        }
      + disk {
          + attach            = false
          + controller_type   = "scsi"
          + datastore_id      = "LocalStorage103"
          + device_address    = (known after apply)
          + disk_mode         = "persistent"
          + disk_sharing      = "sharingNone"
          + eagerly_scrub     = false
          + io_limit          = -1
          + io_reservation    = 0
          + io_share_count    = 0
          + io_share_level    = "normal"
          + keep_on_remove    = false
          + key               = 0
          + label             = "1"
          + path              = (known after apply)
          + size              = 200
          + storage_policy_id = (known after apply)
          + thin_provisioned  = true
          + unit_number       = 30
          + uuid              = (known after apply)
          + write_through     = false
        }

      + network_interface {
          + adapter_type          = "vmxnet3"
          + bandwidth_limit       = -1
          + bandwidth_reservation = 0
          + bandwidth_share_count = (known after apply)
          + bandwidth_share_level = "normal"
          + device_address        = (known after apply)
          + key                   = (known after apply)
          + mac_address           = (known after apply)
          + network_id            = "network-12"
        }
    }

Plan: 2 to add, 0 to change, 0 to destroy.

Below my configuration…

main.tf

module "vm" {
  source = "/home/omar/terraform-vm-foreach/vm"



vsphere_user = var.vsphere_user
vsphere_password = var.vsphere_password
vsphere_server = var.vsphere_server
vsphere_unverified_ssl = var.vsphere_unverified_ssl
compute_cluster = var.compute_cluster
#resource_pool = var.resource_pool
datacenter = var.datacenter
virtual_machines = var.virtual_machines
site = var.site
change = var.change
}

prod.tfvars

vsphere_server         = "192.168.10.100"
vsphere_unverified_ssl = true
datacenter          = "Lab.local"
compute_cluster     = "Lab.local"
change = "CH0000999"


virtual_machines = {
  server1 = {
    datacenter           = "Lab.local"
    compute_cluster      = "Lab.local"
    rspool               = "max"
    network_interface    = "VM Network"
    vmnotes              = "VM1 PRod"
    category             = "dbserver"
    system_cores         = 8
    system_memory        = 2048
    system_disk          = 80
    system_ipv4_address  = "10.237.113.186"
    system_ipv4_netmask  = "24"
    system_ipv4_gateway  = "10.237.113.1"
    datastore            = "LocalStorage103"
    datastore_a          = "LocalStorage103"
    datastore_b          = "datastore1"
    folder               = "/"
    system_name          = "server1"
    system_domain        = "ad1.local"
    dns_server_list      = ["192.168.1.200", "192.168.1.201", "192.168.1.202"]
    disks                = [ {
                vdisklabel = "server1-disk16"
                vdisksize = "20"
                vdisknumber = "10"
                vdatastore = "datastore1"
                },
                {
                vdisklabel = "server2-disk30"
                vdisksize = "20"
                vdisknumber = "30"
                vdatastore = "datastore2"
                },
                   ]
}
}

variables.tf

variable "vsphere_user" {}
variable "vsphere_password" {}
variable "vsphere_server" {}
variable "vsphere_unverified_ssl" {}
variable "compute_cluster" {}
variable "datacenter" {}
variable "site" {}
variable "change" {}
variable "virtual_machines" {}

On VM folder (this is the module)

data.tf

locals{
  vds = var.site == "US" ? "vds_001" : "vds_002"
  hostesx = var.site == "US" ? "192.168.10.103" : "192.168.10.104"
  vmgroup = var.site == "US" ? "RheLicenseVM" : "RheLicenseVM"
  template = var.site == "US" ? "MicroOS-Temp" : "RHEL8_ZH_template"

  virtm = { for_each = var.virtual_machines}

  flat_sandboxes = {
    for sandbox in var.virtual_machines :
    sandbox.system_name => sandbox
  }

  network_subnets = flatten([
    for network_key, network in var.virtual_machines : [
      for subnet in network.disks : {
        network_key       = network_key
        purpose           = subnet.vdatastore
        namedisk          = subnet.vdisklabel
        sizedisk          = subnet.vdisksize
        numberdisk        = subnet.vdisknumber
      }
    ]
  ])

}


data "vsphere_datacenter" "datacenter" {
  name = var.datacenter
}

data "vsphere_resource_pool" "pool" {
  for_each = var.virtual_machines

  name          = each.value.rspool
  datacenter_id = data.vsphere_datacenter.datacenter.id
}

data "vsphere_virtual_machine" "template" {
  name          = "MicroOS-Temp"
  datacenter_id = data.vsphere_datacenter.datacenter.id
}


data "vsphere_datastore" "datastore" {
  for_each = var.virtual_machines

  name          = each.value.datastore
  datacenter_id = data.vsphere_datacenter.datacenter.id
}


data "vsphere_datastore" "datastore2" {
  for_each = {
      #for ns in local.network_subnets: ns.purpose => ns
       for index, ns in local.network_subnets : index => ns 
    }

  name          = each.value.purpose
  datacenter_id = data.vsphere_datacenter.datacenter.id
}


data "vsphere_datastore" "datastore_a" {
  for_each = var.virtual_machines
  name          = each.value.datastore_a
  datacenter_id = data.vsphere_datacenter.datacenter.id
}

data "vsphere_datastore" "datastore_b" {
  for_each = var.virtual_machines

  name          = each.value.datastore_b
  datacenter_id = data.vsphere_datacenter.datacenter.id
}

data "vsphere_network" "network_interface" {
  for_each = var.virtual_machines

  name          = each.value.network_interface
  datacenter_id = data.vsphere_datacenter.datacenter.id
}

data "vsphere_host" "host" {
  name          = local.hostesx
  datacenter_id = data.vsphere_datacenter.datacenter.id
}

instance.tf

resource "vsphere_virtual_machine" "vm" {
  for_each = var.virtual_machines

  name             = each.key
  resource_pool_id = "${data.vsphere_host.host.resource_pool_id}"

  guest_id  = data.vsphere_virtual_machine.template.guest_id
  scsi_type = data.vsphere_virtual_machine.template.scsi_type

  num_cpus = each.value.system_cores

  memory = each.value.system_memory

  host_system_id       = data.vsphere_host.host.id

  annotation = each.value.vmnotes
  firmware = "efi"
  sync_time_with_host = true

  cpu_hot_add_enabled = true

  memory_hot_add_enabled = true

  datastore_id       = data.vsphere_datastore.datastore[each.key].id
  folder   = each.value.folder


  wait_for_guest_ip_timeout  = 0
  wait_for_guest_net_timeout = 0


  scsi_controller_count = 4

  #Network
  network_interface {
    network_id   = data.vsphere_network.network_interface[each.key].id
    adapter_type = data.vsphere_virtual_machine.template.network_interface_types[0]
  }

  disk {
    label = "disk0"
    size  = each.value.system_disk
    eagerly_scrub    = false
    thin_provisioned = true
    #unit_number = 0
  }


dynamic "disk" {
    for_each = each.value.disks
    content {
      label = disk.key
      size = disk.value.vdisksize
      unit_number = disk.value.vdisknumber 
      datastore_id = disk.value.vdatastore
    }
  }

  #cloning from template
  clone {
    template_uuid = data.vsphere_virtual_machine.template.id

customize {
      linux_options {

        host_name = each.value.system_name
        domain    = each.value.system_domain

      }


      network_interface {
        ipv4_address = each.value.system_ipv4_address
        ipv4_netmask = each.value.system_ipv4_netmask
      }

      ipv4_gateway = each.value.system_ipv4_gateway
      dns_server_list = each.value.dns_server_list
    }

  }

}

resource "local_file" "change" {
  content = var.change
  filename = "${path.module}/change"
}

providers.tf

terraform {
  required_providers {
    vsphere = {
      source  = "hashicorp/vsphere"
    }
  }
  required_version = ">= 1.1.0"
}

provider "vsphere" {
  user                 = var.vsphere_user
  password             = var.vsphere_password
  vsphere_server       = var.vsphere_server
  allow_unverified_ssl = var.vsphere_unverified_ssl
}

variable.tf

variable "vsphere_user" {
  type        = string
  description = "User that connects to the vCenter."
  default     = "admin"
  sensitive   = true
}

variable "vsphere_password" {
  type        = string
  description = "Password of the user that connects to the vCenter."
  default     = "admin"
  sensitive   = true
}

variable "vsphere_server" {
  type        = string
  description = "vCenter URL."
  default     = "https://localhost"
}

variable "change" {
  type        = string
  description = "change"
  default     = ""
}



variable "vsphere_unverified_ssl" {
  type        = bool
  description = "Disable verification of vCenter server HTTPS certificate."
  default     = false
}


variable "compute_cluster" {
  type        = string
  description = "Cluster where to provision the servers of the group."
  default     = ""
}

variable "resource_pool" {
  type        = string
  description = "Cluster where to provision the servers of the group."
  default     = ""
}

variable "datacenter" {
  type        = string
  description = "Datacenter where to provision the servers of the group."
  default     = ""
}

variable "vds" {
  type        = string
  description = "select os option"
  default     = "vds"
}

variable "site" {}


variable "virtual_machines" {
  type = map(object({
    system_cores         = number
    system_memory        = number
    system_disk          = number
    system_ipv4_address  = string
    system_ipv4_netmask  = string
    system_ipv4_gateway  = string
    rspool               = string
    category             = string
    datastore            = string
    datastore_a          = string
    datastore_b          = string
    folder               = string
    compute_cluster      = string
    network_interface    = string
    vmnotes             = string
    system_name          = string
    system_domain        = string
    dns_server_list       = list(string)
    disks = list(object({
            vdisklabel=string
            vdisksize=number
            vdisknumber=number
            vdatastore=string}))
  }))

I hope to help me because i don’t understand where is my problem… :frowning:

Thx a lot :slight_smile:

Omar

2

Hi Omar,

Could you edit your post to use the preformatted text for your code blocks as it will make it a lot easier to read by honouring your indentation and not adding additional markdown formatting (such as the bullets). It will make it possible to copy/paste the code out of the post if people want to reproduce the scenario from your code.

You can use 3x ‘backtick’ characters on the line before and after your code blocks:
```
locals {
fred = “123”
}
```

Is then formatted as:

locals {
   fred = "123"
}

Or you can use the ‘<>’ button in the toolbar after highlighting code (This is good if you want to show something inline with your text.

I’ll take a look at this for you then, as it is hard to interpret quickly in its current format.

Thanks in advance.

3

Hi ExtelligenceIT, thx a lot for your suggest :slight_smile:

Maybe now is it better :slight_smile:

1 Like
4

So, I believe the question/issue related to the fact that your module is not providing the datastore_id to the vsphere_virtual_machine disk block as shown above with the <----

In your instance.tf file, in your dynamic "disk" you have the following:

datastore_id = disk.value.vdatastore (1)

where disk is for_each = each.value.disks (2)

and each.value.disks is coming from the for_each at the resource level (3):

resource "vsphere_virtual_machine" "vm" {
  for_each = var.virtual_machines

So if we follow that through from your prod.tfvars:

virtual_machines = {
  server1 = {                           <------- (3)
    disks                = [ {            <------- (2) 
                vdatastore = "datastore1" <------- (1)
                }
                   ]
   }
}

So the value that you are passing into the resources is as expected. However what you really want is the id of the disk as it is being returned from a data resource.

Looking at your available data resources I cannot see a vsphere_datastore data resource that would work for this at the moment.

My approach would be as follows:

  1. Create a local variable that contains a list of unique data stores based upon the content of the virtual_machines.disks list.
  2. Use that with a vsphere_datastore data resource to get the information
  3. Reference that vsphere_datastore data resource in your dynamic disk block, using the name of the datastore name referenced in the disk.value.vdatastore

Here is an illustration - I have shown how this works with multiple VMs in your map and with datastores that show the creation of the unique list.

Copy/Paste this into a main.tf in an empty folder and run init/apply to see it working and play with it as you need. You may well be able to take my datastores local and use it directly in your code, adding a new data resource and updating the dreference in your dynamic disk block to reference the data resource.

locals {
  # This is the 'input' variable containing the virtual machines and their disks
  virtual_machines = {
    server1 = {
      disks = [{
        vdisklabel  = "server1-disk16"
        vdisksize   = "20"
        vdisknumber = "10"
        vdatastore  = "datastore1"
        },
        {
          vdisklabel  = "server1-disk30"
          vdisksize   = "20"
          vdisknumber = "30"
          vdatastore  = "datastore2"
        },
      ]
    }
    server2 = {
      disks = [{
        vdisklabel  = "server2-disk16"
        vdisksize   = "20"
        vdisknumber = "10"
        vdatastore  = "datastore1"
        },
        {
          vdisklabel  = "server2-disk30"
          vdisksize   = "20"
          vdisknumber = "30"
          vdatastore  = "datastore3"
        },
      ]
    }
  }


  # this creates a flattened, distinct list of datastores for all virtual machines
  datastores = distinct(flatten(
    [for value in local.virtual_machines : [
      for disk in value.disks : disk.vdatastore
    ]]
  ))

  # altrenatively, you can use splatting to achieve the same result
  datastores_splat = distinct(flatten(
    [for value in local.virtual_machines :
      value.disks[*].vdatastore
    ]
  ))
}

# This null_resource that simulates the vsphere_datastore data resource
# We will use the id returned to simulate looking up the datastore.id
resource "null_resource" "vsphere_datastore" {
  for_each = toset(local.datastores_splat)

}

output "vsphere_datastore" {
  value       = null_resource.vsphere_datastore
  description = "value of the null_resource simulating the vsphere_datastore data resource"
}

output "local_datastores" {
  value       = local.datastores
  description = "value of the local variable datastores"
}

# The below outputs are for demonstration purposes showing how you would reference the id of the 
# datastore now using the datastore name contained in the disk.value.vdatastore
output "datastore1_id" {
  value = null_resource.vsphere_datastore["datastore1"].id
  #value = null_resource.vsphere_datastore[disk.value.vdatastore].id <---- Example of how you would reference the id of the datastore
}
output "datastore2_id" {
  value = null_resource.vsphere_datastore["datastore2"].id
}
output "datastore3_id" {
  value = null_resource.vsphere_datastore["datastore3"].id
}

Here are the Outputs from running this locally on my machine showing the desired result of referencing the datastores by name and getting the ID:

Outputs:

datastore1_id = "6256160518187887193"
datastore2_id = "5800893338962547104"
datastore3_id = "4069836872107381961"
local_datastores = tolist([
  "datastore1",
  "datastore2",
  "datastore3",
])
vsphere_datastore = {
  "datastore1" = {
    "id" = "6256160518187887193"
    "triggers" = tomap(null) /* of string */
  }
  "datastore2" = {
    "id" = "5800893338962547104"
    "triggers" = tomap(null) /* of string */
  }
  "datastore3" = {
    "id" = "4069836872107381961"
    "triggers" = tomap(null) /* of string */
  }
}

Hope that helps

Happy Terraforming

PS - Thanks for reformatting your original post!

5

This topic was automatically closed 180 days after the last reply. New replies are no longer allowed.