Hello folks.
Requirement: Customize filesystem on the built image. Thus, it needs to be done using autoinstall/kickstart before the system fully boots up.
Problem: The boot commands are not being run on time despite playing around with boot_wait time. Also verified by booting the built ISO, it is still the default filesystem.
Setup*:
Sharing a cleaned version of my setup here that I have tested to have replicated the problem
common.pkr.hcl
packer {
required_plugins {
qemu = {
source = "github.com/hashicorp/qemu"
version = "~> 1"
}
}
}
variable "arch" {
type = string
default = "x86_64"
}
variable "arch_vars" {
type = map(map(string))
default = {
"x86_64" = {
firmware = null
use_pflash = true
machine_type = "pc"
qemu_binary = "/usr/bin/qemu-system-x86_64"
}
"aarch64" = {
firmware = "/usr/share/AAVMF/AAVMF_CODE.fd"
use_pflash = false
machine_type = "virt,gic-version=max"
qemu_binary = "/usr/bin/qemu-system-aarch64"
}
}
}
alma.pkr.hcl
source "null" "null" { communicator = "none" }
build {
sources = ["sources.null.null"]
provisioner "shell-local" {
inline = [
"cloud-localds ${var.input_dir}/${var.appliance_name}-cloud-init.iso ${var.input_dir}/cloud-init.yml",
]
}
}
# Build VM image
source "qemu" "alma" {
cpus = 2
cpu_model = "host"
memory = 2048
accelerator = "kvm"
iso_url = lookup(lookup(var.alma, var.version, {}), "iso_url", "")
iso_checksum = lookup(lookup(var.alma, var.version, {}), "iso_checksum", "")
firmware = lookup(lookup(var.arch_vars, var.arch, {}), "firmware", "")
use_pflash = lookup(lookup(var.arch_vars, var.arch, {}), "use_pflash", "")
machine_type = lookup(lookup(var.arch_vars, var.arch, {}), "machine_type", "")
qemu_binary = lookup(lookup(var.arch_vars, var.arch, {}), "qemu_binary", "")
headless = var.headless
disk_image = true
disk_cache = "unsafe"
disk_interface = "virtio"
net_device = "virtio-net"
format = "qcow2"
disk_compression = false
skip_resize_disk = true
boot_wait = "-1s"
http_directory = "${var.input_dir}/http"
boot_command = [
"<tab><wait>",
" inst.ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ks.cfg",
" inst.text<enter>"
]
output_directory = var.output_dir
qemuargs = [
["-cdrom", "${var.input_dir}/${var.appliance_name}-cloud-init.iso"],
["-serial", "stdio"],
]
ssh_username = "root"
ssh_password = "opennebula"
ssh_timeout = "900s"
shutdown_command = "poweroff"
vm_name = "${var.appliance_name}"
}
build {
sources = ["source.qemu.alma"]
}
variables.pkr.hcl
variable "appliance_name" {
type = string
default = "alma"
}
variable "version" {
type = string
default = "8"
}
variable "input_dir" {
type = string
}
variable "output_dir" {
type = string
}
variable "headless" {
type = bool
default = false
}
variable "alma" {
type = map(map(string))
default = {
"8" = {
iso_url = "https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
iso_checksum = "file:https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/CHECKSUM"
}
"8.aarch64" = {
iso_url = "https://repo.almalinux.org/almalinux/8/cloud/aarch64/images/AlmaLinux-8-GenericCloud-latest.aarch64.qcow2"
iso_checksum = "file:https://repo.almalinux.org/almalinux/8/cloud/aarch64/images/CHECKSUM"
}
"9" = {
iso_url = "https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
iso_checksum = "file:https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/CHECKSUM"
}
"9.aarch64" = {
iso_url = "https://repo.almalinux.org/almalinux/9/cloud/aarch64/images/AlmaLinux-9-GenericCloud-latest.aarch64.qcow2"
iso_checksum = "file:https://repo.almalinux.org/almalinux/9/cloud/aarch64/images/CHECKSUM"
}
}
}
http/ks.cfg
#version=RHEL9
ignoredisk --only-use=vda
autopart --type=plain --fstype=xfs
clearpart --all --initlabel
Similarly for ubuntu
common.pkr.hcl
packer {
required_plugins {
qemu = {
source = "github.com/hashicorp/qemu"
version = "~> 1"
}
}
}
variable "arch" {
type = string
default = "x86_64"
}
variable "arch_vars" {
type = map(map(string))
default = {
"x86_64" = {
firmware = null
use_pflash = true
machine_type = "pc"
qemu_binary = "/usr/bin/qemu-system-x86_64"
}
"aarch64" = {
firmware = "/usr/share/AAVMF/AAVMF_CODE.fd"
use_pflash = false
machine_type = "virt,gic-version=max"
qemu_binary = "/usr/bin/qemu-system-aarch64"
}
}
}
ubuntu.pkr.hcl
source "null" "null" { communicator = "none" }
build {
sources = ["sources.null.null"]
provisioner "shell-local" {
inline = [
"cloud-localds ${var.input_dir}/${var.appliance_name}-cloud-init.iso ${var.input_dir}/cloud-init.yml",
]
}
}
locals {
disk_size = lookup({
"2204oneke" = 3072
"2204oneke.aarch64" = 3072
}, var.version, 0)
}
# Build VM image
source "qemu" "ubuntu" {
cpus = 2
cpu_model = "host"
memory = 2048
accelerator = "kvm"
iso_url = lookup(lookup(var.ubuntu, var.version, {}), "iso_url", "")
iso_checksum = lookup(lookup(var.ubuntu, var.version, {}), "iso_checksum", "")
firmware = lookup(lookup(var.arch_vars, var.arch, {}), "firmware", "")
use_pflash = lookup(lookup(var.arch_vars, var.arch, {}), "use_pflash", "")
machine_type = lookup(lookup(var.arch_vars, var.arch, {}), "machine_type", "")
qemu_binary = lookup(lookup(var.arch_vars, var.arch, {}), "qemu_binary", "")
headless = var.headless
disk_image = true
disk_cache = "unsafe"
disk_interface = "virtio"
net_device = "virtio-net"
format = "qcow2"
disk_compression = false
skip_resize_disk = local.disk_size == 0 ? true : false
disk_size = local.disk_size == 0 ? null : local.disk_size
boot_wait = "-1s"
http_directory = "${var.input_dir}/http"
boot_command = [
"c<wait10s>",
"linux /casper/vmlinuz autoinstall ds=\"nocloud-net;seedfrom=http://{{ .HTTPIP }}:{{ .HTTPPort }}/\" ---",
"<enter><wait5s>",
"initrd /casper/initrd",
"<enter><wait5s>",
"boot",
"<wait5s><enter><wait300>"
]
output_directory = var.output_dir
qemuargs = [
[ "-boot", "menu=on,splash-time=10000" ],
["-cdrom", "${var.input_dir}/${var.appliance_name}-cloud-init.iso"],
["-serial", "stdio"],
]
ssh_username = "root"
ssh_password = "opennebula"
ssh_timeout = "900s"
shutdown_command = "poweroff"
vm_name = "${var.appliance_name}"
}
build {
sources = ["source.qemu.ubuntu"]
}
variables.pkr.hcl
variable "appliance_name" {
type = string
default = "ubuntu"
}
variable "version" {
type = string
default = "2004"
}
variable "input_dir" {
type = string
}
variable "output_dir" {
type = string
}
variable "headless" {
type = bool
default = false
}
variable "ubuntu" {
type = map(map(string))
default = {
"2004" = {
iso_url = "https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/focal/current/SHA256SUMS"
}
"2004min" = {
iso_url = "https://cloud-images.ubuntu.com/minimal/releases/focal/release/ubuntu-20.04-minimal-cloudimg-amd64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/focal/release/SHA256SUMS"
}
"2204" = {
iso_url = "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/jammy/current/SHA256SUMS"
}
"2204.aarch64" = {
iso_url = "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-arm64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/jammy/current/SHA256SUMS"
}
"2204min" = {
iso_url = "https://cloud-images.ubuntu.com/minimal/releases/jammy/release/ubuntu-22.04-minimal-cloudimg-amd64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/jammy/release/SHA256SUMS"
}
"2404" = {
iso_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/noble/current/SHA256SUMS"
}
"2404.aarch64" = {
iso_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-arm64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/noble/current/SHA256SUMS"
}
"2404min" = {
iso_url = "https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-amd64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/noble/release/SHA256SUMS"
}
"2204oneke" = {
iso_url = "https://cloud-images.ubuntu.com/releases/jammy/release-20241206/ubuntu-22.04-server-cloudimg-amd64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/releases/jammy/release-20241206/SHA256SUMS"
}
"2204oneke.aarch64" = {
iso_url = "https://cloud-images.ubuntu.com/releases/jammy/release-20241206/ubuntu-22.04-server-cloudimg-arm64.img"
iso_checksum = "file:https://cloud-images.ubuntu.com/releases/jammy/release-20241206/SHA256SUMS"
}
}
}
http/user-data
autoinstall:
version: 1
identity:
hostname: custom_ubuntu
username: root
password: "$6$Ofuxo6O1xuT5PamW$zjR8s6OEa.qxK1u7z3i.AIzYmblEQDlFs5l2uNd58/J4SUwrGVp55T7y35dxbIVFf6RPjiBzZ1vpXzXW.5gJ41"
storage:
layout:
name: direct
config:
- type: disk
id: disk0
match:
size: largest
ptable: gpt
wipe: superblock-recursive
preserve: false
grub_device: true
- type: partition
id: root-partition
device: disk0
size: -1
- type: format
id: root-format
volume: root-partition
fstype: xfs
label: root
- type: mount
id: root-mount
device: root-format
path: /
packages:
- xfsprogs
late-commands:
- curtin in-target -- update-initramfs -u
- curtin in-target -- update-grub
http/meta-data
These scripts are from one-apps which is part of open nebula project. They have been working fine to customize rest of the OS after bootup. But, I am stuck with this particular point. Happy to share logs from a build if that helps clear things further.