im running
http-server port 8080
iperf3 port 8081
openssh port 22
running iperf3 from host:
~> iperf3 -c 0.0.0.0 -p 8081 -t 20
iperf3: error - unable to connect to server - server may have stopped running or use a different port, firewall issue, etc.: Connection refused
after using tc to redirect traffic between eth0 & microvm-tap
~> tc qdisc add dev eth0 ingress
~> tc filter add dev eth0 parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev microvm-tap
~> tc qdisc add dev microvm-tap ingress
~> tc filter add dev microvm-tap parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev eth0
iperf3 now throws no route to host
~> iperf3 -c 0.0.0.0 -p 8081 -t 20
iperf3: error - unable to connect to server - server may have stopped running or use a different port, firewall issue, etc.: No route to host
trying to access 0.0.0.0:8080 from the browser shows
This site can’t be reached
http://0.0.0.0:8080/ is unreachable.
ERR_ADDRESS_UNREACHABLE
job definition
{
"Affinities": null,
"AllAtOnce": false,
"Constraints": null,
"ConsulNamespace": "",
"ConsulToken": "",
"CreateIndex": 29471,
"Datacenters": [
"dc1"
],
"DispatchIdempotencyToken": "",
"Dispatched": false,
"ID": "microvm",
"JobModifyIndex": 47653,
"Meta": null,
"ModifyIndex": 47677,
"Multiregion": null,
"Name": "microvm",
"Namespace": "default",
"NodePool": "default",
"NomadTokenID": "",
"ParameterizedJob": null,
"ParentID": "",
"Payload": null,
"Periodic": null,
"Priority": 50,
"Region": "global",
"Spreads": null,
"Stable": true,
"Status": "running",
"StatusDescription": "",
"Stop": false,
"SubmitTime": 1712197875563038000,
"TaskGroups": [
{
"Affinities": null,
"Constraints": [
{
"LTarget": "${attr.os.signals}",
"Operand": "set_contains",
"RTarget": "SIGCONT"
},
{
"LTarget": "${attr.consul.version}",
"Operand": "semver",
"RTarget": ">= 1.8.0"
}
],
"Consul": {
"Cluster": "default",
"Namespace": "",
"Partition": ""
},
"Count": 1,
"EphemeralDisk": {
"Migrate": false,
"SizeMB": 300,
"Sticky": false
},
"MaxClientDisconnect": null,
"Meta": null,
"Migrate": {
"HealthCheck": "checks",
"HealthyDeadline": 300000000000,
"MaxParallel": 1,
"MinHealthyTime": 10000000000
},
"Name": "servers",
"Networks": [
{
"CIDR": "",
"DNS": null,
"Device": "",
"DynamicPorts": [
{
"HostNetwork": "default",
"Label": "connect-proxy-microvm",
"To": -1,
"Value": 0
}
],
"IP": "",
"MBits": 0,
"Mode": "bridge",
"ReservedPorts": [
{
"HostNetwork": "default",
"Label": "http",
"To": 8080,
"Value": 8080
},
{
"HostNetwork": "default",
"Label": "iperf",
"To": 8081,
"Value": 8081
},
{
"HostNetwork": "default",
"Label": "ssh",
"To": 22,
"Value": 22
}
]
}
],
"PreventRescheduleOnLost": false,
"ReschedulePolicy": {
"Attempts": 0,
"Delay": 90000000000,
"DelayFunction": "exponential",
"Interval": 0,
"MaxDelay": 3600000000000,
"Unlimited": true
},
"RestartPolicy": {
"Attempts": 3,
"Delay": 3000000000,
"Interval": 60000000000,
"Mode": "fail",
"RenderTemplates": false
},
"Scaling": null,
"Services": [
{
"Address": "",
"AddressMode": "auto",
"CanaryMeta": null,
"CanaryTags": null,
"Checks": null,
"Cluster": "default",
"Connect": {
"Gateway": null,
"Native": false,
"SidecarService": {
"DisableDefaultTCPCheck": false,
"Meta": null,
"Port": "",
"Proxy": null,
"Tags": null
},
"SidecarTask": null
},
"EnableTagOverride": false,
"Identity": null,
"Meta": null,
"Name": "microvm",
"Namespace": "default",
"OnUpdate": "require_healthy",
"PortLabel": "8080",
"Provider": "consul",
"TaggedAddresses": null,
"Tags": null,
"TaskName": ""
}
],
"ShutdownDelay": null,
"Spreads": null,
"StopAfterClientDisconnect": null,
"Tasks": [
{
"Actions": null,
"Affinities": null,
"Artifacts": null,
"CSIPluginConfig": null,
"Config": {
"command": "local/add-interface-microvm-tap.sh"
},
"Constraints": null,
"Consul": null,
"DispatchPayload": null,
"Driver": "raw_exec",
"Env": null,
"Identities": null,
"Identity": {
"Audience": [
"nomadproject.io"
],
"ChangeMode": "",
"ChangeSignal": "",
"Env": false,
"File": false,
"Name": "default",
"ServiceName": "",
"TTL": 0
},
"KillSignal": "",
"KillTimeout": 5000000000,
"Kind": "",
"Leader": false,
"Lifecycle": {
"Hook": "prestart",
"Sidecar": false
},
"LogConfig": {
"Disabled": false,
"MaxFileSizeMB": 10,
"MaxFiles": 10
},
"Meta": null,
"Name": "add-interface-microvm-tap",
"Resources": {
"CPU": 100,
"Cores": 0,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 300,
"MemoryMaxMB": 0,
"NUMA": null,
"Networks": null
},
"RestartPolicy": {
"Attempts": 3,
"Delay": 3000000000,
"Interval": 60000000000,
"Mode": "fail",
"RenderTemplates": false
},
"ScalingPolicies": null,
"Services": null,
"ShutdownDelay": 0,
"Templates": [
{
"ChangeMode": "restart",
"ChangeScript": null,
"ChangeSignal": "",
"DestPath": "local/add-interface-microvm-tap.sh",
"EmbeddedTmpl": "#! /run/current-system/sw/bin/bash -e\n# ip tuntap add microvm-tap mode tap user microvm\n# ip link set microvm-tap up\nIFACE=\"microvm-tap\"\n\nif [ -d /sys/class/net/\"$IFACE\" ]; then\n echo \"WARNING: Removing stale tap interface \"$IFACE\"\" >&2\n ip tuntap del \"$IFACE\" mode tap || true\nfi\nip tuntap add \"$IFACE\" mode tap user microvm\nip link set \"$IFACE\" up\ntc qdisc add dev eth0 ingress\ntc filter add dev eth0 parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev microvm-tap\ntc qdisc add dev microvm-tap ingress\ntc filter add dev microvm-tap parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev eth0\n",
"Envvars": false,
"ErrMissingKey": false,
"Gid": null,
"LeftDelim": "{{",
"Perms": "755",
"RightDelim": "}}",
"SourcePath": "",
"Splay": 5000000000,
"Uid": null,
"VaultGrace": 0,
"Wait": null
}
],
"User": "root",
"Vault": null,
"VolumeMounts": null
},
{
"Actions": null,
"Affinities": null,
"Artifacts": null,
"CSIPluginConfig": null,
"Config": {
"command": "local/copy-system.sh"
},
"Constraints": null,
"Consul": null,
"DispatchPayload": null,
"Driver": "raw_exec",
"Env": null,
"Identities": null,
"Identity": {
"Audience": [
"nomadproject.io"
],
"ChangeMode": "",
"ChangeSignal": "",
"Env": false,
"File": false,
"Name": "default",
"ServiceName": "",
"TTL": 0
},
"KillSignal": "",
"KillTimeout": 5000000000,
"Kind": "",
"Leader": false,
"Lifecycle": {
"Hook": "prestart",
"Sidecar": false
},
"LogConfig": {
"Disabled": false,
"MaxFileSizeMB": 10,
"MaxFiles": 10
},
"Meta": null,
"Name": "copy_system",
"Resources": {
"CPU": 100,
"Cores": 0,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 300,
"MemoryMaxMB": 0,
"NUMA": null,
"Networks": null
},
"RestartPolicy": {
"Attempts": 3,
"Delay": 3000000000,
"Interval": 60000000000,
"Mode": "fail",
"RenderTemplates": false
},
"ScalingPolicies": null,
"Services": null,
"ShutdownDelay": 0,
"Templates": [
{
"ChangeMode": "restart",
"ChangeScript": null,
"ChangeSignal": "",
"DestPath": "local/copy-system.sh",
"EmbeddedTmpl": "#! /run/current-system/sw/bin/bash -e\n\nif ! [ -e /nix/store/a16xckk4nl9xjbbfhr8a85v1ih12afib-microvm-cloud-hypervisor-microvm ] ; then\n /run/current-system/sw/bin/nix copy --from file://@binaryCachePath@?trusted=1 --no-check-sigs /nix/store/a16xckk4nl9xjbbfhr8a85v1ih12afib-microvm-cloud-hypervisor-microvm\nfi\n",
"Envvars": false,
"ErrMissingKey": false,
"Gid": null,
"LeftDelim": "{{",
"Perms": "755",
"RightDelim": "}}",
"SourcePath": "",
"Splay": 5000000000,
"Uid": null,
"VaultGrace": 0,
"Wait": null
}
],
"User": "",
"Vault": null,
"VolumeMounts": null
},
{
"Actions": null,
"Affinities": null,
"Artifacts": null,
"CSIPluginConfig": null,
"Config": {
"command": "local/delete-interface-microvm-tap.sh"
},
"Constraints": null,
"Consul": null,
"DispatchPayload": null,
"Driver": "raw_exec",
"Env": null,
"Identities": null,
"Identity": {
"Audience": [
"nomadproject.io"
],
"ChangeMode": "",
"ChangeSignal": "",
"Env": false,
"File": false,
"Name": "default",
"ServiceName": "",
"TTL": 0
},
"KillSignal": "",
"KillTimeout": 5000000000,
"Kind": "",
"Leader": false,
"Lifecycle": {
"Hook": "poststop",
"Sidecar": false
},
"LogConfig": {
"Disabled": false,
"MaxFileSizeMB": 10,
"MaxFiles": 10
},
"Meta": null,
"Name": "delete-interface-microvm-tap",
"Resources": {
"CPU": 100,
"Cores": 0,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 300,
"MemoryMaxMB": 0,
"NUMA": null,
"Networks": null
},
"RestartPolicy": {
"Attempts": 3,
"Delay": 3000000000,
"Interval": 60000000000,
"Mode": "fail",
"RenderTemplates": false
},
"ScalingPolicies": null,
"Services": null,
"ShutdownDelay": 0,
"Templates": [
{
"ChangeMode": "restart",
"ChangeScript": null,
"ChangeSignal": "",
"DestPath": "local/delete-interface-microvm-tap.sh",
"EmbeddedTmpl": "#! /run/current-system/sw/bin/bash\nIFACE=\"microvm-tap\"\n# ip link set \"$IFACE\" down\n# ip tuntap del \"$IFACE\" mode tap\nIFACE=\"microvm-tap\"\n\nif [ -d /sys/class/net/\"$IFACE\" ]; then\n echo \"WARNING: Removing stale tap interface \"$IFACE\"\" >&2\n ip tuntap del \"$IFACE\" mode tap || true\nfi\nip tuntap add \"$IFACE\" mode tap user microvm\nip link set \"$IFACE\" up\n",
"Envvars": false,
"ErrMissingKey": false,
"Gid": null,
"LeftDelim": "{{",
"Perms": "755",
"RightDelim": "}}",
"SourcePath": "",
"Splay": 5000000000,
"Uid": null,
"VaultGrace": 0,
"Wait": null
}
],
"User": "root",
"Vault": null,
"VolumeMounts": null
},
{
"Actions": null,
"Affinities": null,
"Artifacts": null,
"CSIPluginConfig": null,
"Config": {
"command": "local/hypervisor.sh"
},
"Constraints": null,
"Consul": null,
"DispatchPayload": null,
"Driver": "raw_exec",
"Env": null,
"Identities": null,
"Identity": {
"Audience": [
"nomadproject.io"
],
"ChangeMode": "",
"ChangeSignal": "",
"Env": false,
"File": false,
"Name": "default",
"ServiceName": "",
"TTL": 0
},
"KillSignal": "SIGCONT",
"KillTimeout": 95000000000,
"Kind": "",
"Leader": true,
"Lifecycle": null,
"LogConfig": {
"Disabled": false,
"MaxFileSizeMB": 10,
"MaxFiles": 10
},
"Meta": null,
"Name": "hypervisor",
"Resources": {
"CPU": 100,
"Cores": 0,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 300,
"MemoryMaxMB": 0,
"NUMA": null,
"Networks": null
},
"RestartPolicy": {
"Attempts": 3,
"Delay": 3000000000,
"Interval": 60000000000,
"Mode": "fail",
"RenderTemplates": false
},
"ScalingPolicies": null,
"Services": null,
"ShutdownDelay": 0,
"Templates": [
{
"ChangeMode": "restart",
"ChangeScript": null,
"ChangeSignal": "",
"DestPath": "local/hypervisor.sh",
"EmbeddedTmpl": "#! /run/current-system/sw/bin/bash -e\n\nmkdir -p /run/microvms/svr/test/my-microvm\ncd /run/microvms/svr/test/my-microvm\n\n# start hypervisor\n/nix/store/a16xckk4nl9xjbbfhr8a85v1ih12afib-microvm-cloud-hypervisor-microvm/bin/microvm-run &\n# e2:f5:78:c9:82:3d\n# ip a >&2\n# /nix/store/1dqbn07n9x3yr4m39k1p4lz6yjpr8977-toybox-0.8.5/bin/netstat -nlp >&2\n# /nix/store/ybi39s1z4cz2ka2z94n2h861w7z62zxr-nmap-7.92/bin/nping --tcp --ttl 255 -p 8080 192.168.246.1 >&2\n# /nix/store/ybi39s1z4cz2ka2z94n2h861w7z62zxr-nmap-7.92/bin/nping --tcp --ttl 255 -p 8081 192.168.246.1 >&2\n# /nix/store/ybi39s1z4cz2ka2z94n2h861w7z62zxr-nmap-7.92/bin/nping --tcp --ttl 255 -p 8088 192.168.246.1 >&2\n# /nix/store/7b0fmrb1q3ljc2mfw2dy5z9wvbamqhyg-curl-7.76.1-bin/bin/curl 192.168.246.1:8080 >&2\n\n# stop hypervisor on signal\nfunction handle_signal() {\n echo \"Received signal, shutting down\" >&2\n date >&2\n /nix/store/a16xckk4nl9xjbbfhr8a85v1ih12afib-microvm-cloud-hypervisor-microvm/bin/microvm-shutdown\n echo \"Done\" >&2\n date >&2\n exit\n}\ntrap handle_signal CONT\nwait\n",
"Envvars": false,
"ErrMissingKey": false,
"Gid": null,
"LeftDelim": "{{",
"Perms": "755",
"RightDelim": "}}",
"SourcePath": "",
"Splay": 5000000000,
"Uid": null,
"VaultGrace": 0,
"Wait": null
}
],
"User": "root",
"Vault": null,
"VolumeMounts": null
},
{
"Actions": null,
"Affinities": null,
"Artifacts": null,
"CSIPluginConfig": null,
"Config": {
"command": "local/virtiofsd-ro-store.sh"
},
"Constraints": null,
"Consul": null,
"DispatchPayload": null,
"Driver": "raw_exec",
"Env": null,
"Identities": null,
"Identity": {
"Audience": [
"nomadproject.io"
],
"ChangeMode": "",
"ChangeSignal": "",
"Env": false,
"File": false,
"Name": "default",
"ServiceName": "",
"TTL": 0
},
"KillSignal": "",
"KillTimeout": 5000000000,
"Kind": "",
"Leader": false,
"Lifecycle": {
"Hook": "prestart",
"Sidecar": true
},
"LogConfig": {
"Disabled": false,
"MaxFileSizeMB": 10,
"MaxFiles": 10
},
"Meta": null,
"Name": "virtiofsd-ro-store",
"Resources": {
"CPU": 100,
"Cores": 0,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 300,
"MemoryMaxMB": 0,
"NUMA": null,
"Networks": null
},
"RestartPolicy": {
"Attempts": 3,
"Delay": 3000000000,
"Interval": 60000000000,
"Mode": "fail",
"RenderTemplates": false
},
"ScalingPolicies": null,
"Services": null,
"ShutdownDelay": 0,
"Templates": [
{
"ChangeMode": "restart",
"ChangeScript": null,
"ChangeSignal": "",
"DestPath": "local/virtiofsd-ro-store.sh",
"EmbeddedTmpl": "#! /run/current-system/sw/bin/bash -e\nmkdir -p /run/microvms/svr/test/my-microvm\nchown microvm:kvm /run/microvms/svr/test/my-microvm\ncd /run/microvms/svr/test/my-microvm\nmkdir -p /nix/store\nexec /run/current-system/sw/bin/virtiofsd \\\n--socket-path=microvm-virtiofs-ro-store.sock \\\n--socket-group=kvm \\\n--shared-dir=/nix/store \\\n--sandbox=none \\\n--thread-pool-size `nproc` \\\n--cache=always\n",
"Envvars": false,
"ErrMissingKey": false,
"Gid": null,
"LeftDelim": "{{",
"Perms": "755",
"RightDelim": "}}",
"SourcePath": "",
"Splay": 5000000000,
"Uid": null,
"VaultGrace": 0,
"Wait": null
}
],
"User": "root",
"Vault": null,
"VolumeMounts": null
},
{
"Actions": null,
"Affinities": null,
"Artifacts": null,
"CSIPluginConfig": null,
"Config": {
"command": "local/make-dirs.sh"
},
"Constraints": null,
"Consul": null,
"DispatchPayload": null,
"Driver": "raw_exec",
"Env": null,
"Identities": null,
"Identity": {
"Audience": [
"nomadproject.io"
],
"ChangeMode": "",
"ChangeSignal": "",
"Env": false,
"File": false,
"Name": "default",
"ServiceName": "",
"TTL": 0
},
"KillSignal": "",
"KillTimeout": 5000000000,
"Kind": "",
"Leader": false,
"Lifecycle": {
"Hook": "prestart",
"Sidecar": false
},
"LogConfig": {
"Disabled": false,
"MaxFileSizeMB": 10,
"MaxFiles": 10
},
"Meta": null,
"Name": "volume-dirs",
"Resources": {
"CPU": 100,
"Cores": 0,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 300,
"MemoryMaxMB": 0,
"NUMA": null,
"Networks": null
},
"RestartPolicy": {
"Attempts": 3,
"Delay": 3000000000,
"Interval": 60000000000,
"Mode": "fail",
"RenderTemplates": false
},
"ScalingPolicies": null,
"Services": null,
"ShutdownDelay": 0,
"Templates": [
{
"ChangeMode": "restart",
"ChangeScript": null,
"ChangeSignal": "",
"DestPath": "local/make-dirs.sh",
"EmbeddedTmpl": "#! /run/current-system/sw/bin/bash -e\n\n",
"Envvars": false,
"ErrMissingKey": false,
"Gid": null,
"LeftDelim": "{{",
"Perms": "755",
"RightDelim": "}}",
"SourcePath": "",
"Splay": 5000000000,
"Uid": null,
"VaultGrace": 0,
"Wait": null
}
],
"User": "",
"Vault": null,
"VolumeMounts": null
},
{
"Actions": null,
"Affinities": null,
"Artifacts": null,
"CSIPluginConfig": null,
"Config": {
"image": "${meta.connect.sidecar_image}",
"args": [
"-c",
"${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
"-l",
"${meta.connect.log_level}",
"--concurrency",
"${meta.connect.proxy_concurrency}",
"--disable-hot-restart"
]
},
"Constraints": [
{
"LTarget": "${attr.consul.version}",
"Operand": "semver",
"RTarget": ">= 1.8.0"
},
{
"LTarget": "${attr.consul.grpc}",
"Operand": ">",
"RTarget": "0"
}
],
"Consul": null,
"DispatchPayload": null,
"Driver": "docker",
"Env": null,
"Identities": null,
"Identity": {
"Audience": [
"nomadproject.io"
],
"ChangeMode": "",
"ChangeSignal": "",
"Env": false,
"File": false,
"Name": "default",
"ServiceName": "",
"TTL": 0
},
"KillSignal": "",
"KillTimeout": 5000000000,
"Kind": "connect-proxy:microvm",
"Leader": false,
"Lifecycle": {
"Hook": "prestart",
"Sidecar": true
},
"LogConfig": {
"Disabled": false,
"MaxFileSizeMB": 2,
"MaxFiles": 2
},
"Meta": null,
"Name": "connect-proxy-microvm",
"Resources": {
"CPU": 250,
"Cores": 0,
"Devices": null,
"DiskMB": 0,
"IOPS": 0,
"MemoryMB": 128,
"MemoryMaxMB": 0,
"NUMA": null,
"Networks": null
},
"RestartPolicy": {
"Attempts": 3,
"Delay": 3000000000,
"Interval": 60000000000,
"Mode": "fail",
"RenderTemplates": false
},
"ScalingPolicies": null,
"Services": null,
"ShutdownDelay": 5000000000,
"Templates": null,
"User": "",
"Vault": null,
"VolumeMounts": null
}
],
"Update": {
"AutoPromote": false,
"AutoRevert": false,
"Canary": 0,
"HealthCheck": "checks",
"HealthyDeadline": 300000000000,
"MaxParallel": 1,
"MinHealthyTime": 10000000000,
"ProgressDeadline": 600000000000,
"Stagger": 30000000000
},
"Volumes": null
}
],
"Type": "service",
"Update": {
"AutoPromote": false,
"AutoRevert": false,
"Canary": 0,
"HealthCheck": "",
"HealthyDeadline": 0,
"MaxParallel": 1,
"MinHealthyTime": 0,
"ProgressDeadline": 0,
"Stagger": 30000000000
},
"VaultNamespace": "",
"VaultToken": "",
"Version": 218
}
Any suggestion is appreciated