nomad
nomad copied to clipboard
hashicorp
Qemu Ports Not Mapping
Nomad Version 1.0.3
I am using a Nomad 1.0.3/ Consul 1.9.3/ Centos 7 Running within VirtualBox VMs.
Running Nomad & Consul as root
It appears as if the networking/port configuration is not working correctly for the Qemu Driver. I have tried all the port configurations and none are exposing the VM port (8080) I have. I have verified through several methods that the VM is not the issue and port 8080 is exposed on all clients.
HCL Config
job "javaVM" {
datacenters = ["dc1"]
group "vms" {
network {
// I have also tried dynamic here to no avail
port "http" {
static = 8080
}
}
ephemeral_disk {
size = 1024
}
task "petclinicvm" {
# The "driver" parameter specifies the task driver that should be used to
# run the task.
driver = "qemu"
config {
image_path = "local/java-petclinic.qcow2"
port_map {
http = 8080
}
// I have also tried directly setting it but this causes the task to fail
// args = [
// "-net user,hostfwd=tcp::8080-:8080",
// "-net nic"
// ]
}
artifact {
source = "http://10.0.0.200:8000/JavaVM/java-petclinic.qcow2"
destination = "local"
}
resources {
cpu = 1000
memory = 2048
}
}
}
}
Job Status
ID = bd012930-556f-10ad-dbab-fc242223d4fb
Eval ID = 7abdd977
Name = javaVM.vms[0]
Node ID = 62049756
Node Name = client2
Job ID = javaVM
Job Version = 0
Client Status = running
Client Description = Tasks are running
Desired Status = run
Desired Description = <none>
Created = 38s ago
Modified = 20s ago
Deployment ID = b561a57c
Deployment Health = healthy
Allocation Addresses
Label Dynamic Address
*http yes 10.0.0.76:8080
Task "petclinicvm" is "running"
Task Resources
CPU Memory Disk Addresses
4168/1000 MHz 826 MiB/2.0 GiB 1.0 GiB
Task Events:
Started At = 2021-02-16T14:49:59Z
Finished At = N/A
Total Restarts = 0
Last Restart = N/A
Recent Events:
Time Type Description
2021-02-16T09:49:59-05:00 Started Task started by client
2021-02-16T09:49:52-05:00 Downloading Artifacts Client is downloading artifacts
2021-02-16T09:49:52-05:00 Task Setup Building Task Directory
2021-02-16T09:49:52-05:00 Received Task received by client
Ports on host where VM is running
[root@client2 ~]# netstat -tulpn
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 192.168.122.1:53 0.0.0.0:* LISTEN 1784/dnsmasq
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1074/sshd
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 1296/master
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 687/rpcbind
tcp6 0 0 :::8500 :::* LISTEN 1059/consul
tcp6 0 0 :::8502 :::* LISTEN 1059/consul
tcp6 0 0 :::22 :::* LISTEN 1074/sshd
tcp6 0 0 :::8600 :::* LISTEN 1059/consul
tcp6 0 0 ::1:25 :::* LISTEN 1296/master
tcp6 0 0 :::4646 :::* LISTEN 1065/nomad
tcp6 0 0 :::8301 :::* LISTEN 1059/consul
tcp6 0 0 :::111 :::* LISTEN 687/rpcbind
udp 0 0 192.168.122.1:53 0.0.0.0:* 1784/dnsmasq
udp 0 0 0.0.0.0:67 0.0.0.0:* 1784/dnsmasq
udp 0 0 0.0.0.0:68 0.0.0.0:* 869/dhclient
udp 0 0 0.0.0.0:111 0.0.0.0:* 687/rpcbind
udp 0 0 0.0.0.0:53437 0.0.0.0:* 5892/qemu-kvm
udp 0 0 0.0.0.0:39165 0.0.0.0:* 5892/qemu-kvm
udp 0 0 127.0.0.1:323 0.0.0.0:* 707/chronyd
udp 0 0 0.0.0.0:861 0.0.0.0:* 687/rpcbind
udp 0 0 0.0.0.0:48355 0.0.0.0:* 5892/qemu-kvm
udp 0 0 0.0.0.0:54510 0.0.0.0:* 5892/qemu-kvm
udp 0 0 0.0.0.0:54684 0.0.0.0:* 5892/qemu-kvm
udp 0 0 0.0.0.0:48554 0.0.0.0:* 5892/qemu-kvm
udp 0 0 0.0.0.0:60862 0.0.0.0:* 5892/qemu-kvm
udp 0 0 0.0.0.0:48886 0.0.0.0:* 5892/qemu-kvm
udp6 0 0 :::8301 :::* 1059/consul
udp6 0 0 :::111 :::* 687/rpcbind
udp6 0 0 ::1:323 :::* 707/chronyd
udp6 0 0 :::8600 :::* 1059/consul
udp6 0 0 :::861 :::* 687/rpcbind
I found this in the Interweb Ethers so Im not sure if this truly means much but it appears no ports configs are being passed to the Qemu command when run
[root@client2 ~]# ps -ef | awk -e '/qemu/ && !/awk/' | sed -e 's/[^/]*//' -e 's/ -/\n\t-/g'
/usr/libexec/qemu-kvm
-machine type=pc,accel=tcg
-name java-petclinic.qcow2
-m 2048M
-drive file=local/java-petclinic.qcow2
-nographic
The last anecdote I would like to add is that when I setup qemu the command for the binary is 'qemu-kvm' and I just created a link within the path named 'qemu-system-x86_64'. As far as I can tell it seems to be the same thing but renamed.
Am I missing something?
Hi @tschmi5! You're not missing anything... we got network isolation working for QEMU recently https://github.com/hashicorp/nomad/issues/9860 but we still need to get the port mapping fixed. We have a draft PR open at https://github.com/hashicorp/nomad/pull/10035 but we need to get that re-assigned to finish it up.
Hello ! Any news on this issue ? I can't use any qemu VM with nomad if I can't access it :(
Why is qemu config.port_map in the docs if it is not implemented? This is misleading.
