TL;DR: starting up a container with device type gpu fails.
lxc config
architecture: x86_64
config:
image.architecture: x86_64
image.description: Ubuntu 16.04 LTS server (20180126)
image.os: ubuntu
image.release: xenial
limits.cpu: 0-22,24-46
limits.kernel.rtprio: "99"
limits.memory: "128849018880"
limits.memory.swap: "false"
nvidia.runtime: "true"
raw.lxc: |
lxc.log.level=debug
lxc.hook.pre-start=bash -c 'ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111;mkdir /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;chmod 777 /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111'
security.nesting: "true"
security.privileged: "false"
user.colosseum.cluster_index: "0"
volatile.base_image: 548facb87b55530fc957fe88b185bf0f1c34fb88f402e95d31c4ed7b359483dd
volatile.can0.hwaddr: 00:16:3e:09:c6:05
volatile.cloud-init.instance-id: 710cbe05-aa24-4f70-989b-a5b73f1fdb93
volatile.idmap.base: "0"
volatile.idmap.current: '[{"Isuid":true,"Isgid":false,"Hostid":100000,"Nsid":0,"Maprange":4000},{"Isuid":false,"Isgid":true,"Hostid":100000,"Nsid":0,"Maprange":4000},{"Isuid":true,"Isgid":false,"Hostid":10011,"Nsid":4000,"Maprange":1},{"Isuid":false,"Isgid":true,"Hostid":5001,"Nsid":4000,"Maprange":1},{"Isuid":true,"Isgid":false,"Hostid":104001,"Nsid":4001,"Maprange":61535},{"Isuid":false,"Isgid":true,"Hostid":104001,"Nsid":4001,"Maprange":61535}]'
volatile.idmap.next: '[{"Isuid": true, "Isgid": false, "Nsid": 0, "Hostid": 100000,
"Maprange": 4000}, {"Isuid": false, "Isgid": true, "Nsid": 0, "Hostid": 100000,
"Maprange": 4000}, {"Isuid": true, "Isgid": false, "Nsid": 4000, "Hostid": 10011,
"Maprange": 1}, {"Isuid": false, "Isgid": true, "Nsid": 4000, "Hostid": 5001,
"Maprange": 1}, {"Isuid": true, "Isgid": false, "Nsid": 4001, "Hostid": 104001,
"Maprange": 61535}, {"Isuid": false, "Isgid": true, "Nsid": 4001, "Hostid": 104001,
"Maprange": 61535}]'
volatile.last_state.idmap: '[]'
volatile.last_state.power: STOPPED
volatile.last_state.ready: "false"
volatile.tr.hwaddr: 00:16:3e:56:43:c7
volatile.usrp.hwaddr: 00:16:3e:05:04:53
volatile.uuid: 756451b1-305e-45cf-a795-8c18150d498d
volatile.uuid.generation: 756451b1-305e-45cf-a795-8c18150d498d
devices:
can0:
name: can0
nictype: macvlan
parent: competitor0
type: nic
gpu0:
gputype: mig
mig.ci: "0"
mig.gi: "1"
pci: CA:00.0
type: gpu
logs_dir:
path: /logs/
source: /data0
type: disk
nas:
path: /share/
source: /share/event-nas-a/neu-test-team-1/
type: disk
snapshot:
path: /socket/
source: /docker/armory-compute-node0/socket
type: disk
tr:
name: tr0
nictype: macvlan
parent: traffic0
type: nic
usrp:
mtu: "9000"
name: usrp
nictype: macvlan
parent: usrp0
type: nic
usrp-usb:
productid: "6010"
type: usb
vendorid: "0403"
ephemeral: false
profiles:
- clean
stateful: false
description: ""
created_at: 2024-01-04T16:48:17.972314216Z
name: neu-test-team-1-colosseumcli-venv-srn111
status: Stopped
status_code: 102
last_used_at: 2024-01-04T19:05:51.476428774Z
location: none
type: container
project: default
why lxc.hook.pre-start?
Notice my lxc config contains a lxc.hook.pre-start with an ugly bash -e in it. This is a workaround to a previous error:
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104183507.953 DEBUG conf - ../src/src/lxc/conf.c:run_buffer:311 - Script exec /snap/lxd/current/lxc/hooks/nvidia produced output: mkdir: cannot create directory ‘/var/snap/lxd/common/lx
d/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook’
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104183507.953 DEBUG conf - ../src/src/lxc/conf.c:run_buffer:311 - Script exec /snap/lxd/current/lxc/hooks/nvidia produced output: : Permission denied
From logs below, you can see this seems to have at least fixed that particular error. Yet still the container does not start.
lxc start command
root@srn-111m:~# lxc start neu-test-team-1-colosseumcli-venv-srn111
Error: Failed to start device "gpu0": MIG device gi=0 ci=0 doesn't exist on GPU 0
Try `lxc info --show-log neu-test-team-1-colosseumcli-venv-srn111` for more info
lxc log
Name: neu-test-team-1-colosseumcli-venv-srn111
Status: STOPPED
Type: container
Architecture: x86_64
Created: 2024/01/04 16:48 UTC
Last Used: 2024/01/04 20:15 UTC
Log:
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.254 INFO lxccontainer - ../src/src/lxc/lxccontainer.c:do_lxcapi_start:998 - Set process title to [lxc monitor] /var/snap/lxd/common/lxd/containers neu-test-team-1-colosseumcli-venv-srn111
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.255 INFO start - ../src/src/lxc/start.c:lxc_check_inherited:325 - Closed inherited fd 4
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.255 INFO start - ../src/src/lxc/start.c:lxc_check_inherited:325 - Closed inherited fd 5
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.255 INFO start - ../src/src/lxc/start.c:lxc_check_inherited:325 - Closed inherited fd 6
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.255 INFO start - ../src/src/lxc/start.c:lxc_check_inherited:325 - Closed inherited fd 20
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.255 DEBUG lxccontainer - ../src/src/lxc/lxccontainer.c:wait_on_daemonized_start:859 - First child 3403113 exited
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.255 INFO lsm - ../src/src/lxc/lsm/lsm.c:lsm_init_static:38 - Initialized LSM security driver AppArmor
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.255 INFO conf - ../src/src/lxc/conf.c:run_script_argv:341 - Executing script "/proc/3145560/exe callhook /var/snap/lxd/common/lxd "default" "neu-test-team-1-colosseumcli-venv-srn111" start" for container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.275 INFO conf - ../src/src/lxc/conf.c:run_script_argv:341 - Executing script "/bin/mount -t shiftfs -o mark,passthrough=3 "/var/snap/lxd/common/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/rootfs" "/var/snap/lxd/common/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/rootfs"" for container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.278 INFO conf - ../src/src/lxc/conf.c:run_script_argv:341 - Executing script "bash -c 'ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111;mkdir /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;chmod 777 /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111'" for container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.281 DEBUG conf - ../src/src/lxc/conf.c:run_buffer:311 - Script exec bash -c 'ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111;mkdir /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;chmod 777 /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111' produced output: total 9
d--x------ 5 100000 root 7 Jan 4 18:43 .
drwx--x--x 1 root root 164 Jan 4 16:49 ..
-r-------- 1 root root 7914 Jan 4 20:15 backup.yaml
drwxrwxrwx 2 root root 3 Jan 4 19:03 hook
-rw-r--r-- 1 root root 1048 Jan 26 2018 metadata.yaml
drwxr-xr-x 28 root root 28 Aug 10 15:26 rootfs
drwxr-xr-x 2 root root 7 Jan 26 2018 templates
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.281 DEBUG conf - ../src/src/lxc/conf.c:run_buffer:311 - Script exec bash -c 'ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111;mkdir /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;chmod 777 /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111' produced output: mkdir: cannot create directory ‘/var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook’
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.281 DEBUG conf - ../src/src/lxc/conf.c:run_buffer:311 - Script exec bash -c 'ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111;mkdir /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;chmod 777 /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111' produced output: : File exists
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 DEBUG conf - ../src/src/lxc/conf.c:run_buffer:311 - Script exec bash -c 'ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111;mkdir /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;chmod 777 /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook;ls -la /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111' produced output: total 9
d--x------ 5 100000 root 7 Jan 4 18:43 .
drwx--x--x 1 root root 164 Jan 4 16:49 ..
-r-------- 1 root root 7914 Jan 4 20:15 backup.yaml
drwxrwxrwx 2 root root 3 Jan 4 19:03 hook
-rw-r--r-- 1 root root 1048 Jan 26 2018 metadata.yaml
drwxr-xr-x 28 root root 28 Aug 10 15:26 rootfs
drwxr-xr-x 2 root root 7 Jan 26 2018 templates
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 DEBUG seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:656 - Host native arch is [3221225534]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:807 - Processing "[all]"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:807 - Processing "reject_force_umount # comment this to allow umount -f; not recommended"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:524 - Set seccomp rule to reject force umounts
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:524 - Set seccomp rule to reject force umounts
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:524 - Set seccomp rule to reject force umounts
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:807 - Processing "[all]"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:807 - Processing "kexec_load errno 38"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding native rule for syscall[246:kexec_load] action[327718:errno] arch[0]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[246:kexec_load] action[327718:errno] arch[1073741827]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[246:kexec_load] action[327718:errno] arch[1073741886]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:807 - Processing "open_by_handle_at errno 38"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding native rule for syscall[304:open_by_handle_at] action[327718:errno] arch[0]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[304:open_by_handle_at] action[327718:errno] arch[1073741827]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[304:open_by_handle_at] action[327718:errno] arch[1073741886]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:807 - Processing "init_module errno 38"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding native rule for syscall[175:init_module] action[327718:errno] arch[0]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[175:init_module] action[327718:errno] arch[1073741827]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.283 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[175:init_module] action[327718:errno] arch[1073741886]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:807 - Processing "finit_module errno 38"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding native rule for syscall[313:finit_module] action[327718:errno] arch[0]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[313:finit_module] action[327718:errno] arch[1073741827]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[313:finit_module] action[327718:errno] arch[1073741886]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:807 - Processing "delete_module errno 38"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding native rule for syscall[176:delete_module] action[327718:errno] arch[0]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[176:delete_module] action[327718:errno] arch[1073741827]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO seccomp - ../src/src/lxc/seccomp.c:do_resolve_add_rule:564 - Adding compat rule for syscall[176:delete_module] action[327718:errno] arch[1073741886]
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO seccomp - ../src/src/lxc/seccomp.c:parse_config_v2:1017 - Merging compat seccomp contexts into main context
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.284 INFO start - ../src/src/lxc/start.c:lxc_init:881 - Container "neu-test-team-1-colosseumcli-venv-srn111" is initialized
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.308 INFO cgfsng - ../src/src/lxc/cgroups/cgfsng.c:cgfsng_monitor_create:1391 - The monitor process uses "lxc.monitor.neu-test-team-1-colosseumcli-venv-srn111" as cgroup
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.309 DEBUG storage - ../src/src/lxc/storage/storage.c:get_storage_by_name:209 - Detected rootfs type "dir"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.309 DEBUG storage - ../src/src/lxc/storage/storage.c:get_storage_by_name:209 - Detected rootfs type "dir"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.310 INFO cgfsng - ../src/src/lxc/cgroups/cgfsng.c:cgfsng_payload_create:1499 - The container process uses "lxc.payload.neu-test-team-1-colosseumcli-venv-srn111" as inner and "lxc.payload.neu-test-team-1-colosseumcli-venv-srn111" as limit cgroup
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 INFO start - ../src/src/lxc/start.c:lxc_spawn:1762 - Cloned CLONE_NEWUSER
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 INFO start - ../src/src/lxc/start.c:lxc_spawn:1762 - Cloned CLONE_NEWNS
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 INFO start - ../src/src/lxc/start.c:lxc_spawn:1762 - Cloned CLONE_NEWPID
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 INFO start - ../src/src/lxc/start.c:lxc_spawn:1762 - Cloned CLONE_NEWUTS
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 INFO start - ../src/src/lxc/start.c:lxc_spawn:1762 - Cloned CLONE_NEWIPC
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 DEBUG start - ../src/src/lxc/start.c:lxc_try_preserve_namespace:139 - Preserved user namespace via fd 42 and stashed path as user:/proc/3403114/fd/42
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 DEBUG start - ../src/src/lxc/start.c:lxc_try_preserve_namespace:139 - Preserved mnt namespace via fd 43 and stashed path as mnt:/proc/3403114/fd/43
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 DEBUG start - ../src/src/lxc/start.c:lxc_try_preserve_namespace:139 - Preserved pid namespace via fd 44 and stashed path as pid:/proc/3403114/fd/44
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 DEBUG start - ../src/src/lxc/start.c:lxc_try_preserve_namespace:139 - Preserved uts namespace via fd 45 and stashed path as uts:/proc/3403114/fd/45
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 DEBUG start - ../src/src/lxc/start.c:lxc_try_preserve_namespace:139 - Preserved ipc namespace via fd 46 and stashed path as ipc:/proc/3403114/fd/46
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 WARN conf - ../src/src/lxc/conf.c:lxc_map_ids:3621 - newuidmap binary is missing
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 WARN conf - ../src/src/lxc/conf.c:lxc_map_ids:3627 - newgidmap binary is missing
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 DEBUG conf - ../src/src/lxc/conf.c:lxc_map_ids:3642 - No newuidmap and newgidmap binary found. Trying to write directly with euid 0
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 DEBUG cgfsng - ../src/src/lxc/cgroups/cgfsng.c:cgfsng_setup_limits_legacy:3168 - Set controller "memory.limit_in_bytes" set to "128849018880"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 DEBUG cgfsng - ../src/src/lxc/cgroups/cgfsng.c:cgfsng_setup_limits_legacy:3168 - Set controller "memory.soft_limit_in_bytes" set to "115964116992"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 DEBUG cgfsng - ../src/src/lxc/cgroups/cgfsng.c:cgfsng_setup_limits_legacy:3168 - Set controller "memory.swappiness" set to "0"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 INFO cgfsng - ../src/src/lxc/cgroups/cgfsng.c:cgfsng_setup_limits_legacy:3172 - Limits for the legacy cgroup hierarchies have been setup
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 WARN conf - ../src/src/lxc/conf.c:lxc_map_ids:3621 - newuidmap binary is missing
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 WARN conf - ../src/src/lxc/conf.c:lxc_map_ids:3627 - newgidmap binary is missing
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 INFO conf - ../src/src/lxc/conf.c:lxc_map_ids:3632 - Caller maps host root. Writing mapping directly
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.311 NOTICE utils - ../src/src/lxc/utils.c:lxc_drop_groups:1367 - Dropped supplimentary groups
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.312 WARN cgfsng - ../src/src/lxc/cgroups/cgfsng.c:fchowmodat:1619 - No such file or directory - Failed to fchownat(40, memory.oom.group, 65536, 0, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW )
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.313 INFO start - ../src/src/lxc/start.c:do_start:1104 - Unshared CLONE_NEWNET
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.313 NOTICE utils - ../src/src/lxc/utils.c:lxc_drop_groups:1367 - Dropped supplimentary groups
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.313 NOTICE utils - ../src/src/lxc/utils.c:lxc_switch_uid_gid:1343 - Switched to gid 0
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.313 NOTICE utils - ../src/src/lxc/utils.c:lxc_switch_uid_gid:1352 - Switched to uid 0
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.313 INFO start - ../src/src/lxc/start.c:do_start:1204 - Unshared CLONE_NEWCGROUP
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.314 DEBUG start - ../src/src/lxc/start.c:lxc_try_preserve_namespace:139 - Preserved net namespace via fd 4 and stashed path as net:/proc/3403114/fd/4
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.315 DEBUG network - ../src/src/lxc/network.c:netdev_configure_server_phys:1223 - Instantiated phys "mac4ef6f2e4" with ifindex "832"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.315 DEBUG network - ../src/src/lxc/network.c:netdev_configure_server_phys:1223 - Instantiated phys "macb7f4fa4d" with ifindex "833"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.316 DEBUG network - ../src/src/lxc/network.c:netdev_configure_server_phys:1223 - Instantiated phys "macf7fa7ce9" with ifindex "834"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.396 DEBUG network - ../src/src/lxc/network.c:lxc_network_move_created_netdev_priv:3552 - Moved network device "mac4ef6f2e4" with ifindex 832 to network namespace of 3403132 and renamed to physLUBFw2
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.464 DEBUG network - ../src/src/lxc/network.c:lxc_network_move_created_netdev_priv:3552 - Moved network device "macb7f4fa4d" with ifindex 833 to network namespace of 3403132 and renamed to physZDN8jR
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.504 DEBUG network - ../src/src/lxc/network.c:lxc_network_move_created_netdev_priv:3552 - Moved network device "macf7fa7ce9" with ifindex 834 to network namespace of 3403132 and renamed to phys4rGkmv
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.505 INFO conf - ../src/src/lxc/conf.c:run_script_argv:341 - Executing script "/bin/mount -t shiftfs -o passthrough=3 "/var/snap/lxd/common/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/rootfs" "/var/snap/lxd/common/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/rootfs"" for container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.507 DEBUG conf - ../src/src/lxc/conf.c:lxc_mount_rootfs:1437 - Mounted rootfs "/var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/rootfs" onto "/var/snap/lxd/common/lxc/" with options "(null)"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.507 INFO conf - ../src/src/lxc/conf.c:setup_utsname:876 - Set hostname to "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.536 DEBUG network - ../src/src/lxc/network.c:setup_hw_addr:3821 - Mac address "00:16:3e:09:c6:05" on "can0" has been setup
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.537 DEBUG network - ../src/src/lxc/network.c:lxc_network_setup_in_child_namespaces_common:3962 - Network device "can0" has been setup
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.560 DEBUG network - ../src/src/lxc/network.c:setup_hw_addr:3821 - Mac address "00:16:3e:56:43:c7" on "tr0" has been setup
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.560 DEBUG network - ../src/src/lxc/network.c:lxc_network_setup_in_child_namespaces_common:3962 - Network device "tr0" has been setup
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG network - ../src/src/lxc/network.c:setup_hw_addr:3821 - Mac address "00:16:3e:05:04:53" on "usrp" has been setup
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG network - ../src/src/lxc/network.c:lxc_network_setup_in_child_namespaces_common:3962 - Network device "usrp" has been setup
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 INFO network - ../src/src/lxc/network.c:lxc_setup_network_in_child_namespaces:4019 - Finished setting up network devices with caller assigned names
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 INFO conf - ../src/src/lxc/conf.c:mount_autodev:1220 - Preparing "/dev"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_autodev:1252 - Using mount options: size=500000,mode=755
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 INFO conf - ../src/src/lxc/conf.c:mount_autodev:1281 - Prepared "/dev"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:lxc_mount_auto_mounts:736 - Invalid argument - Tried to ensure procfs is unmounted
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:lxc_mount_auto_mounts:759 - Invalid argument - Tried to ensure sysfs is unmounted
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/dev/fuse" on "/var/snap/lxd/common/lxc//dev/fuse" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/dev/fuse" were 4106, required extra flags are 10
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/dev/fuse" on "/var/snap/lxd/common/lxc//dev/fuse" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/dev/net/tun" on "/var/snap/lxd/common/lxc//dev/net/tun" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/dev/net/tun" were 4106, required extra flags are 10
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/dev/net/tun" on "/var/snap/lxd/common/lxc//dev/net/tun" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/proc/sys/fs/binfmt_misc" on "/var/snap/lxd/common/lxc//proc/sys/fs/binfmt_misc" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/proc/sys/fs/binfmt_misc" were 4110, required extra flags are 14
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/proc/sys/fs/binfmt_misc" on "/var/snap/lxd/common/lxc//proc/sys/fs/binfmt_misc" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/sys/firmware/efi/efivars" on "/var/snap/lxd/common/lxc//sys/firmware/efi/efivars" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/sys/firmware/efi/efivars" were 4110, required extra flags are 14
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/sys/firmware/efi/efivars" on "/var/snap/lxd/common/lxc//sys/firmware/efi/efivars" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/sys/fs/fuse/connections" on "/var/snap/lxd/common/lxc//sys/fs/fuse/connections" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/sys/fs/fuse/connections" were 4110, required extra flags are 14
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/sys/fs/fuse/connections" on "/var/snap/lxd/common/lxc//sys/fs/fuse/connections" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/sys/fs/pstore" on "/var/snap/lxd/common/lxc//sys/fs/pstore" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/sys/fs/pstore" were 4110, required extra flags are 14
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/sys/fs/pstore" on "/var/snap/lxd/common/lxc//sys/fs/pstore" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/sys/kernel/config" on "/var/snap/lxd/common/lxc//sys/kernel/config" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/sys/kernel/config" were 4110, required extra flags are 14
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/sys/kernel/config" on "/var/snap/lxd/common/lxc//sys/kernel/config" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/sys/kernel/debug" on "/var/snap/lxd/common/lxc//sys/kernel/debug" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/sys/kernel/debug" were 4110, required extra flags are 14
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.588 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/sys/kernel/debug" on "/var/snap/lxd/common/lxc//sys/kernel/debug" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/sys/kernel/security" on "/var/snap/lxd/common/lxc//sys/kernel/security" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/sys/kernel/security" were 4110, required extra flags are 14
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/sys/kernel/security" on "/var/snap/lxd/common/lxc//sys/kernel/security" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/sys/kernel/tracing" on "/var/snap/lxd/common/lxc//sys/kernel/tracing" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/sys/kernel/tracing" were 4110, required extra flags are 14
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/sys/kernel/tracing" on "/var/snap/lxd/common/lxc//sys/kernel/tracing" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/dev/mqueue" on "/var/snap/lxd/common/lxc//dev/mqueue" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/dev/mqueue" were 4110, required extra flags are 14
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/dev/mqueue" on "/var/snap/lxd/common/lxc//dev/mqueue" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "proc" on "/var/snap/lxd/common/lxc//dev/.lxc/proc" with filesystem type "proc"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "sys" on "/var/snap/lxd/common/lxc//dev/.lxc/sys" with filesystem type "sysfs"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/var/snap/lxd/common/lxd/devlxd" on "/var/snap/lxd/common/lxc//dev/lxd" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/var/snap/lxd/common/lxd/devlxd" were 4096, required extra flags are 0
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2473 - Mountflags already were 4096, skipping remount
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/var/snap/lxd/common/lxd/devlxd" on "/var/snap/lxd/common/lxc//dev/lxd" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/disk.logs_dir.logs-" on "/var/snap/lxd/common/lxc//logs/" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/disk.logs_dir.logs-" were 4096, required extra flags are 0
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2473 - Mountflags already were 4096, skipping remount
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/disk.logs_dir.logs-" on "/var/snap/lxd/common/lxc//logs/" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/disk.nas.share-" on "/var/snap/lxd/common/lxc//share/" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/disk.nas.share-" were 3078, required extra flags are 6
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/disk.nas.share-" on "/var/snap/lxd/common/lxc//share/" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/disk.snapshot.socket-" on "/var/snap/lxd/common/lxc//socket/" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/disk.snapshot.socket-" were 4096, required extra flags are 0
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2473 - Mountflags already were 4096, skipping remount
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/disk.snapshot.socket-" on "/var/snap/lxd/common/lxc//socket/" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/unix.usrp--usb.dev-bus-usb-001-002" on "/var/snap/lxd/common/lxc//dev/bus/usb/001/002" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/unix.usrp--usb.dev-bus-usb-001-002" were 4096, required extra flags are 0
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2473 - Mountflags already were 4096, skipping remount
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/unix.usrp--usb.dev-bus-usb-001-002" on "/var/snap/lxd/common/lxc//dev/bus/usb/001/002" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/unix.usrp--usb.dev-bus-usb-001-003" on "/var/snap/lxd/common/lxc//dev/bus/usb/001/003" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/unix.usrp--usb.dev-bus-usb-001-003" were 4096, required extra flags are 0
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2473 - Mountflags already were 4096, skipping remount
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/var/snap/lxd/common/lxd/devices/neu-test-team-1-colosseumcli-venv-srn111/unix.usrp--usb.dev-bus-usb-001-003" on "/var/snap/lxd/common/lxc//dev/bus/usb/001/003" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2445 - Remounting "/var/snap/lxd/common/lxd/shmounts/neu-test-team-1-colosseumcli-venv-srn111" on "/var/snap/lxd/common/lxc//dev/.lxd-mounts" to respect bind or remount options
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2464 - Flags for "/var/snap/lxd/common/lxd/shmounts/neu-test-team-1-colosseumcli-venv-srn111" were 4096, required extra flags are 0
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2473 - Mountflags already were 4096, skipping remount
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 DEBUG conf - ../src/src/lxc/conf.c:mount_entry:2508 - Mounted "/var/snap/lxd/common/lxd/shmounts/neu-test-team-1-colosseumcli-venv-srn111" on "/var/snap/lxd/common/lxc//dev/.lxd-mounts" with filesystem type "none"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.589 INFO conf - ../src/src/lxc/conf.c:run_script_argv:341 - Executing script "/snap/lxd/current/lxcfs/lxc.mount.hook" for container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.620 INFO conf - ../src/src/lxc/conf.c:run_script_argv:341 - Executing script "/snap/lxd/current/lxc/hooks/nvidia" for container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.635 DEBUG conf - ../src/src/lxc/conf.c:run_buffer:311 - Script exec /snap/lxd/current/lxc/hooks/nvidia produced output: INFO: Writing nvidia-container-cli log at /var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook/nvidia.log.
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.636 DEBUG conf - ../src/src/lxc/conf.c:run_buffer:311 - Script exec /snap/lxd/current/lxc/hooks/nvidia produced output: + exec nvidia-container-cli --debug=/var/snap/lxd/common/lxd/storage-pools/lxd/containers/neu-test-team-1-colosseumcli-venv-srn111/hook/nvidia.log --user configure --no-cgroups --ldconfig=@/usr/sbin/ldconfig.real --device=MIG-GPU-932bfb2c-849a-2a22-1e84-9eff6bc02b39/1/0 --compute --utility --require= --require= /var/snap/lxd/common/lxc/
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.670 DEBUG conf - ../src/src/lxc/conf.c:run_buffer:311 - Script exec /snap/lxd/current/lxc/hooks/nvidia produced output: nvidia-container-cli.real: device error: MIG-GPU-932bfb2c-849a-2a22-1e84-9eff6bc02b39/1/0: unknown device
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.674 ERROR conf - ../src/src/lxc/conf.c:run_buffer:322 - Script exited with status 1
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.674 ERROR conf - ../src/src/lxc/conf.c:lxc_setup:4437 - Failed to run mount hooks
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.674 ERROR start - ../src/src/lxc/start.c:do_start:1272 - Failed to setup container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.674 ERROR sync - ../src/src/lxc/sync.c:sync_wait:34 - An error occurred in another process (expected sequence number 4)
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.677 WARN network - ../src/src/lxc/network.c:lxc_delete_network_priv:3631 - Failed to rename interface with index 0 from "can0" to its initial name "mac4ef6f2e4"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.680 WARN network - ../src/src/lxc/network.c:lxc_delete_network_priv:3631 - Failed to rename interface with index 0 from "tr0" to its initial name "macb7f4fa4d"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.683 WARN network - ../src/src/lxc/network.c:lxc_delete_network_priv:3631 - Failed to rename interface with index 0 from "usrp" to its initial name "macf7fa7ce9"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.683 DEBUG network - ../src/src/lxc/network.c:lxc_delete_network:4173 - Deleted network devices
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.683 ERROR start - ../src/src/lxc/start.c:__lxc_start:2107 - Failed to spawn container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.683 WARN start - ../src/src/lxc/start.c:lxc_abort:1036 - No such process - Failed to send SIGKILL via pidfd 41 for process 3403132
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.683 INFO conf - ../src/src/lxc/conf.c:run_script_argv:341 - Executing script "/snap/lxd/current/bin/lxd callhook /var/snap/lxd/common/lxd "default" "neu-test-team-1-colosseumcli-venv-srn111" stopns" for container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201512.683 ERROR lxccontainer - ../src/src/lxc/lxccontainer.c:wait_on_daemonized_start:878 - Received container state "ABORTING" instead of "RUNNING"
lxc 20240104201513.186 ERROR af_unix - ../src/src/lxc/af_unix.c:lxc_abstract_unix_recv_fds_iov:218 - Connection reset by peer - Failed to receive response
lxc 20240104201513.186 ERROR commands - ../src/src/lxc/commands.c:lxc_cmd_rsp_recv_fds:128 - Failed to receive file descriptors for command "get_init_pid"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201513.208 INFO conf - ../src/src/lxc/conf.c:run_script_argv:341 - Executing script "/snap/lxd/current/lxcfs/lxc.reboot.hook" for container "neu-test-team-1-colosseumcli-venv-srn111"
lxc neu-test-team-1-colosseumcli-venv-srn111 20240104201513.711 INFO conf - ../src/src/lxc/conf.c:run_script_argv:341 - Executing script "/snap/lxd/current/bin/lxd callhook /var/snap/lxd/common/lxd "default" "neu-test-team-1-colosseumcli-venv-srn111" stop" for container "neu-test-team-1-colosseumcli-venv-srn111"
system/nvidia information
root@srn-111m:~# uname -a
Linux srn-111m 5.4.0-167-generic #184-Ubuntu SMP Tue Oct 31 09:21:49 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
root@srn-111m:~# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 20.04.6 LTS
Release: 20.04
Codename: focal
root@srn-111m:~# snap list
Name Version Rev Tracking Publisher Notes
core20 20230801 2015 latest/stable canonical✓ base
core22 20230404 617 - canonical✓ base
lxd 5.19-31ff7b6 26093 4.0/stable/… canonical✓ -
snapd 2.60.4 20290 latest/stable canonical✓ snapd
root@srn-111m:~# lxc --version
5.19
root@srn-111m:~# nvidia-smi -h | head -1
NVIDIA System Management Interface -- v530.30.02
root@srn-111m:~# nvidia-smi -L
GPU 0: NVIDIA A100 80GB PCIe (UUID: GPU-932bfb2c-849a-2a22-1e84-9eff6bc02b39)
MIG 3g.40gb Device 0: (UUID: MIG-0929581a-2be6-5dec-bf34-0954c6c529a6)
MIG 3g.40gb Device 1: (UUID: MIG-3adbed3a-d03d-549f-a016-ba1fd7359f23)
nvidia script
Script invoked above, from /snap/lxd/current/lxc/hooks/nvidia
#! /bin/bash
#
# Copyright (c) 2017, 2018 NVIDIA CORPORATION.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
set -eu
# NVIDIA_VISIBLE_DEVICES="" *or* NVIDIA_VISIBLE_DEVICES="void"
# GPU support was explicitly disabled, exit early.
if [ -z "${NVIDIA_VISIBLE_DEVICES-x}" ] || [ "${NVIDIA_VISIBLE_DEVICES:-}" = "void" ]; then
exit 0
fi
# https://github.com/nvidia/nvidia-container-runtime#cuda_version
if [ -n "${CUDA_VERSION:-}" ] && [ -z "${NVIDIA_REQUIRE_CUDA:-}" ]; then
# Legacy CUDA image: default to all devices and all driver capabilities.
if [ -z "${NVIDIA_VISIBLE_DEVICES+x}" ]; then
NVIDIA_VISIBLE_DEVICES="all"
fi
if [ -z "${NVIDIA_DRIVER_CAPABILITIES:-}" ]; then
NVIDIA_DRIVER_CAPABILITIES="all"
fi
if [[ "${CUDA_VERSION}" =~ ^[0-9]+\.[0-9]+ ]]; then
NVIDIA_REQUIRE_CUDA="cuda>=${BASH_REMATCH[0]}"
fi
else
# NVIDIA_VISIBLE_DEVICES unset and it's not a legacy CUDA image.
# This is not a GPU image, exit early.
if [ -z "${NVIDIA_VISIBLE_DEVICES+x}" ]; then
exit 0
fi
fi
export PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin
if ! command -v nvidia-container-cli >/dev/null; then
echo "ERROR: Missing tool nvidia-container-cli, see https://github.com/NVIDIA/libnvidia-container" >&2
exit 1
fi
in_userns() {
[ -e /proc/self/uid_map ] || { echo no; return; }
while read line; do
fields=$(echo $line | awk '{ print $1 " " $2 " " $3 }')
[ "$fields" = "0 0 4294967295" ] && { echo no; return; } || true
echo $fields | grep -q " 0 1$" && { echo userns-root; return; } || true
done < /proc/self/uid_map
if [ -e /proc/1/uid_map ]; then
if [ "$(cat /proc/self/uid_map)" = "$(cat /proc/1/uid_map)" ]; then
echo userns-root
return
fi
fi
echo yes
}
get_ldconfig() {
command -v "ldconfig.real" || command -v "ldconfig"
return $?
}
capability_to_cli() {
case "$1" in
compute) echo "--compute";;
compat32) echo "--compat32";;
display) echo "--display";;
graphics) echo "--graphics";;
utility) echo "--utility";;
video) echo "--video";;
*) exit 1;;
esac
return
}
# Same behavior as strconv.ParseBool in golang
parse_bool() {
case "$1" in
1|t|T|TRUE|true|True) echo "true";;
0|f|F|FALSE|false|False) echo "false";;
*) exit 1;;
esac
return
}
usage() {
cat <<EOF
nvidia-container-cli hook for LXC
Special arguments:
[ -h | --help ]: Print this help message and exit.
Optional arguments:
[ --no-load-kmods ]: Do not try to load the NVIDIA kernel modules.
[ --disable-require ]: Disable all the constraints of the form NVIDIA_REQUIRE_*.
[ --debug <path> ]: The path to the log file.
[ --ldcache <path> ]: The path to the host system's DSO cache.
[ --root <path> ]: The path to the driver root directory.
[ --ldconfig <path> ]: The path to the ldconfig binary, use a '@' prefix for a host path.
EOF
return 0
}
options=$(getopt -o h -l help,no-load-kmods,disable-require,debug:,ldcache:,root:,ldconfig: -- "$@")
if [ $? -ne 0 ]; then
usage
exit 1
fi
eval set -- "$options"
CLI_LOAD_KMODS="true"
CLI_DISABLE_REQUIRE="false"
CLI_DEBUG=
CLI_LDCACHE=
CLI_ROOT=
CLI_LDCONFIG=
while :; do
case "$1" in
--help) usage && exit 1;;
--no-load-kmods) CLI_LOAD_KMODS="false"; shift 1;;
--disable-require) CLI_DISABLE_REQUIRE="true"; shift 1;;
--debug) CLI_DEBUG=$2; shift 2;;
--ldcache) CLI_LDCACHE=$2; shift 2;;
--root) CLI_ROOT=$2; shift 2;;
--ldconfig) CLI_LDCONFIG=$2; shift 2;;
--) shift 1; break;;
*) break;;
esac
done
HOOK_SECTION=
HOOK_TYPE=
case "${LXC_HOOK_VERSION:-0}" in
0) HOOK_SECTION="${2:-}"; HOOK_TYPE="${3:-}";;
1) HOOK_SECTION="${LXC_HOOK_SECTION:-}"; HOOK_TYPE="${LXC_HOOK_TYPE:-}";;
*) echo "ERROR: Unsupported hook version: ${LXC_HOOK_VERSION}." >&2; exit 1;;
esac
if [ "${HOOK_SECTION}" != "lxc" ]; then
echo "ERROR: Not running through LXC." >&2
exit 1
fi
if [ "${HOOK_TYPE}" != "mount" ]; then
echo "ERROR: This hook must be used as a \"mount\" hook." >&2
exit 1
fi
USERNS=$(in_userns)
if [ "${USERNS}" != "yes" ]; then
# This is a limitation of libnvidia-container.
echo "FIXME: This hook currently only works in unprivileged mode." >&2
exit 1
fi
if [ "${USERNS}" = "yes" ]; then
CLI_LOAD_KMODS="false"
if ! grep -q nvidia_uvm /proc/modules; then
echo "WARN: Kernel module nvidia_uvm is not loaded, nvidia-container-cli might fail. Make sure the NVIDIA device driver is installed and loaded." >&2
fi
fi
# https://github.com/nvidia/nvidia-container-runtime#nvidia_disable_require
if [ -n "${NVIDIA_DISABLE_REQUIRE:-}" ]; then
if [ "$(parse_bool "${NVIDIA_DISABLE_REQUIRE}")" = "true" ]; then
CLI_DISABLE_REQUIRE="true"
fi
fi
if [ -z "${CLI_DEBUG}" ]; then
if [ "${LXC_LOG_LEVEL}" = "DEBUG" ] || [ "${LXC_LOG_LEVEL}" = "TRACE" ]; then
rootfs_path="${LXC_ROOTFS_PATH#*:}"
hookdir="${rootfs_path/%rootfs/hook}"
if mkdir -p "${hookdir}"; then
CLI_DEBUG="${hookdir}/nvidia.log"
fi
fi
fi
# A '@' prefix means a host path.
if [ -z "${CLI_LDCONFIG}" ]; then
if host_ldconfig=$(get_ldconfig); then
CLI_LDCONFIG="@${host_ldconfig}"
fi
fi
# https://github.com/nvidia/nvidia-container-runtime#nvidia_visible_devices
CLI_DEVICES="${NVIDIA_VISIBLE_DEVICES}"
# https://github.com/nvidia/nvidia-container-runtime#nvidia_driver_capabilities
CLI_CAPABILITIES=
if [ -n "${NVIDIA_DRIVER_CAPABILITIES:-}" ]; then
CLI_CAPABILITIES="${NVIDIA_DRIVER_CAPABILITIES//,/ }"
fi
if [ "${CLI_CAPABILITIES}" = "all" ]; then
CLI_CAPABILITIES="compute compat32 display graphics utility video"
fi
if [ -z "${CLI_CAPABILITIES}" ]; then
CLI_CAPABILITIES="utility"
fi
global_args=()
configure_args=()
if [ -n "${CLI_DEBUG}" ]; then
echo "INFO: Writing nvidia-container-cli log at ${CLI_DEBUG}." >&2
global_args+=("--debug=${CLI_DEBUG}")
fi
if [ "${CLI_LOAD_KMODS}" = "true" ]; then
global_args+=(--load-kmods)
fi
if [ "${USERNS}" = "yes" ]; then
global_args+=(--user)
configure_args+=(--no-cgroups)
fi
if [ -n "${CLI_LDCACHE}" ]; then
global_args+=(--ldcache="${CLI_LDCACHE}")
fi
if [ -n "${CLI_ROOT}" ]; then
global_args+=(--root="${CLI_ROOT}")
fi
if [ -n "${CLI_LDCONFIG}" ]; then
configure_args+=(--ldconfig="${CLI_LDCONFIG}")
fi
if [ -n "${CLI_DEVICES}" ] && [ "${CLI_DEVICES}" != "none" ]; then
configure_args+=(--device="${CLI_DEVICES}")
fi
for cap in ${CLI_CAPABILITIES}; do
if arg=$(capability_to_cli "${cap}"); then
configure_args+=("${arg}")
else
echo "ERROR: Unknown driver capability \"${cap}\"." >&2
exit 1
fi
done
# https://github.com/nvidia/nvidia-container-runtime#nvidia_require_
if [ "${CLI_DISABLE_REQUIRE}" = "false" ]; then
for req in $(compgen -e "NVIDIA_REQUIRE_"); do
configure_args+=("--require=${!req}")
done
fi
if [ -d "/sys/kernel/security/apparmor" ]; then
# Try to transition to the unconfined AppArmor profile.
echo "changeprofile unconfined" > /proc/self/attr/current || true
fi
set -x
exec nvidia-container-cli ${global_args[@]} configure "${configure_args[@]}" "${LXC_ROOTFS_MOUNT}"
Reading logs, I see failure:
Script exec /snap/lxd/current/lxc/hooks/nvidia produced output: nvidia-container-cli.real: device error: MIG-GPU-932bfb2c-849a-2a22-1e84-9eff6bc02b39/1/0: unknown device
What do you recommend next to debug/fix this?