top - 16:10:54 up 1:26, 1 user, load average: 1.68, 1.17, 0.59
Tasks: 109 total, 1 running, 108 sleeping, 0 stopped, 0 zombie
%Cpu(s): 43.3 us, 0.5 sy, 0.0 ni, 56.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.2 st
KiB Mem : 2046940 total, 663244 free, 213220 used, 1170476 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 1621404 avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
3933 root 20 0 35820 18092 8292 S 115.6 0.9 8:23.25 machine-config-
2832 root 20 0 1059904 98464 45736 S 0.7 4.8 1:02.02 hyperkube
2974 root 20 0 828680 54108 19456 S 0.3 2.6 0:41.27 crio
3734 root 20 0 265368 22824 11416 S 0.3 1.1 0:07.11 flanneld
Using libvirt. Some time (it varies) after a worker comes up via the Machine API actuator, the MCD starts hot looping.
# crictl logs 64d4b837eed28
I0919 14:43:16.878108 6667 start.go:42] Version: 0.0.0-182-g83741924
I0919 14:43:16.903225 6667 start.go:78] chrooting into rootMount /rootfs
I0919 14:43:16.903248 6667 start.go:84] moving to / inside the chroot
I0919 14:43:16.903258 6667 daemon.go:80] Starting MachineConfigDaemon
SIGABRT: abort
PC=0x428870 m=0 sigcode=0
goroutine 1 [running]:
runtime.deferreturn(0xc420366870)
/usr/local/go/src/runtime/panic.go:316 fp=0xc4204097a0 sp=0xc420409798 pc=0x428870
github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/watch.(*StreamWatcher).Stop(0xc420366870)
/go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/watch/streamwatcher.go:78 +0x6a fp=0xc4204097c8 sp=0xc4204097a0 pc=0x7c0d1a
github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/watch.Until(0x0, 0x1277e20, 0xc420366870, 0xc4204099e0, 0x1, 0x1, 0x0, 0x126ff40, 0xc4200a1420)
/go/src/github.com/openshift/machine-config-operator/vendor/k8s.io/apimachinery/pkg/watch/until.go:68 +0x304 fp=0xc4204098f8 sp=0xc4204097c8 pc=0x7c14d4
github.com/openshift/machine-config-operator/pkg/daemon.waitUntilUpdate(0x1296440, 0xc42029ab70, 0xc42003e02a, 0xc, 0x1, 0x1)
/go/src/github.com/openshift/machine-config-operator/pkg/daemon/node.go:57 +0x493 fp=0xc420409ab0 sp=0xc4204098f8 pc=0xf34883
github.com/openshift/machine-config-operator/pkg/daemon.(*Daemon).process(0xc4200bc140, 0x1203258, 0xc4204bdb58)
/go/src/github.com/openshift/machine-config-operator/pkg/daemon/daemon.go:102 +0x129 fp=0xc420409b08 sp=0xc420409ab0 pc=0xf32929
github.com/openshift/machine-config-operator/pkg/daemon.(*Daemon).Run(0xc4200bc140, 0xc42009e360, 0x0, 0x0)
/go/src/github.com/openshift/machine-config-operator/pkg/daemon/daemon.go:83 +0xd3 fp=0xc420409b78 sp=0xc420409b08 pc=0xf326e3
main.runStartCmd(0x19f9440, 0x1a21480, 0x0, 0x0)
/go/src/github.com/openshift/machine-config-operator/cmd/machine-config-daemon/start.go:91 +0x553 fp=0xc420409ce0 sp=0xc420409b78 pc=0xf3f193
github.com/openshift/machine-config-operator/vendor/github.com/spf13/cobra.(*Command).execute(0x19f9440, 0x1a21480, 0x0, 0x0, 0x19f9440, 0x1a21480)
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/spf13/cobra/command.go:766 +0x2c1 fp=0xc420409dd0 sp=0xc420409ce0 pc=0x57ec21
github.com/openshift/machine-config-operator/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0x19f91e0, 0x0, 0xc42010bf78, 0x403dfc)
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/spf13/cobra/command.go:852 +0x30a fp=0xc420409f10 sp=0xc420409dd0 pc=0x57f7da
github.com/openshift/machine-config-operator/vendor/github.com/spf13/cobra.(*Command).Execute(0x19f91e0, 0xc42010bf60, 0x0)
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/spf13/cobra/command.go:800 +0x2b fp=0xc420409f40 sp=0xc420409f10 pc=0x57f4ab
main.main()
/go/src/github.com/openshift/machine-config-operator/cmd/machine-config-daemon/main.go:27 +0x31 fp=0xc420409f88 sp=0xc420409f40 pc=0xf3ea21
runtime.main()
/usr/local/go/src/runtime/proc.go:198 +0x212 fp=0xc420409fe0 sp=0xc420409f88 pc=0x42ad72
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc420409fe8 sp=0xc420409fe0 pc=0x455051
goroutine 17 [chan receive]:
github.com/openshift/machine-config-operator/vendor/github.com/golang/glog.(*loggingT).flushDaemon(0x1a032a0)
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/golang/glog/glog.go:879 +0x8b
created by github.com/openshift/machine-config-operator/vendor/github.com/golang/glog.init.0
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/golang/glog/glog.go:410 +0x203
goroutine 18 [IO wait, 99 minutes]:
internal/poll.runtime_pollWait(0x7f32a0b2cea0, 0x72, 0xc42003c6b0)
/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc42038bb18, 0x72, 0xc4203b6600, 0x1000, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc42038bb18, 0xc42003c600, 0x10, 0x10)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).ReadMsg(0xc42038bb00, 0xc42003c6b0, 0x10, 0x10, 0xc4203b6620, 0x1000, 0x1000, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/internal/poll/fd_unix.go:231 +0x1f1
net.(*netFD).readMsg(0xc42038bb00, 0xc42003c6b0, 0x10, 0x10, 0xc4203b6620, 0x1000, 0x1000, 0xfea400, 0xc4200725d0, 0xc42003c5c8, ...)
/usr/local/go/src/net/fd_unix.go:214 +0x90
net.(*UnixConn).readMsg(0xc4200c2760, 0xc42003c6b0, 0x10, 0x10, 0xc4203b6620, 0x1000, 0x1000, 0xc42003c5c8, 0x1291880, 0xc42003c500, ...)
/usr/local/go/src/net/unixsock_posix.go:115 +0x91
net.(*UnixConn).ReadMsgUnix(0xc4200c2760, 0xc42003c6b0, 0x10, 0x10, 0xc4203b6620, 0x1000, 0x1000, 0xc4203bc5b0, 0x7, 0xc42003c5ce, ...)
/usr/local/go/src/net/unixsock.go:137 +0xaa
github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus.(*oobReader).Read(0xc4203b6600, 0xc42003c6b0, 0x10, 0x10, 0x1020, 0xc4203b6600, 0x0)
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus/transport_unix.go:21 +0x8f
io.ReadAtLeast(0x1270080, 0xc4203b6600, 0xc42003c6b0, 0x10, 0x10, 0x10, 0x10a3780, 0xc420040101, 0xc4203b6600)
/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x1270080, 0xc4203b6600, 0xc42003c6b0, 0x10, 0x10, 0xc420040180, 0x21, 0x21)
/usr/local/go/src/io/io.go:327 +0x58
github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus.(*unixTransport).ReadMessage(0xc42039c180, 0xc4200ae540, 0x1, 0x1)
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus/transport_unix.go:85 +0x113
github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus.(*Conn).inWorker(0xc4200f7000)
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus/conn.go:285 +0x4b
created by github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus.(*Conn).Auth
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus/auth.go:118 +0x6c9
goroutine 19 [chan receive, 99 minutes]:
github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus.(*Conn).outWorker(0xc4200f7000)
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus/conn.go:427 +0x63
created by github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus.(*Conn).Auth
/go/src/github.com/openshift/machine-config-operator/vendor/github.com/godbus/dbus/auth.go:119 +0x6ee
goroutine 35 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7f32a0b2cdd0, 0x72, 0xc420407858)
/usr/local/go/src/runtime/netpoll.go:173 +0x57
internal/poll.(*pollDesc).wait(0xc4203ba418, 0x72, 0xffffffffffffff00, 0x12722c0, 0x19a9318)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9b
internal/poll.(*pollDesc).waitRead(0xc4203ba418, 0xc4204a0000, 0x4000, 0x4000)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Read(0xc4203ba400, 0xc4204a0000, 0x4000, 0x4000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:157 +0x17d
net.(*netFD).Read(0xc4203ba400, 0xc4204a0000, 0x4000, 0x4000, 0x0, 0x8, 0x3ffb)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc42000c068, 0xc4204a0000, 0x4000, 0x4000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:176 +0x6a
crypto/tls.(*block).readFromUntil(0xc4203f0000, 0x7f32a0b70068, 0xc42000c068, 0x5, 0xc42000c068, 0x0)
/usr/local/go/src/crypto/tls/conn.go:493 +0x96
crypto/tls.(*Conn).readRecord(0xc4203dc000, 0x1204b17, 0xc4203dc120, 0xaa)
/usr/local/go/src/crypto/tls/conn.go:595 +0xe0
crypto/tls.(*Conn).Read(0xc4203dc000, 0xc420443000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/crypto/tls/conn.go:1156 +0x100
bufio.(*Reader).Read(0xc4200ca300, 0xc42041e2d8, 0x9, 0x9, 0xc420407c50, 0x468333, 0xc420407c60)
/usr/local/go/src/bufio/bufio.go:216 +0x238
io.ReadAtLeast(0x126fd00, 0xc4200ca300, 0xc42041e2d8, 0x9, 0x9, 0x9, 0xc4200a0401, 0xc420407d00, 0x7b51fb)
/usr/local/go/src/io/io.go:309 +0x86
io.ReadFull(0x126fd00, 0xc4200ca300, 0xc42041e2d8, 0x9, 0x9, 0xc4200a0050, 0xc420407d30, 0x7b510d)
/usr/local/go/src/io/io.go:327 +0x58
github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2.readFrameHeader(0xc42041e2d8, 0x9, 0x9, 0x126fd00, 0xc4200ca300, 0x0, 0x0, 0xc4204de000, 0x0)
/go/src/github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2/frame.go:237 +0x7b
github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42041e2a0, 0xc4204de000, 0x0, 0x0, 0x0)
/go/src/github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2/frame.go:492 +0xa4
github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2.(*clientConnReadLoop).run(0xc420407fb8, 0x1203d08, 0xc4200d8fb8)
/go/src/github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2/transport.go:1616 +0x8e
github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2.(*ClientConn).readLoop(0xc4204201c0)
/go/src/github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2/transport.go:1544 +0x68
created by github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2.(*Transport).newClientConn
/go/src/github.com/openshift/machine-config-operator/vendor/golang.org/x/net/http2/transport.go:619 +0x684
rax 0x0
rbx 0x0
rcx 0x1
rdx 0xc420366870
rdi 0xc420042280
rsi 0x3
rbp 0xc4204097b8
rsp 0xc420409798
r8 0xc4200421d0
r9 0x17
r10 0xc420042070
r11 0x1
r12 0x0
r13 0x20
r14 0xdd
r15 0x100
rip 0x428870
rflags 0x202
cs 0x33
fs 0x0
gs 0x0