<14>1 2022-08-23T15:00:57.032416Z 1d781741f9c3 networklog - ingest/muxer.go:1394 [gw@1 indexer="tcp://gravwell:4023" ingester="networkLog" ingesteruuid="f9847335-af85-44cd-b502-96dc71468cb6"] initializing connection
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x63 pc=0x7fb639e95ef4]
runtime stack:
runtime.throw({0x8b4a6c?, 0x0?})
runtime/panic.go:992 +0x71
runtime.sigpanic()
runtime/signal_unix.go:802 +0x3a9
goroutine 25 [syscall]:
runtime.cgocall(0x710bfe, 0xc00004b598)
runtime/cgocall.go:157 +0x5c fp=0xc00004b570 sp=0xc00004b538 pc=0x40529c
net._C2func_getaddrinfo(0xc0001ec190, 0x0, 0xc000161380, 0xc0000aa6f8)
_cgo_gotypes.go:95 +0x56 fp=0xc00004b598 sp=0xc00004b570 pc=0x4ce4b6
net.cgoLookupIPCNAME.func1({0xc0001ec190, 0x0?, 0x407e38?}, 0xc0000a846e?, 0x4d273b?)
net/cgo_unix.go:160 +0x9f fp=0xc00004b5f0 sp=0xc00004b598 pc=0x4cfd5f
net.cgoLookupIPCNAME({0x890889, 0x3}, {0xc0000a846e, 0x8})
net/cgo_unix.go:160 +0x16d fp=0xc00004b738 sp=0xc00004b5f0 pc=0x4cf5cd
net.cgoIPLookup(0xbbcaf0?, {0x890889?, 0xc0001ec0a0?}, {0xc0000a846e?, 0x0?})
net/cgo_unix.go:217 +0x3b fp=0xc00004b7a8 sp=0xc00004b738 pc=0x4cfe1b
net.cgoLookupIP.func1()
net/cgo_unix.go:227 +0x36 fp=0xc00004b7e0 sp=0xc00004b7a8 pc=0x4d0256
runtime.goexit()
runtime/asm_amd64.s:1571 +0x1 fp=0xc00004b7e8 sp=0xc00004b7e0 pc=0x466001
created by net.cgoLookupIP
net/cgo_unix.go:227 +0x12a
goroutine 1 [select]:
github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).WaitForHotContext(0xc0000e38c0, {0x939d80, 0xc0000ae000}, 0x0)
github.com/gravwell/gravwell/v3/ingest/muxer.go:723 +0x245
github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).WaitForHot(...)
github.com/gravwell/gravwell/v3/ingest/muxer.go:697
main.main()
github.com/gravwell/gravwell/v3/ingesters/networkLog/main.go:200 +0x118c
goroutine 19 [chan receive]:
github.com/gravwell/gravwell/v3/chancacher.(*ChanCacher).run(0xc0000a2bd0)
github.com/gravwell/gravwell/v3/chancacher/chancacher.go:207 +0x50
created by github.com/gravwell/gravwell/v3/chancacher.NewChanCacher
github.com/gravwell/gravwell/v3/chancacher/chancacher.go:198 +0xaad
goroutine 20 [chan receive]:
github.com/gravwell/gravwell/v3/chancacher.(*ChanCacher).run(0xc0000a2c60)
github.com/gravwell/gravwell/v3/chancacher/chancacher.go:207 +0x50
created by github.com/gravwell/gravwell/v3/chancacher.NewChanCacher
github.com/gravwell/gravwell/v3/chancacher/chancacher.go:198 +0xaad
goroutine 21 [select]:
net.(*Resolver).lookupIPAddr(0xbbcae0, {0x939db8?, 0xc0000aca80}, {0x890889, 0x3}, {0xc0000a846e, 0x8})
net/lookup.go:325 +0x51b
net.(*Resolver).internetAddrList(0x939db8?, {0x939db8?, 0xc0000aca80?}, {0x890889, 0x3}, {0xc0000a846e?, 0xd?})
net/ipsock.go:288 +0x67a
net.(*Resolver).resolveAddrList(0x1129088?, {0x939db8, 0xc0000aca80}, {0x890dc1, 0x4}, {0x890889?, 0x79df39d6689b2f00?}, {0xc0000a846e, 0xd}, {0x0, ...})
net/dial.go:221 +0x41b
net.(*Dialer).DialContext(0xc00012bac0, {0x939d80, 0xc0000ae000}, {0x890889, 0x3}, {0xc0000a846e, 0xd})
net/dial.go:406 +0x448
net.(*Dialer).Dial(...)
net/dial.go:351
net.DialTimeout({0x890889?, 0xc00012bb68?}, {0xc0000a846e?, 0xc0000afaf6?}, 0x4?)
net/dial.go:334 +0x85
github.com/gravwell/gravwell/v3/ingest.newTcpConn({0xc0000a846e?, 0x13?})
github.com/gravwell/gravwell/v3/ingest/simple.go:262 +0x39
github.com/gravwell/gravwell/v3/ingest.newTCPConnection({0xc0000a846e, 0xd}, {0x0, 0x0}, {0x64, 0xfa, 0xbc, 0x31, 0xa3, 0x21, ...}, ...)
github.com/gravwell/gravwell/v3/ingest/simple.go:252 +0x105
github.com/gravwell/gravwell/v3/ingest.initConnection({{0xc0000a8468, 0x13}, {0x0, 0x0}, {0xc00002a077, 0xd}}, {0xc0001e2570, 0x1, 0x1}, {0x0, ...}, ...)
github.com/gravwell/gravwell/v3/ingest/simple.go:108 +0x10c
github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).getConnection(0xc0000e38c0, {{0xc0000a8468, 0x13}, {0x0, 0x0}, {0xc00002a077, 0xd}})
github.com/gravwell/gravwell/v3/ingest/muxer.go:1396 +0x3c5
github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).connRoutine(0xc0000e38c0, 0x0)
github.com/gravwell/gravwell/v3/ingest/muxer.go:1289 +0x725
created by github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).Start
github.com/gravwell/gravwell/v3/ingest/muxer.go:457 +0x1be
goroutine 22 [sleep]:
time.Sleep(0x12a05f200)
runtime/time.go:194 +0x12e
github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).stateReportRoutine(0xc0000e38c0)
github.com/gravwell/gravwell/v3/ingest/muxer.go:525 +0x4b
created by github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).Start
github.com/gravwell/gravwell/v3/ingest/muxer.go:462 +0x296
goroutine 23 [chan receive]:
github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).getNewConnSet(0xc0000e38c0, 0x1f4?, 0x0?, 0x1)
github.com/gravwell/gravwell/v3/ingest/muxer.go:1022 +0xe9
github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).writeRelayRoutine(0xc0000e38c0, 0xc0000ac9c0, 0xc0001ae150)
github.com/gravwell/gravwell/v3/ingest/muxer.go:1067 +0x178
created by github.com/gravwell/gravwell/v3/ingest.(*IngestMuxer).connRoutine
github.com/gravwell/gravwell/v3/ingest/muxer.go:1253 +0x210
goroutine 24 [select]:
net.cgoLookupIP({0x939d48, 0xc0001e07c0}, {0x890889, 0x3}, {0xc0000a846e, 0x8})
net/cgo_unix.go:228 +0x1aa
net.(*Resolver).lookupIP(0xbbcae0, {0x939d48, 0xc0001e07c0}, {0x890889, 0x3}, {0xc0000a846e, 0x8})
net/lookup_unix.go:96 +0x128
net.glob..func1({0x939d48?, 0xc0001e07c0?}, 0x0?, {0x890889?, 0xc0000ac9c0?}, {0xc0000a846e?, 0x0?})
net/hook.go:23 +0x3d
net.(*Resolver).lookupIPAddr.func1()
net/lookup.go:319 +0x9f
internal/singleflight.(*Group).doCall(0xbbcaf0, 0xc000186a50, {0xc0001ec0a0, 0xc}, 0x0?)
internal/singleflight/singleflight.go:95 +0x3b
created by internal/singleflight.(*Group).DoChan
internal/singleflight/singleflight.go:88 +0x2ec
The pcap ingester would remain running and the lab could be completed as described
See above.
Kris indicated that he did not have this problem, so it may be specific to my environment.