client: Hang on shutdown on simnet.

[JoeGruffins]

Trying out the changes in #3084 on simnet but I don't think related. Shutting down once after buying tickets and coming back. The second shutdown hangs and looks like at dcr.(*ExchangeWallet).cachedBestBlock

partial panic log

2024-11-18 09:35:26.399 [INF] BW: Attempting to logout...
2024-11-18 09:35:26.399 [INF] BW: Shutting down...
2024-11-18 09:35:26.399 [INF] WEB: Web server off
2024-11-18 09:35:26.399 [INF] RPC: RPC server off
2024-11-18 09:35:26.400 [INF] CORE[DB]: Backing up database...
2024-11-18 09:35:26.401 [DBG] CORE[DB]: Total DB size 36864 bytes, 12328 bytes unused (33.44%)
2024-11-18 09:35:26.401 [INF] CORE[dcr][SPV]: Unloading wallet
2024-11-18 09:35:26.401 [INF] CORE[dcr][SPV]: SPV wallet closed
2024-11-18 09:35:26.402 [TRC] CORE[dcr]: Storing value log head: {Fid:0 Len:43 Offset:11448}

2024-11-18 09:35:26.405 [TRC] CORE[dcr]: Got compaction priority: {level:0 score:1.73 dropPrefixes:[]}
2024-11-18 09:35:26.405 [TRC] CORE[dcr]: Running for level: 0

2024-11-18 09:35:26.405 [TRC] CORE[dcr]: LOG Compact. Added 23 keys. Skipped 2 keys. Iteration took: 132.859µs
2024-11-18 09:35:26.406 [TRC] CORE[dcr]: Discard stats: map[]
2024-11-18 09:35:26.407 [TRC] CORE[dcr]: LOG Compact 0->1, del 2 tables, add 1 tables, took 2.515098ms

2024-11-18 09:35:26.407 [TRC] CORE[dcr]: Compaction for level: 0 DONE
2024-11-18 09:35:26.407 [TRC] CORE[dcr]: Force compaction on level 0 done
SIGQUIT: quit
PC=0x482101 m=0 sigcode=128

goroutine 0 gp=0x3790100 m=0 mp=0x3791f40 [idle]:
runtime.futex(0x3792080, 0x80, 0x0, 0x0, 0x0, 0x0)
	/usr/lib/go/src/runtime/sys_linux_amd64.s:557 +0x21 fp=0x7ffcef7ad720 sp=0x7ffcef7ad718 pc=0x482101
runtime.futexsleep(0x7ffcef7ad798?, 0x3790100?, 0x7ffcef7ad798?)
	/usr/lib/go/src/runtime/os_linux.go:69 +0x30 fp=0x7ffcef7ad770 sp=0x7ffcef7ad720 pc=0x43a7b0
runtime.notesleep(0x3792080)
	/usr/lib/go/src/runtime/lock_futex.go:170 +0x87 fp=0x7ffcef7ad7a8 sp=0x7ffcef7ad770 pc=0x411807
runtime.mPark(...)
	/usr/lib/go/src/runtime/proc.go:1866
runtime.stoplockedm()
	/usr/lib/go/src/runtime/proc.go:3140 +0x73 fp=0x7ffcef7ad800 sp=0x7ffcef7ad7a8 pc=0x446733
runtime.schedule()
	/usr/lib/go/src/runtime/proc.go:3974 +0x3a fp=0x7ffcef7ad838 sp=0x7ffcef7ad800 pc=0x448a7a
runtime.park_m(0xc000625340)
	/usr/lib/go/src/runtime/proc.go:4102 +0x1eb fp=0x7ffcef7ad890 sp=0x7ffcef7ad838 pc=0x448f0b
runtime.mcall()
	/usr/lib/go/src/runtime/asm_amd64.s:459 +0x4e fp=0x7ffcef7ad8a8 sp=0x7ffcef7ad890 pc=0x47e28e

goroutine 1 gp=0xc0000061c0 m=nil [semacquire, 5 minutes]:
runtime.gopark(0xc00009f508?, 0x0?, 0x20?, 0x24?, 0x0?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc000806b80 sp=0xc000806b60 pc=0x477aae
runtime.goparkunlock(...)
	/usr/lib/go/src/runtime/proc.go:430
runtime.semacquire1(0xc000698008, 0x0, 0x1, 0x0, 0x12)
	/usr/lib/go/src/runtime/sema.go:178 +0x225 fp=0xc000806be8 sp=0xc000806b80 pc=0x454605
sync.runtime_Semacquire(0xc000599548?)
	/usr/lib/go/src/runtime/sema.go:71 +0x25 fp=0xc000806c20 sp=0xc000806be8 pc=0x4792c5
sync.(*WaitGroup).Wait(0xc00081a360?)
	/usr/lib/go/src/sync/waitgroup.go:118 +0x48 fp=0xc000806c48 sp=0xc000806c20 pc=0x49ae08
main.runCore(0xc0004dc340)
	/home/joe/git/dcrdex/client/cmd/bisonw/main.go:180 +0xef2 fp=0xc000806ef0 sp=0xc000806c48 pc=0x18c33b2
main.main()
	/home/joe/git/dcrdex/client/cmd/bisonw/main_notray.go:21 +0x79 fp=0xc000806f50 sp=0xc000806ef0 pc=0x18c41f9
runtime.main()
	/usr/lib/go/src/runtime/proc.go:272 +0x28b fp=0xc000806fe0 sp=0xc000806f50 pc=0x44128b
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000806fe8 sp=0xc000806fe0 pc=0x4802c1

goroutine 2 gp=0xc000006c40 m=nil [force gc (idle), 2 minutes]:
runtime.gopark(0x713a2e72077?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc000098fa8 sp=0xc000098f88 pc=0x477aae
runtime.goparkunlock(...)
	/usr/lib/go/src/runtime/proc.go:430
runtime.forcegchelper()
	/usr/lib/go/src/runtime/proc.go:337 +0xb3 fp=0xc000098fe0 sp=0xc000098fa8 pc=0x4415d3
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000098fe8 sp=0xc000098fe0 pc=0x4802c1
created by runtime.init.7 in goroutine 1
	/usr/lib/go/src/runtime/proc.go:325 +0x1a

goroutine 3 gp=0xc000007180 m=nil [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc000099780 sp=0xc000099760 pc=0x477aae
runtime.goparkunlock(...)
	/usr/lib/go/src/runtime/proc.go:430
runtime.bgsweep(0xc0000c6000)
	/usr/lib/go/src/runtime/mgcsweep.go:317 +0xdf fp=0xc0000997c8 sp=0xc000099780 pc=0x428d7f
runtime.gcenable.gowrap1()
	/usr/lib/go/src/runtime/mgc.go:203 +0x25 fp=0xc0000997e0 sp=0xc0000997c8 pc=0x41d465
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc0000997e8 sp=0xc0000997e0 pc=0x4802c1
created by runtime.gcenable in goroutine 1
	/usr/lib/go/src/runtime/mgc.go:203 +0x66

goroutine 4 gp=0xc000007340 m=nil [GC scavenge wait]:
runtime.gopark(0x106ab1?, 0xdec8c?, 0x0?, 0x0?, 0x0?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc000099f78 sp=0xc000099f58 pc=0x477aae
runtime.goparkunlock(...)
	/usr/lib/go/src/runtime/proc.go:430
runtime.(*scavengerState).park(0x378f0e0)
	/usr/lib/go/src/runtime/mgcscavenge.go:425 +0x49 fp=0xc000099fa8 sp=0xc000099f78 pc=0x426769
runtime.bgscavenge(0xc0000c6000)
	/usr/lib/go/src/runtime/mgcscavenge.go:658 +0x59 fp=0xc000099fc8 sp=0xc000099fa8 pc=0x426cf9
runtime.gcenable.gowrap2()
	/usr/lib/go/src/runtime/mgc.go:204 +0x25 fp=0xc000099fe0 sp=0xc000099fc8 pc=0x41d405
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000099fe8 sp=0xc000099fe0 pc=0x4802c1
created by runtime.gcenable in goroutine 1
	/usr/lib/go/src/runtime/mgc.go:204 +0xa5

.....

goroutine 84 gp=0xc000684700 m=nil [chan receive]:
runtime.gopark(0x1b595a0?, 0xc000bcd1d0?, 0x20?, 0x81?, 0xc0003d8120?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc000807cc0 sp=0xc000807ca0 pc=0x477aae
runtime.chanrecv(0xc00082b1f0, 0x0, 0x1)
	/usr/lib/go/src/runtime/chan.go:639 +0x41c fp=0xc000807d38 sp=0xc000807cc0 pc=0x40c11c
runtime.chanrecv1(0xc0006ee0a8?, 0xc000807d58?)
	/usr/lib/go/src/runtime/chan.go:489 +0x12 fp=0xc000807d60 sp=0xc000807d38 pc=0x40bcd2
decred.org/dcrdex/dex.(*ConnectionMaster).Wait(...)
	/home/joe/git/dcrdex/dex/runner.go:190
decred.org/dcrdex/dex.(*ConnectionMaster).Disconnect(0xc00069ea50)
	/home/joe/git/dcrdex/dex/runner.go:200 +0xa5 fp=0xc000807d98 sp=0xc000807d60 pc=0x7b50c5
decred.org/dcrdex/client/core.(*xcWallet).Disconnect(0xc0006ee000)
	/home/joe/git/dcrdex/client/core/wallet.go:459 +0x30 fp=0xc000807db8 sp=0xc000807d98 pc=0xa8c530
decred.org/dcrdex/client/core.(*Core).Run(0xc0002e3400, {0x29204e0, 0xc00055af50})
	/home/joe/git/dcrdex/client/core/core.go:1834 +0xb65 fp=0xc000807f98 sp=0xc000807db8 pc=0xa11105
main.runCore.func3()
	/home/joe/git/dcrdex/client/cmd/bisonw/main.go:113 +0x55 fp=0xc000807fe0 sp=0xc000807f98 pc=0x18c3b55
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000807fe8 sp=0xc000807fe0 pc=0x4802c1
created by main.runCore in goroutine 1
	/home/joe/git/dcrdex/client/cmd/bisonw/main.go:111 +0xb53

goroutine 181 gp=0xc0007a8380 m=nil [sync.RWMutex.RLock, 4 minutes]:
runtime.gopark(0xc0008c6e70?, 0x29204e0?, 0x50?, 0x80?, 0x30?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc000f85120 sp=0xc000f85100 pc=0x477aae
runtime.goparkunlock(...)
	/usr/lib/go/src/runtime/proc.go:430
runtime.semacquire1(0xc0006fe0e0, 0x0, 0x3, 0x0, 0x16)
	/usr/lib/go/src/runtime/sema.go:178 +0x225 fp=0xc000f85188 sp=0xc000f85120 pc=0x454605
sync.runtime_SemacquireRWMutexR(0xc000b11340?, 0x8?, 0x170d54a?)
	/usr/lib/go/src/runtime/sema.go:100 +0x25 fp=0xc000f851c0 sp=0xc000f85188 pc=0x4793e5
sync.(*RWMutex).RLock(...)
	/usr/lib/go/src/sync/rwmutex.go:72
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).cachedBestBlock(0xc0006fe008)
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:6853 +0x4f fp=0xc000f85218 sp=0xc000f851c0 pc=0x16f478f
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).tickets(0xc0006fe008, {0x29204e0?, 0xc000828050?})
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:5517 +0x53 fp=0xc000f85278 sp=0xc000f85218 pc=0x16ec013
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).StakeStatus(0xc0006fe008)
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:5415 +0x45 fp=0xc000f854d8 sp=0xc000f85278 pc=0x16eb525
decred.org/dcrdex/client/asset/dcr.(*NativeWallet).StakeStatus(0xc0005e5840?)
	<autogenerated>:1 +0x1f fp=0xc000f854f0 sp=0xc000f854d8 pc=0x171ad1f
decred.org/dcrdex/client/core.(*Core).StakeStatus(0x73b207f010a0?, 0x5e58c0?)
	/home/joe/git/dcrdex/client/core/core.go:10542 +0x2f fp=0xc000f85510 sp=0xc000f854f0 pc=0xa5a08f
decred.org/dcrdex/client/webserver.(*WebServer).apiStakeStatus(0xc0006fa2d0, {0x73b207f010a0, 0xc0005e58c0}, 0xc000e5d400)
	/home/joe/git/dcrdex/client/webserver/api.go:1738 +0x8c fp=0xc000f85588 sp=0xc000f85510 pc=0xbbed0c
decred.org/dcrdex/client/webserver.(*WebServer).apiStakeStatus-fm({0x73b207f010a0?, 0xc0005e58c0?}, 0xc00ad7f5b8?)
	<autogenerated>:1 +0x36 fp=0xc000f855b8 sp=0xc000f85588 pc=0xbd5436
net/http.HandlerFunc.ServeHTTP(0xc0006fa2d0?, {0x73b207f010a0?, 0xc0005e58c0?}, 0x8e087d?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc000f855e0 sp=0xc000f855b8 pc=0x75c2e9
decred.org/dcrdex/client/webserver.(*WebServer).rejectUnauthed-fm.(*WebServer).rejectUnauthed.func1({0x73b207f010a0, 0xc0005e58c0}, 0xc000e5d400)
	/home/joe/git/dcrdex/client/webserver/middleware.go:132 +0x68 fp=0xc000f85628 sp=0xc000f855e0 pc=0xbd3a28
net/http.HandlerFunc.ServeHTTP(0xc0005fe500?, {0x73b207f010a0?, 0xc0005e58c0?}, 0xc000bec410?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc000f85650 sp=0xc000f85628 pc=0x75c2e9
github.com/go-chi/chi/v5.(*ChainHandler).ServeHTTP(0x1b68f20?, {0x73b207f010a0?, 0xc0005e58c0?}, 0xc000b11300?)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/chain.go:31 +0x26 fp=0xc000f85680 sp=0xc000f85650 pc=0x8dc4a6
github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc000112300, {0x73b207f010a0, 0xc0005e58c0}, 0xc000e5d400)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:437 +0x1f2 fp=0xc000f856d0 sp=0xc000f85680 pc=0x8df032
github.com/go-chi/chi/v5.(*Mux).routeHTTP-fm({0x73b207f010a0?, 0xc0005e58c0?}, 0xc00079a360?)
	<autogenerated>:1 +0x36 fp=0xc000f85700 sp=0xc000f856d0 pc=0x8e2f96
net/http.HandlerFunc.ServeHTTP(0x1b5c480?, {0x73b207f010a0?, 0xc0005e58c0?}, 0x10?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc000f85728 sp=0xc000f85700 pc=0x75c2e9
github.com/go-chi/chi/v5/middleware.AllowContentType.func1.1({0x73b207f010a0, 0xc0005e58c0}, 0xc000e5d400)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/middleware/content_type.go:41 +0xdd fp=0xc000f85780 sp=0xc000f85728 pc=0x91581d
net/http.HandlerFunc.ServeHTTP(0xc00ad7f7f8?, {0x73b207f010a0?, 0xc0005e58c0?}, 0xc00ad7f7d8?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc000f857a8 sp=0xc000f85780 pc=0x75c2e9
github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc000112300, {0x73b207f010a0, 0xc0005e58c0}, 0xc000e5d400)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:71 +0x32f fp=0xc000f85808 sp=0xc000f857a8 pc=0x8dcf8f
github.com/go-chi/chi/v5.(*Mux).Mount.func1({0x73b207f010a0, 0xc0005e58c0}, 0xc000e5d400)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:312 +0x1bb fp=0xc000f85858 sp=0xc000f85808 pc=0x8de85b
net/http.HandlerFunc.ServeHTTP(0x1b68f20?, {0x73b207f010a0?, 0xc0005e58c0?}, 0xc000b56d85?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc000f85880 sp=0xc000f85858 pc=0x75c2e9
github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc000040900, {0x73b207f010a0, 0xc0005e58c0}, 0xc000e5d400)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:437 +0x1f2 fp=0xc000f858d0 sp=0xc000f85880 pc=0x8df032
github.com/go-chi/chi/v5.(*Mux).routeHTTP-fm({0x73b207f010a0?, 0xc0005e58c0?}, 0x980000c000becee0?)
	<autogenerated>:1 +0x36 fp=0xc000f85900 sp=0xc000f858d0 pc=0x8e2f96
net/http.HandlerFunc.ServeHTTP(0xc000becee0?, {0x73b207f010a0?, 0xc0005e58c0?}, 0x10?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc000f85928 sp=0xc000f85900 pc=0x75c2e9
github.com/go-chi/chi/v5/middleware.Recoverer.func1({0x73b207f010a0?, 0xc0005e58c0?}, 0x1e6b9d7?)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/middleware/recoverer.go:37 +0x6c fp=0xc000f85988 sp=0xc000f85928 pc=0x916e0c
net/http.HandlerFunc.ServeHTTP(0xc0008c6d80?, {0x73b207f010a0?, 0xc0005e58c0?}, 0x1f33ff5?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc000f859b0 sp=0xc000f85988 pc=0x75c2e9
decred.org/dcrdex/client/webserver.(*WebServer).securityMiddleware-fm.(*WebServer).securityMiddleware.func1({0x73b207f010a0, 0xc0005e58c0}, 0xc000e5d400)
	/home/joe/git/dcrdex/client/webserver/middleware.go:34 +0x174 fp=0xc000f85a00 sp=0xc000f859b0 pc=0xbd1f94
net/http.HandlerFunc.ServeHTTP(0x36ac3e0?, {0x73b207f010a0?, 0xc0005e58c0?}, 0x73b20c153580?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc000f85a28 sp=0xc000f85a00 pc=0x75c2e9
decred.org/dcrdex/client/webserver.New.RequestLogger.func3.1({0x291cbb0, 0xc000b47500}, 0xc000e5d2c0)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/middleware/logger.go:57 +0x154 fp=0xc000f85ae0 sp=0xc000f85a28 pc=0xbcc554
net/http.HandlerFunc.ServeHTTP(0x29204e0?, {0x291cbb0?, 0xc000b47500?}, 0x36a4110?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc000f85b08 sp=0xc000f85ae0 pc=0x75c2e9
github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc000040900, {0x291cbb0, 0xc000b47500}, 0xc000e5d180)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:88 +0x2ee fp=0xc000f85b68 sp=0xc000f85b08 pc=0x8dcf4e
net/http.serverHandler.ServeHTTP({0x2918de0?}, {0x291cbb0?, 0xc000b47500?}, 0x6?)
	/usr/lib/go/src/net/http/server.go:3210 +0x8e fp=0xc000f85b98 sp=0xc000f85b68 pc=0x77b7ee
net/http.(*conn).serve(0xc0006c47e0, {0x29204a8, 0xc0007f80c0})
	/usr/lib/go/src/net/http/server.go:2092 +0x5d0 fp=0xc000f85fb8 sp=0xc000f85b98 pc=0x75ac90
net/http.(*Server).Serve.gowrap3()
	/usr/lib/go/src/net/http/server.go:3360 +0x28 fp=0xc000f85fe0 sp=0xc000f85fb8 pc=0x760108
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000f85fe8 sp=0xc000f85fe0 pc=0x4802c1
created by net/http.(*Server).Serve in goroutine 107
	/usr/lib/go/src/net/http/server.go:3360 +0x485

goroutine 105 gp=0xc000a81880 m=nil [chan receive, 5 minutes]:
runtime.gopark(0x44ad80?, 0xc000796580?, 0x80?, 0x18?, 0x7b4be8?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc0000abe98 sp=0xc0000abe78 pc=0x477aae
runtime.chanrecv(0xc000880e70, 0x0, 0x1)
	/usr/lib/go/src/runtime/chan.go:639 +0x41c fp=0xc0000abf10 sp=0xc0000abe98 pc=0x40c11c
runtime.chanrecv1(0xc000c53200?, 0x29204e0?)
	/usr/lib/go/src/runtime/chan.go:489 +0x12 fp=0xc0000abf38 sp=0xc0000abf10 pc=0x40bcd2
decred.org/dcrdex/dex.(*ConnectionMaster).Wait(...)
	/home/joe/git/dcrdex/dex/runner.go:190
main.runCore.func6()
	/home/joe/git/dcrdex/client/cmd/bisonw/main.go:173 +0x1fe fp=0xc0000abfe0 sp=0xc0000abf38 pc=0x18c36be
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc0000abfe8 sp=0xc0000abfe0 pc=0x4802c1
created by main.runCore in goroutine 1
	/home/joe/git/dcrdex/client/cmd/bisonw/main.go:163 +0xee5

goroutine 106 gp=0xc000a81a40 m=nil [select]:
runtime.gopark(0xc0006b9720?, 0x2?, 0xff?, 0xff?, 0xc0006b96cc?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc0006b9568 sp=0xc0006b9548 pc=0x477aae
runtime.selectgo(0xc0006b9720, 0xc0006b96c8, 0x477aae?, 0x0, 0xc000067908?, 0x1)
	/usr/lib/go/src/runtime/select.go:335 +0x7a5 fp=0xc0006b9690 sp=0xc0006b9568 pc=0x453565
net/http.(*Server).Shutdown(0xc0006fa1e0, {0x2920400, 0x37b8560})
	/usr/lib/go/src/net/http/server.go:3078 +0x24e fp=0xc0006b9768 sp=0xc0006b9690 pc=0x75f2ce
decred.org/dcrdex/client/webserver.(*WebServer).Connect.func1()
	/home/joe/git/dcrdex/client/webserver/webserver.go:692 +0x85 fp=0xc0006b97e0 sp=0xc0006b9768 pc=0xbcde85
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc0006b97e8 sp=0xc0006b97e0 pc=0x4802c1
created by decred.org/dcrdex/client/webserver.(*WebServer).Connect in goroutine 105
	/home/joe/git/dcrdex/client/webserver/webserver.go:689 +0x3e7

goroutine 109 gp=0xc000685dc0 m=nil [semacquire, 5 minutes]:
runtime.gopark(0x0?, 0x789800?, 0xe0?, 0x24?, 0x7601a0?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc0004feef0 sp=0xc0004feed0 pc=0x477aae
runtime.goparkunlock(...)
	/usr/lib/go/src/runtime/proc.go:430
runtime.semacquire1(0xc0005b60b8, 0x0, 0x1, 0x0, 0x12)
	/usr/lib/go/src/runtime/sema.go:178 +0x225 fp=0xc0004fef58 sp=0xc0004feef0 pc=0x454605
sync.runtime_Semacquire(0xc0007a60b0?)
	/usr/lib/go/src/runtime/sema.go:71 +0x25 fp=0xc0004fef90 sp=0xc0004fef58 pc=0x4792c5
sync.(*WaitGroup).Wait(0xb0a920?)
	/usr/lib/go/src/sync/waitgroup.go:118 +0x48 fp=0xc0004fefb8 sp=0xc0004fef90 pc=0x49ae08
decred.org/dcrdex/dex.(*ConnectionMaster).Connect.func1()
	/home/joe/git/dcrdex/dex/runner.go:140 +0x29 fp=0xc0004fefe0 sp=0xc0004fefb8 pc=0x7b4ce9
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc0004fefe8 sp=0xc0004fefe0 pc=0x4802c1
created by decred.org/dcrdex/dex.(*ConnectionMaster).Connect in goroutine 105
	/home/joe/git/dcrdex/dex/runner.go:139 +0x148

goroutine 486 gp=0xc000b01340 m=nil [select]:
runtime.gopark(0xc000c70f88?, 0x2?, 0x80?, 0xa0?, 0xc000c70f34?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc000c70dd8 sp=0xc000c70db8 pc=0x477aae
runtime.selectgo(0xc000c70f88, 0xc000c70f30, 0x2920e68?, 0x0, 0x12141c160000000b?, 0x1)
	/usr/lib/go/src/runtime/select.go:335 +0x7a5 fp=0xc000c70f00 sp=0xc000c70dd8 pc=0x453565
decred.org/dcrwallet/v4/p2p.(*RemotePeer).writeMessages.func1()
	/home/joe/go/pkg/mod/decred.org/dcrwallet/[email protected]/p2p/peering.go:473 +0x11f fp=0xc000c70fe0 sp=0xc000c70f00 pc=0x169aa1f
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000c70fe8 sp=0xc000c70fe0 pc=0x4802c1
created by decred.org/dcrwallet/v4/p2p.(*RemotePeer).writeMessages in goroutine 483
	/home/joe/go/pkg/mod/decred.org/dcrwallet/[email protected]/p2p/peering.go:464 +0x94

goroutine 515 gp=0xc000d1d340 m=nil [sync.RWMutex.RLock, 4 minutes]:
runtime.gopark(0x714f468702923258?, 0xb4315ce73e2a15dd?, 0x80?, 0x44?, 0x29413495b16da98f?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc000b41aa8 sp=0xc000b41a88 pc=0x477aae
runtime.goparkunlock(...)
	/usr/lib/go/src/runtime/proc.go:430
runtime.semacquire1(0xc0006fe0e0, 0x0, 0x3, 0x0, 0x16)
	/usr/lib/go/src/runtime/sema.go:178 +0x225 fp=0xc000b41b10 sp=0xc000b41aa8 pc=0x454605
sync.runtime_SemacquireRWMutexR(0xc000828050?, 0x5?, 0xc0008c66f0?)
	/usr/lib/go/src/runtime/sema.go:100 +0x25 fp=0xc000b41b48 sp=0xc000b41b10 pc=0x4793e5
sync.(*RWMutex).RLock(...)
	/usr/lib/go/src/sync/rwmutex.go:72
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).SyncStatus.func1()
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:4836 +0x68 fp=0xc000b41b88 sp=0xc000b41b48 pc=0x16e7428
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).SyncStatus(0xc0006fe008)
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:4870 +0x233 fp=0xc000b41c18 sp=0xc000b41b88 pc=0x16e7373
decred.org/dcrdex/client/asset/dcr.(*NativeWallet).mixFunds(0xc0006d6420)
	/home/joe/git/dcrdex/client/asset/dcr/native_wallet.go:206 +0x25 fp=0xc000b41c48 sp=0xc000b41c18 pc=0x16fcec5
decred.org/dcrdex/client/asset/dcr.initNativeWallet.func1()
	/home/joe/git/dcrdex/client/asset/dcr/native_wallet.go:107 +0x75 fp=0xc000b41c98 sp=0xc000b41c48 pc=0x16fc435
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).handleTipChange(0xc0006fe008, {0x29204e0, 0xc000828050}, 0xc000b56a40, 0x281)
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:6726 +0x293 fp=0xc000b41e70 sp=0xc000b41c98 pc=0x16f39d3
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).monitorBlocks(0xc0006fe008, {0x29204e0, 0xc000828050})
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:6697 +0x30a fp=0xc000b41f78 sp=0xc000b41e70 pc=0x16f300a
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).Connect.func2()
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:1088 +0x7d fp=0xc000b41fe0 sp=0xc000b41f78 pc=0x16c7e1d
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000b41fe8 sp=0xc000b41fe0 pc=0x4802c1
created by decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).Connect in goroutine 411
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:1085 +0x465

goroutine 518 gp=0xc00060b880 m=nil [semacquire, 4 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x20?, 0x0?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc000c5def0 sp=0xc000c5ded0 pc=0x477aae
runtime.goparkunlock(...)
	/usr/lib/go/src/runtime/proc.go:430
runtime.semacquire1(0xc0006fe038, 0x0, 0x1, 0x0, 0x12)
	/usr/lib/go/src/runtime/sema.go:178 +0x225 fp=0xc000c5df58 sp=0xc000c5def0 pc=0x454605
sync.runtime_Semacquire(0x0?)
	/usr/lib/go/src/runtime/sema.go:71 +0x25 fp=0xc000c5df90 sp=0xc000c5df58 pc=0x4792c5
sync.(*WaitGroup).Wait(0x0?)
	/usr/lib/go/src/sync/waitgroup.go:118 +0x48 fp=0xc000c5dfb8 sp=0xc000c5df90 pc=0x49ae08
decred.org/dcrdex/dex.(*ConnectionMaster).Connect.func1()
	/home/joe/git/dcrdex/dex/runner.go:140 +0x29 fp=0xc000c5dfe0 sp=0xc000c5dfb8 pc=0x7b4ce9
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000c5dfe8 sp=0xc000c5dfe0 pc=0x4802c1
created by decred.org/dcrdex/dex.(*ConnectionMaster).Connect in goroutine 411
	/home/joe/git/dcrdex/dex/runner.go:139 +0x148

goroutine 497 gp=0xc000c2e540 m=nil [sync.RWMutex.RLock, 4 minutes]:
runtime.gopark(0xc00ae94210?, 0x29204e0?, 0x50?, 0x80?, 0x30?)
	/usr/lib/go/src/runtime/proc.go:424 +0xce fp=0xc0012a1120 sp=0xc0012a1100 pc=0x477aae
runtime.goparkunlock(...)
	/usr/lib/go/src/runtime/proc.go:430
runtime.semacquire1(0xc0006fe0e0, 0x0, 0x3, 0x0, 0x16)
	/usr/lib/go/src/runtime/sema.go:178 +0x225 fp=0xc0012a1188 sp=0xc0012a1120 pc=0x454605
sync.runtime_SemacquireRWMutexR(0xc000698170?, 0x8?, 0x170d54a?)
	/usr/lib/go/src/runtime/sema.go:100 +0x25 fp=0xc0012a11c0 sp=0xc0012a1188 pc=0x4793e5
sync.(*RWMutex).RLock(...)
	/usr/lib/go/src/sync/rwmutex.go:72
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).cachedBestBlock(0xc0006fe008)
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:6853 +0x4f fp=0xc0012a1218 sp=0xc0012a11c0 pc=0x16f478f
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).tickets(0xc0006fe008, {0x29204e0?, 0xc000828050?})
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:5517 +0x53 fp=0xc0012a1278 sp=0xc0012a1218 pc=0x16ec013
decred.org/dcrdex/client/asset/dcr.(*ExchangeWallet).StakeStatus(0xc0006fe008)
	/home/joe/git/dcrdex/client/asset/dcr/dcr.go:5415 +0x45 fp=0xc0012a14d8 sp=0xc0012a1278 pc=0x16eb525
decred.org/dcrdex/client/asset/dcr.(*NativeWallet).StakeStatus(0xc001286040?)
	<autogenerated>:1 +0x1f fp=0xc0012a14f0 sp=0xc0012a14d8 pc=0x171ad1f
decred.org/dcrdex/client/core.(*Core).StakeStatus(0x73b207f010a0?, 0x1286100?)
	/home/joe/git/dcrdex/client/core/core.go:10542 +0x2f fp=0xc0012a1510 sp=0xc0012a14f0 pc=0xa5a08f
decred.org/dcrdex/client/webserver.(*WebServer).apiStakeStatus(0xc0006fa2d0, {0x73b207f010a0, 0xc001286100}, 0xc006d14280)
	/home/joe/git/dcrdex/client/webserver/api.go:1738 +0x8c fp=0xc0012a1588 sp=0xc0012a1510 pc=0xbbed0c
decred.org/dcrdex/client/webserver.(*WebServer).apiStakeStatus-fm({0x73b207f010a0?, 0xc001286100?}, 0xc0012a15b8?)
	<autogenerated>:1 +0x36 fp=0xc0012a15b8 sp=0xc0012a1588 pc=0xbd5436
net/http.HandlerFunc.ServeHTTP(0xc0006fa2d0?, {0x73b207f010a0?, 0xc001286100?}, 0x8e087d?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc0012a15e0 sp=0xc0012a15b8 pc=0x75c2e9
decred.org/dcrdex/client/webserver.(*WebServer).rejectUnauthed-fm.(*WebServer).rejectUnauthed.func1({0x73b207f010a0, 0xc001286100}, 0xc006d14280)
	/home/joe/git/dcrdex/client/webserver/middleware.go:132 +0x68 fp=0xc0012a1628 sp=0xc0012a15e0 pc=0xbd3a28
net/http.HandlerFunc.ServeHTTP(0xc0005fe500?, {0x73b207f010a0?, 0xc001286100?}, 0xc000c14480?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc0012a1650 sp=0xc0012a1628 pc=0x75c2e9
github.com/go-chi/chi/v5.(*ChainHandler).ServeHTTP(0x1b68f20?, {0x73b207f010a0?, 0xc001286100?}, 0xc000698100?)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/chain.go:31 +0x26 fp=0xc0012a1680 sp=0xc0012a1650 pc=0x8dc4a6
github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc000112300, {0x73b207f010a0, 0xc001286100}, 0xc006d14280)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:437 +0x1f2 fp=0xc0012a16d0 sp=0xc0012a1680 pc=0x8df032
github.com/go-chi/chi/v5.(*Mux).routeHTTP-fm({0x73b207f010a0?, 0xc001286100?}, 0xc00079a360?)
	<autogenerated>:1 +0x36 fp=0xc0012a1700 sp=0xc0012a16d0 pc=0x8e2f96
net/http.HandlerFunc.ServeHTTP(0x1b5c480?, {0x73b207f010a0?, 0xc001286100?}, 0x10?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc0012a1728 sp=0xc0012a1700 pc=0x75c2e9
github.com/go-chi/chi/v5/middleware.AllowContentType.func1.1({0x73b207f010a0, 0xc001286100}, 0xc006d14280)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/middleware/content_type.go:41 +0xdd fp=0xc0012a1780 sp=0xc0012a1728 pc=0x91581d
net/http.HandlerFunc.ServeHTTP(0xc0012a17f8?, {0x73b207f010a0?, 0xc001286100?}, 0xc0012a17d8?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc0012a17a8 sp=0xc0012a1780 pc=0x75c2e9
github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc000112300, {0x73b207f010a0, 0xc001286100}, 0xc006d14280)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:71 +0x32f fp=0xc0012a1808 sp=0xc0012a17a8 pc=0x8dcf8f
github.com/go-chi/chi/v5.(*Mux).Mount.func1({0x73b207f010a0, 0xc001286100}, 0xc006d14280)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:312 +0x1bb fp=0xc0012a1858 sp=0xc0012a1808 pc=0x8de85b
net/http.HandlerFunc.ServeHTTP(0x1b68f20?, {0x73b207f010a0?, 0xc001286100?}, 0xc000c10005?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc0012a1880 sp=0xc0012a1858 pc=0x75c2e9
github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc000040900, {0x73b207f010a0, 0xc001286100}, 0xc006d14280)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:437 +0x1f2 fp=0xc0012a18d0 sp=0xc0012a1880 pc=0x8df032
github.com/go-chi/chi/v5.(*Mux).routeHTTP-fm({0x73b207f010a0?, 0xc001286100?}, 0x8f0000c000c14400?)
	<autogenerated>:1 +0x36 fp=0xc0012a1900 sp=0xc0012a18d0 pc=0x8e2f96
net/http.HandlerFunc.ServeHTTP(0xc000c14400?, {0x73b207f010a0?, 0xc001286100?}, 0x10?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc0012a1928 sp=0xc0012a1900 pc=0x75c2e9
github.com/go-chi/chi/v5/middleware.Recoverer.func1({0x73b207f010a0?, 0xc001286100?}, 0x1e6b9d7?)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/middleware/recoverer.go:37 +0x6c fp=0xc0012a1988 sp=0xc0012a1928 pc=0x916e0c
net/http.HandlerFunc.ServeHTTP(0xc00ae94090?, {0x73b207f010a0?, 0xc001286100?}, 0x1f33ff5?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc0012a19b0 sp=0xc0012a1988 pc=0x75c2e9
decred.org/dcrdex/client/webserver.(*WebServer).securityMiddleware-fm.(*WebServer).securityMiddleware.func1({0x73b207f010a0, 0xc001286100}, 0xc006d14280)
	/home/joe/git/dcrdex/client/webserver/middleware.go:34 +0x174 fp=0xc0012a1a00 sp=0xc0012a19b0 pc=0xbd1f94
net/http.HandlerFunc.ServeHTTP(0x36ac3e0?, {0x73b207f010a0?, 0xc001286100?}, 0x73b20c1e1708?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc0012a1a28 sp=0xc0012a1a00 pc=0x75c2e9
decred.org/dcrdex/client/webserver.New.RequestLogger.func3.1({0x291cbb0, 0xc006d700e0}, 0xc006d14140)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/middleware/logger.go:57 +0x154 fp=0xc0012a1ae0 sp=0xc0012a1a28 pc=0xbcc554
net/http.HandlerFunc.ServeHTTP(0x29204e0?, {0x291cbb0?, 0xc006d700e0?}, 0x36a4110?)
	/usr/lib/go/src/net/http/server.go:2220 +0x29 fp=0xc0012a1b08 sp=0xc0012a1ae0 pc=0x75c2e9
github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc000040900, {0x291cbb0, 0xc006d700e0}, 0xc006d14000)
	/home/joe/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:88 +0x2ee fp=0xc0012a1b68 sp=0xc0012a1b08 pc=0x8dcf4e
net/http.serverHandler.ServeHTTP({0x2918de0?}, {0x291cbb0?, 0xc006d700e0?}, 0x6?)
	/usr/lib/go/src/net/http/server.go:3210 +0x8e fp=0xc0012a1b98 sp=0xc0012a1b68 pc=0x77b7ee
net/http.(*conn).serve(0xc0005e06c0, {0x29204a8, 0xc0007f80c0})
	/usr/lib/go/src/net/http/server.go:2092 +0x5d0 fp=0xc0012a1fb8 sp=0xc0012a1b98 pc=0x75ac90
net/http.(*Server).Serve.gowrap3()
	/usr/lib/go/src/net/http/server.go:3360 +0x28 fp=0xc0012a1fe0 sp=0xc0012a1fb8 pc=0x760108
runtime.goexit({})
	/usr/lib/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc0012a1fe8 sp=0xc0012a1fe0 pc=0x4802c1
created by net/http.(*Server).Serve in goroutine 107
	/usr/lib/go/src/net/http/server.go:3360 +0x485

rax    0xca
rbx    0x0
rcx    0x482103
rdx    0x0
rdi    0x3792080
rsi    0x80
rbp    0x7ffcef7ad760
rsp    0x7ffcef7ad718
r8     0x0
r9     0x0
r10    0x0
r11    0x200286
r12    0x7ffcef7ad740
r13    0xc0004cf794
r14    0x3790100
r15    0x1ffffffee7ffff
rip    0x482101
rflags 0x200286
cs     0x33
fs     0x0
gs     0x0