Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

runtime: a Windows application launched via Steam sometimes freezes #71242

Open
hajimehoshi opened this issue Jan 13, 2025 · 15 comments
Open

runtime: a Windows application launched via Steam sometimes freezes #71242

hajimehoshi opened this issue Jan 13, 2025 · 15 comments
Labels
BugReport Issues describing a possible bug in the Go implementation. compiler/runtime Issues related to the Go compiler and/or runtime. NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. OS-Windows
Milestone

Comments

@hajimehoshi
Copy link
Member

hajimehoshi commented Jan 13, 2025

Go version

go version go1.23.2 windows/amd64

Output of go env in your module/workspace:

set GO111MODULE=
set GOARCH=amd64
set GOBIN=
set GOCACHE=C:\Users\hajimehoshi\AppData\Local\go-build
set GOENV=C:\Users\hajimehoshi\AppData\Roaming\go\env
set GOEXE=.exe
set GOEXPERIMENT=
set GOFLAGS=
set GOHOSTARCH=amd64
set GOHOSTOS=windows
set GOINSECURE=
set GOMODCACHE=C:\Users\hajimehoshi\go\pkg\mod
set GONOPROXY=
set GONOSUMDB=
set GOOS=windows
set GOPATH=C:\Users\hajimehoshi\go
set GOPRIVATE=
set GOPROXY=https://proxy.golang.org,direct
set GOROOT=C:\Program Files\Go
set GOSUMDB=sum.golang.org
set GOTMPDIR=
set GOTOOLCHAIN=auto
set GOTOOLDIR=C:\Program Files\Go\pkg\tool\windows_amd64
set GOVCS=
set GOVERSION=go1.23.2
set GODEBUG=
set GOTELEMETRY=local
set GOTELEMETRYDIR=C:\Users\hajimehoshi\AppData\Roaming\go\telemetry
set GCCGO=gccgo
set GOAMD64=v1
set AR=ar
set CC=gcc
set CXX=g++
set CGO_ENABLED=1
set GOMOD=C:\Users\hajimehoshi\ebiten\go.mod
set GOWORK=
set CGO_CFLAGS=-O2 -g
set CGO_CPPFLAGS=
set CGO_CXXFLAGS=-O2 -g
set CGO_FFLAGS=-O2 -g
set CGO_LDFLAGS=-O2 -g
set PKG_CONFIG=pkg-config
set GOGCCFLAGS=-m64 -mthreads -Wl,--no-gc-sections -fmessage-length=0 -ffile-prefix-map=C:\Users\HAJIME~1\AppData\Local\Temp\go-build1125585036=/tmp/go-build -gno-record-gcc-switches

What did you do?

Compile this Go program to an execute file (with -ldflags="-H=windowsgui")

EDIT: I could minized the case further. See #71242 (comment)

package main

import (
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"syscall"
	"time"
	"unsafe"
)

const (
	WS_OVERLAPPEDWINDOW = 0x00000000 | 0x00C00000 | 0x00080000 | 0x00040000 | 0x00020000 | 0x00010000
	CW_USEDEFAULT       = ^0x7fffffff
	SW_SHOW             = 5
	WM_DESTROY          = 2
)

type (
	ATOM      uint16
	HANDLE    uintptr
	HINSTANCE HANDLE
	HICON     HANDLE
	HCURSOR   HANDLE
	HBRUSH    HANDLE
	HWND      HANDLE
	HMENU     HANDLE
)

type WNDCLASSEX struct {
	Size       uint32
	Style      uint32
	WndProc    uintptr
	ClsExtra   int32
	WndExtra   int32
	Instance   HINSTANCE
	Icon       HICON
	Cursor     HCURSOR
	Background HBRUSH
	MenuName   *uint16
	ClassName  *uint16
	IconSm     HICON
}

type RECT struct {
	Left, Top, Right, Bottom int32
}

type POINT struct {
	X, Y int32
}

type MSG struct {
	Hwnd    HWND
	Message uint32
	WParam  uintptr
	LParam  uintptr
	Time    uint32
	Pt      POINT
}

func GetModuleHandle(modulename *uint16) HINSTANCE {
	r, _, _ := syscall.SyscallN(procGetModuleHandle.Addr(), uintptr(unsafe.Pointer(modulename)))
	return HINSTANCE(r)
}

func RegisterClassEx(w *WNDCLASSEX) ATOM {
	r, _, _ := syscall.SyscallN(procRegisterClassEx.Addr(), uintptr(unsafe.Pointer(w)))
	return ATOM(r)
}

func CreateWindowEx(exStyle uint, className, windowName *uint16,
	style uint, x, y, width, height int, parent HWND, menu HMENU,
	instance HINSTANCE, param unsafe.Pointer) HWND {
	r, _, _ := syscall.SyscallN(procCreateWindowEx.Addr(), uintptr(exStyle), uintptr(unsafe.Pointer(className)),
		uintptr(unsafe.Pointer(windowName)), uintptr(style), uintptr(x), uintptr(y), uintptr(width), uintptr(height),
		uintptr(parent), uintptr(menu), uintptr(instance), uintptr(param))
	return HWND(r)
}

func AdjustWindowRect(rect *RECT, style uint, menu bool) bool {
	var iMenu uintptr
	if menu {
		iMenu = 1
	}
	r, _, _ := syscall.SyscallN(procAdjustWindowRect.Addr(), uintptr(unsafe.Pointer(rect)), uintptr(style), iMenu)
	return r != 0
}

func ShowWindow(hwnd HWND, cmdshow int) bool {
	r, _, _ := syscall.SyscallN(procShowWindow.Addr(), uintptr(hwnd), uintptr(cmdshow))
	return r != 0
}

func GetMessage(msg *MSG, hwnd HWND, msgFilterMin, msgFilterMax uint32) int {
	r, _, _ := syscall.SyscallN(procGetMessage.Addr(), uintptr(unsafe.Pointer(msg)), uintptr(hwnd), uintptr(msgFilterMin), uintptr(msgFilterMax))
	return int(r)
}

func TranslateMessage(msg *MSG) bool {
	r, _, _ := syscall.SyscallN(procTranslateMessage.Addr(), uintptr(unsafe.Pointer(msg)))
	return r != 0
}

func DispatchMessage(msg *MSG) uintptr {
	r, _, _ := syscall.SyscallN(procDispatchMessage.Addr(), uintptr(unsafe.Pointer(msg)))
	return r
}

func DefWindowProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	r, _, _ := syscall.SyscallN(procDefWindowProc.Addr(), uintptr(hwnd), uintptr(msg), wParam, lParam)
	return r
}

func PostQuitMessage(exitCode int) {
	syscall.SyscallN(procPostQuitMessage.Addr(), uintptr(exitCode))
}

var (
	kernel32            = syscall.NewLazyDLL("kernel32.dll")
	procGetModuleHandle = kernel32.NewProc("GetModuleHandleW")

	user32               = syscall.NewLazyDLL("user32.dll")
	procRegisterClassEx  = user32.NewProc("RegisterClassExW")
	procCreateWindowEx   = user32.NewProc("CreateWindowExW")
	procAdjustWindowRect = user32.NewProc("AdjustWindowRect")
	procShowWindow       = user32.NewProc("ShowWindow")
	procGetMessage       = user32.NewProc("GetMessageW")
	procTranslateMessage = user32.NewProc("TranslateMessage")
	procDispatchMessage  = user32.NewProc("DispatchMessageW")
	procDefWindowProc    = user32.NewProc("DefWindowProcW")
	procPostQuitMessage  = user32.NewProc("PostQuitMessage")
)

func init() {
	runtime.LockOSThread()
}

func main() {
	className, err := syscall.UTF16PtrFromString("Sample Window Class")
	if err != nil {
		panic(err)
	}
	inst := GetModuleHandle(className)

	wc := WNDCLASSEX{
		Size:      uint32(unsafe.Sizeof(WNDCLASSEX{})),
		WndProc:   syscall.NewCallback(wndProc),
		Instance:  inst,
		ClassName: className,
	}

	RegisterClassEx(&wc)

	wr := RECT{
		Left:   0,
		Top:    0,
		Right:  320,
		Bottom: 240,
	}
	title, err := syscall.UTF16PtrFromString("My Title")
	if err != nil {
		panic(err)
	}
	AdjustWindowRect(&wr, WS_OVERLAPPEDWINDOW, false)
	hwnd := CreateWindowEx(
		0, className,
		title,
		WS_OVERLAPPEDWINDOW,
		CW_USEDEFAULT, CW_USEDEFAULT, int(wr.Right-wr.Left), int(wr.Bottom-wr.Top),
		0, 0, inst, nil,
	)
	if hwnd == 0 {
		panic(syscall.GetLastError())
	}

	ShowWindow(hwnd, SW_SHOW)

	go func() {
		for {
			_ = make([]byte, 256*1024)
			time.Sleep(time.Millisecond)
		}
	}()

	go func() {
		f, err := os.Create("log.txt")
		if err != nil {
			panic(err)
		}
		defer f.Close()

		log.SetOutput(f)

		for {
			time.Sleep(time.Second)
			var gcStats debug.GCStats
			debug.ReadGCStats(&gcStats)
			log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
			if err := f.Sync(); err != nil {
				panic(err)
			}
		}
	}()

	var msg MSG
	for GetMessage(&msg, 0, 0, 0) != 0 {
		TranslateMessage(&msg)
		DispatchMessage(&msg)
	}
}

func wndProc(hwnd HWND, msg uint32, wparam, lparam uintptr) uintptr {
	switch msg {
	case WM_DESTROY:
		PostQuitMessage(0)
	}
	return DefWindowProc(hwnd, msg, wparam, lparam)
}

Replace an exe file in a Steam game with the compiled exe file, and run it via Steam client.

What did you see happen?

The application sometimes freezes for more than 10 seconds. For example, I saw this log

2025/01/13 23:23:41 LastGC: 2025-01-13 23:23:41.6060533 +0900 JST, NumGC: 51, PauseTotal: 1.3186ms
2025/01/13 23:23:42 LastGC: 2025-01-13 23:23:42.6470915 +0900 JST, NumGC: 102, PauseTotal: 4.5945ms
2025/01/13 23:23:43 LastGC: 2025-01-13 23:23:43.6434896 +0900 JST, NumGC: 152, PauseTotal: 7.8752ms
2025/01/13 23:23:44 LastGC: 2025-01-13 23:23:44.6481645 +0900 JST, NumGC: 204, PauseTotal: 10.224ms
2025/01/13 23:23:45 LastGC: 2025-01-13 23:23:45.6448025 +0900 JST, NumGC: 255, PauseTotal: 12.5067ms
2025/01/13 23:23:46 LastGC: 2025-01-13 23:23:46.6584686 +0900 JST, NumGC: 309, PauseTotal: 14.2881ms
2025/01/13 23:23:47 LastGC: 2025-01-13 23:23:47.6550747 +0900 JST, NumGC: 362, PauseTotal: 17.056ms
2025/01/13 23:23:48 LastGC: 2025-01-13 23:23:48.6681264 +0900 JST, NumGC: 413, PauseTotal: 18.5012ms
2025/01/13 23:23:49 LastGC: 2025-01-13 23:23:49.6577372 +0900 JST, NumGC: 464, PauseTotal: 21.2877ms
2025/01/13 23:23:50 LastGC: 2025-01-13 23:23:50.6675317 +0900 JST, NumGC: 514, PauseTotal: 24.9508ms
2025/01/13 23:23:51 LastGC: 2025-01-13 23:23:51.6642908 +0900 JST, NumGC: 563, PauseTotal: 27.2671ms
2025/01/13 23:23:52 LastGC: 2025-01-13 23:23:52.6696781 +0900 JST, NumGC: 612, PauseTotal: 29.6692ms
2025/01/13 23:23:53 LastGC: 2025-01-13 23:23:53.6818947 +0900 JST, NumGC: 661, PauseTotal: 31.9968ms
2025/01/13 23:23:54 LastGC: 2025-01-13 23:23:54.6830572 +0900 JST, NumGC: 711, PauseTotal: 34.9958ms
2025/01/13 23:23:55 LastGC: 2025-01-13 23:23:55.6889185 +0900 JST, NumGC: 761, PauseTotal: 38.2468ms
2025/01/13 23:23:56 LastGC: 2025-01-13 23:23:56.6869067 +0900 JST, NumGC: 813, PauseTotal: 41.5747ms
2025/01/13 23:23:57 LastGC: 2025-01-13 23:23:57.6920325 +0900 JST, NumGC: 863, PauseTotal: 45.5415ms
2025/01/13 23:24:18 LastGC: 2025-01-13 23:23:58.2810119 +0900 JST, NumGC: 894, PauseTotal: 47.2387ms
2025/01/13 23:24:19 LastGC: 2025-01-13 23:24:19.3442472 +0900 JST, NumGC: 945, PauseTotal: 51.3869ms
2025/01/13 23:24:20 LastGC: 2025-01-13 23:24:20.3460036 +0900 JST, NumGC: 995, PauseTotal: 54.0004ms
2025/01/13 23:24:21 LastGC: 2025-01-13 23:24:21.3371656 +0900 JST, NumGC: 1047, PauseTotal: 55.3437ms
2025/01/13 23:24:22 LastGC: 2025-01-13 23:24:22.344327 +0900 JST, NumGC: 1098, PauseTotal: 56.9757ms
2025/01/13 23:24:23 LastGC: 2025-01-13 23:24:23.3523815 +0900 JST, NumGC: 1147, PauseTotal: 61.8229ms
2025/01/13 23:24:24 LastGC: 2025-01-13 23:24:24.3560493 +0900 JST, NumGC: 1200, PauseTotal: 64.7476ms
2025/01/13 23:24:25 LastGC: 2025-01-13 23:24:25.3552534 +0900 JST, NumGC: 1250, PauseTotal: 67.5551ms

You can see a freeze happens between 22:23:57 and 22:24:18.

What did you expect to see?

The application doesn't freeze.

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 13, 2025

This was originally reported at hajimehoshi/ebiten#3181 by @corfe83.

  • A Steam overlay might cause this issue but we are not sure. This freeze happens even when the Steam overlay is disabled.
  • This freeze never happens when GC is suspended by debug.SetGCPercent(-1)
  • This freeze never happens when the application is launched without Steam.

@gabyhelp gabyhelp added the BugReport Issues describing a possible bug in the Go implementation. label Jan 13, 2025
@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 13, 2025

I could reproduce this even with a console application (without -ldflags="-H=windowsgui"). runtime.LockOSThread is not needed. Replacing an exe in a Steam game is still required.

package main

import (
	"log"
	"runtime"
	"runtime/debug"
	"time"
)

func main() {
	go func() {
		for {
			_ = make([]byte, 256*1024)
			time.Sleep(time.Millisecond)
		}
	}()

	for {
		time.Sleep(time.Second)
		var gcStats debug.GCStats
		debug.ReadGCStats(&gcStats)
		log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
	}
}
2025/01/14 00:19:14 LastGC: 2025-01-14 00:19:14.3262459 +0900 JST, NumGC: 51, PauseTotal: 1.016ms
2025/01/14 00:19:15 LastGC: 2025-01-14 00:19:15.3519275 +0900 JST, NumGC: 100, PauseTotal: 6.9976ms
2025/01/14 00:19:16 LastGC: 2025-01-14 00:19:16.3609673 +0900 JST, NumGC: 148, PauseTotal: 11.6081ms
2025/01/14 00:19:17 LastGC: 2025-01-14 00:19:17.3715984 +0900 JST, NumGC: 197, PauseTotal: 15.5458ms
2025/01/14 00:19:18 LastGC: 2025-01-14 00:19:18.3726086 +0900 JST, NumGC: 245, PauseTotal: 17.6441ms
2025/01/14 00:19:19 LastGC: 2025-01-14 00:19:19.3673871 +0900 JST, NumGC: 293, PauseTotal: 19.8266ms
2025/01/14 00:19:20 LastGC: 2025-01-14 00:19:20.3597451 +0900 JST, NumGC: 341, PauseTotal: 25.4462ms
2025/01/14 00:19:21 LastGC: 2025-01-14 00:19:21.3536769 +0900 JST, NumGC: 388, PauseTotal: 30.0621ms
2025/01/14 00:19:37 LastGC: 2025-01-14 00:19:37.1879618 +0900 JST, NumGC: 424, PauseTotal: 30.2376ms
2025/01/14 00:19:38 LastGC: 2025-01-14 00:19:38.1708792 +0900 JST, NumGC: 468, PauseTotal: 31.8384ms
2025/01/14 00:19:39 LastGC: 2025-01-14 00:19:39.1731811 +0900 JST, NumGC: 515, PauseTotal: 33.0891ms
2025/01/14 00:19:40 LastGC: 2025-01-14 00:19:40.1890122 +0900 JST, NumGC: 562, PauseTotal: 33.7643ms
2025/01/14 00:19:41 LastGC: 2025-01-14 00:19:41.1767277 +0900 JST, NumGC: 608, PauseTotal: 34.0499ms

A freeze occurs between 00:19:21 and 00:19:37

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 13, 2025

Apparently most of the threads got stuck at NtWaitForSingleObject when freezing, but I have no idea what was going on there...

Thread 0x4 (gameoverlayrenderer64!OverlayHookD3D3 seems a hooked function by Steam)
Image

Thread 0x5
Image

@hajimehoshi hajimehoshi changed the title a Windows application launched via Steam sometimes freezes runtime: a Windows application launched via Steam sometimes freezes Jan 13, 2025
@gopherbot gopherbot added the compiler/runtime Issues related to the Go compiler and/or runtime. label Jan 13, 2025
@mknyszek mknyszek added the NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. label Jan 13, 2025
@mknyszek mknyszek added this to the Backlog milestone Jan 13, 2025
@mknyszek
Copy link
Contributor

CC @golang/runtime

@qmuntal
Copy link
Member

qmuntal commented Jan 13, 2025

Does this issue reproduce with Go 1.22?

@hajimehoshi
Copy link
Member Author

Does this issue reproduce with Go 1.22?

Yes, I could reproduce this with Go 1.22.9.

@mknyszek
Copy link
Contributor

mknyszek commented Jan 13, 2025

Do you have a stack trace for each thread, for example from a debugger?

My guess based on the information provided would be that the threads are stuck in some GC-related thing (spinning up GC mark workers? that only happens on the first GC if GOMAXPROCS doesn't ever change) on a note (https://cs.opensource.google/go/go/+/master:src/runtime/os_windows.go;l=676;drc=4f881115d4067bda8a236aabcae8c41cdd13b4d0). I couldn't tell you why that would be a problem, since this happens all the time.

Alternatively, the threads are going to sleep for STW, which is also based on a note (https://cs.opensource.google/go/go/+/master:src/runtime/proc.go;l=1612;drc=f025d19e7b3f0c66242760c213cc2b54cb100f69), and perhaps something is preventing the thread stopping the world from continuing promptly. Note that the thread which is stopping-the-world spins, sleeping in 100ms increments (but is awoken early when the last thread goes to sleep). So it might be that too. We could confirm/deny this by disabling the GC but calling runtime.ReadMemStats at some regular interval. This also happens all the time without issue, though, so I also don't really have a guess as to why this would be a problem running under Steam.

@hajimehoshi
Copy link
Member Author

Do you have a stack trace for each thread, for example from a debugger?

I'm afraid I'm not familiar with WinDbg. How can I get the stack traces as a file? I'll try tomorrow.

Also I can insert printlns in the runtime by -overlay. I'll try this later too.

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

I dumped stack traces by x64dbg:

Stack traces by x64dbg
Thread ID           Address          To               From             Size       Party  Comment                                      
10876 - Main Thread                                                                      
                    0000009585FFF798 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    0000009585FFF838 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    0000009585FFF9A8 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009585FFF9D8 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    0000009585FFF9F8 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009585FFFA30 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009585FFFA48 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    0000009585FFFAB0 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    0000009585FFFAE8 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    0000009585FFFB18 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    0000009585FFFC90 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    0000009585FFFCC8 000000000080EDCB 000000000080E9B1 58         User   innovation2007_windows_amd64.000000000080E9B1
                    0000009585FFFD20 000000000083A7F3 000000000080EDCB 2A7A12A1F0 User   innovation2007_windows_amd64.000000000080EDCB
                    000000C000129F10 000000000083572E 000000000083A7F3 20         User   innovation2007_windows_amd64.000000000083A7F3
                    000000C000129F30 00000000007E7FC9 000000000083572E 90         User   innovation2007_windows_amd64.000000000083572E
                    000000C000129FC0 00000000007E7EA5 00000000007E7FC9 18         User   innovation2007_windows_amd64.00000000007E7FC9
                    000000C000129FD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C000129FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
26956                                                                                    
                    0000009586FFF778 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    0000009586FFF818 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    0000009586FFF988 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009586FFF9B8 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    0000009586FFF9E0 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009586FFFA18 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009586FFFA30 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    0000009586FFFA98 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    0000009586FFFAD0 000000000080BD51 00000000007DB9D2 20         User   innovation2007_windows_amd64.00000000007DB9D2
                    0000009586FFFAF0 000000000080A73D 000000000080BD51 28         User   innovation2007_windows_amd64.000000000080BD51
                    0000009586FFFB18 000000000080A68A 000000000080A73D 28         User   innovation2007_windows_amd64.000000000080A73D
                    0000009586FFFB40 000000000083A765 000000000080A68A 8          User   innovation2007_windows_amd64.000000000080A68A
                    0000009586FFFB48 000000000083E477 000000000083A765 8          User   innovation2007_windows_amd64.000000000083A765
                    0000009586FFFB50 0000000000000000 000000000083E477            User   innovation2007_windows_amd64.000000000083E477
20696                                                                                    
                    00000095861FF8D8 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                    00000095861FFBB8 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                    00000095861FFBE8 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                    00000095861FFC68 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
29680                                                                                    
                    00000095865FF9F8 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                    00000095865FFCD8 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                    00000095865FFD08 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                    00000095865FFD88 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
12772                                                                                    
                    00000095863FFC28 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                    00000095863FFF08 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                    00000095863FFF38 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                    00000095863FFFB8 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
21908                                                                                    
                    00000095867FF328 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095867FF3C8 00007FFD45D11D4F 00007FFDB8D99CEE 70         User   kernelbase.WaitForSingleObjectEx+8E
                    00000095867FF438 000000000083E029 00007FFD45D11D4F 170        User   gameoverlayrenderer64.OverlayHookD3D3+2720F
                    00000095867FF5A8 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095867FF5D8 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    00000095867FF5F8 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095867FF630 0000000000801C56 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095867FF648 0000000000802592 0000000000801C56 560        User   innovation2007_windows_amd64.0000000000801C56
                    00000095867FFBA8 0000000000813D35 0000000000802592 18         User   innovation2007_windows_amd64.0000000000802592
                    00000095867FFBC0 0000000000813A36 0000000000813D35 60         User   innovation2007_windows_amd64.0000000000813D35
                    00000095867FFC20 000000000081379A 0000000000813A36 80         User   innovation2007_windows_amd64.0000000000813A36
                    00000095867FFCA0 000000000080A73D 000000000081379A 28         User   innovation2007_windows_amd64.000000000081379A
                    00000095867FFCC8 000000000080A68A 000000000080A73D 28         User   innovation2007_windows_amd64.000000000080A73D
                    00000095867FFCF0 000000000083A765 000000000080A68A 8          User   innovation2007_windows_amd64.000000000080A68A
                    00000095867FFCF8 000000000083E477 000000000083A765 8          User   innovation2007_windows_amd64.000000000083A765
                    00000095867FFD00 0000000000000000 000000000083E477            User   innovation2007_windows_amd64.000000000083E477
29272                                                                                    
                    00000095869FFB78 00000000007EA1B4 00000000007EC202 40         User   innovation2007_windows_amd64.00000000007EC202
                    00000095869FFBB8 00000000007E9F27 00000000007EA1B4 A8         User   innovation2007_windows_amd64.00000000007EA1B4
                    00000095869FFC60 00000000007EBF14 00000000007E9F27 68         User   innovation2007_windows_amd64.00000000007E9F27
                    00000095869FFCC8 00000000007E8407 00000000007EBF14 50         User   innovation2007_windows_amd64.00000000007EBF14
                    00000095869FFD18 000000000083A869 00000000007E8407 2A79728218 User   innovation2007_windows_amd64.00000000007E8407
                    000000C000127F30 00000000007E80CC 000000000083A869 90         User   innovation2007_windows_amd64.000000000083A869
                    000000C000127FC0 00000000007E7EA5 00000000007E80CC 18         User   innovation2007_windows_amd64.00000000007E80CC
                    000000C000127FD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C000127FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
12632                                                                                    
                    0000009586BFF208 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    0000009586BFF2A8 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    0000009586BFF418 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009586BFF448 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    0000009586BFF468 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009586BFF4A0 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009586BFF4B8 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    0000009586BFF520 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    0000009586BFF558 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    0000009586BFF588 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    0000009586BFF700 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    0000009586BFF738 000000000080EDCB 000000000080E9B1 58         User   innovation2007_windows_amd64.000000000080E9B1
                    0000009586BFF790 000000000083A7F3 000000000080EDCB 2A79534780 User   innovation2007_windows_amd64.000000000080EDCB
                    000000C000133F10 000000000083572E 000000000083A7F3 20         User   innovation2007_windows_amd64.000000000083A7F3
                    000000C000133F30 00000000007E7FC9 000000000083572E 90         User   innovation2007_windows_amd64.000000000083572E
                    000000C000133FC0 00000000007E7EA5 00000000007E7FC9 18         User   innovation2007_windows_amd64.00000000007E7FC9
                    000000C000133FD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C000133FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
12836                                                                                    
                    0000009586DFEE48 00007FFDB8D9BEAD 00007FFDBB9907A4 30         System ntdll.NtReleaseMutant+14
                    0000009586DFEE78 00007FFD45D11EC0 00007FFDB8D9BEAD 70         User   kernelbase.ReleaseMutex+D
                    0000009586DFEEE8 000000000083E029 00007FFD45D11EC0 170        User   gameoverlayrenderer64.OverlayHookD3D3+27380
                    0000009586DFF058 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009586DFF088 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    0000009586DFF0B0 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009586DFF0E8 0000000000801C56 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009586DFF100 0000000000802592 0000000000801C56 560        User   innovation2007_windows_amd64.0000000000801C56
                    0000009586DFF660 0000000000813D35 0000000000802592 18         User   innovation2007_windows_amd64.0000000000802592
                    0000009586DFF678 00000000007EDC96 0000000000813D35 30         User   innovation2007_windows_amd64.0000000000813D35
                    0000009586DFF6A8 00000000007F3825 00000000007EDC96 18         User   innovation2007_windows_amd64.00000000007EDC96
                    0000009586DFF6C0 00000000007EBC87 00000000007F3825 68         User   innovation2007_windows_amd64.00000000007F3825
                    0000009586DFF728 00000000007E843E 00000000007EBC87 50         User   innovation2007_windows_amd64.00000000007EBC87
                    0000009586DFF778 000000000083A869 00000000007E843E 2A7932E7B8 User   innovation2007_windows_amd64.00000000007E843E
                    000000C00012DF30 00000000007E80CC 000000000083A869 90         User   innovation2007_windows_amd64.000000000083A869
                    000000C00012DFC0 00000000007E7EA5 00000000007E80CC 18         User   innovation2007_windows_amd64.00000000007E80CC
                    000000C00012DFD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C00012DFE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
22360                                                                                    
                    00000095871FF598 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095871FF638 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    00000095871FF7A8 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095871FF7D8 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    00000095871FF7F8 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095871FF830 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095871FF848 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    00000095871FF8B0 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    00000095871FF8E8 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    00000095871FF918 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    00000095871FFA90 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    00000095871FFAC8 000000000080EDCB 000000000080E9B1 58         User   innovation2007_windows_amd64.000000000080E9B1
                    00000095871FFB20 000000000083A7F3 000000000080EDCB 2A78F2C3F0 User   innovation2007_windows_amd64.000000000080EDCB
                    000000C00012BF10 000000000083572E 000000000083A7F3 20         User   innovation2007_windows_amd64.000000000083A7F3
                    000000C00012BF30 00000000007E7FC9 000000000083572E 90         User   innovation2007_windows_amd64.000000000083572E
                    000000C00012BFC0 00000000007E7EA5 00000000007E7FC9 18         User   innovation2007_windows_amd64.00000000007E7FC9
                    000000C00012BFD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C00012BFE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
1584                                                                                     
                    00000095873FF388 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095873FF428 00007FFD45D11D4F 00007FFDB8D99CEE 70         User   kernelbase.WaitForSingleObjectEx+8E
                    00000095873FF498 000000000083E029 00007FFD45D11D4F 170        User   gameoverlayrenderer64.OverlayHookD3D3+2720F
                    00000095873FF608 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095873FF638 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    00000095873FF660 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095873FF698 0000000000801C56 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095873FF6B0 0000000000802592 0000000000801C56 560        User   innovation2007_windows_amd64.0000000000801C56
                    00000095873FFC10 0000000000813D35 0000000000802592 18         User   innovation2007_windows_amd64.0000000000802592
                    00000095873FFC28 00000000007EDC96 0000000000813D35 30         User   innovation2007_windows_amd64.0000000000813D35
                    00000095873FFC58 00000000007F3825 00000000007EDC96 18         User   innovation2007_windows_amd64.00000000007EDC96
                    00000095873FFC70 00000000007EBC87 00000000007F3825 68         User   innovation2007_windows_amd64.00000000007F3825
                    00000095873FFCD8 00000000007E8407 00000000007EBC87 50         User   innovation2007_windows_amd64.00000000007EBC87
                    00000095873FFD28 000000000083A869 00000000007E8407 2A78D36208 User   innovation2007_windows_amd64.00000000007E8407
                    000000C000135F30 00000000007E80CC 000000000083A869 90         User   innovation2007_windows_amd64.000000000083A869
                    000000C000135FC0 00000000007E7EA5 00000000007E80CC 18         User   innovation2007_windows_amd64.00000000007E80CC
                    000000C000135FD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C000135FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
10740                                                                                    
                    00000095875FF488 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095875FF528 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    00000095875FF698 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095875FF6C8 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    00000095875FF6E8 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095875FF720 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095875FF738 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    00000095875FF7A0 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    00000095875FF7D8 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    00000095875FF808 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    00000095875FF980 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    00000095875FF9B8 000000000080EDCB 000000000080E9B1 58         User   innovation2007_windows_amd64.000000000080E9B1
                    00000095875FFA10 000000000083A7F3 000000000080EDCB 2A78B322B8 User   innovation2007_windows_amd64.000000000080EDCB
                    000000C000131CC8 000000000083572E 000000000083A7F3 20         User   innovation2007_windows_amd64.000000000083A7F3
                    000000C000131CE8 00000000007EAF67 000000000083572E 38         User   innovation2007_windows_amd64.000000000083572E
                    000000C000131D20 00000000007EA88A 00000000007EAF67 60         User   innovation2007_windows_amd64.00000000007EAF67
                    000000C000131D80 00000000007DCDF4 00000000007EA88A 28         User   innovation2007_windows_amd64.00000000007EA88A
                    000000C000131DA8 00000000008314DE 00000000007DCDF4 A0         User   innovation2007_windows_amd64.00000000007DCDF4
                    000000C000131E48 00000000007DCEE5 00000000008314DE 28         User   innovation2007_windows_amd64.00000000008314DE
                    000000C000131E70 0000000000878527 00000000007DCEE5 D8         User   innovation2007_windows_amd64.00000000007DCEE5
                    000000C000131F48 000000000080737D 0000000000878527 90         User   innovation2007_windows_amd64.0000000000878527
                    000000C000131FD8 000000000083C821 000000000080737D 8          User   innovation2007_windows_amd64.000000000080737D
                    000000C000131FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
29196                                                                                    
                    00000095877FF1C8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095877FF268 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    00000095877FF3D8 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095877FF408 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    00000095877FF430 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095877FF468 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095877FF480 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    00000095877FF4E8 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    00000095877FF520 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    00000095877FF550 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    00000095877FF6C8 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    00000095877FF700 000000000080A777 000000000080E9B1 28         User   innovation2007_windows_amd64.000000000080E9B1
                    00000095877FF728 000000000080A68A 000000000080A777 28         User   innovation2007_windows_amd64.000000000080A777
                    00000095877FF750 000000000083A765 000000000080A68A 8          User   innovation2007_windows_amd64.000000000080A68A
                    00000095877FF758 000000000083E477 000000000083A765 8          User   innovation2007_windows_amd64.000000000083A765
                    00000095877FF760 0000000000000000 000000000083E477            User   innovation2007_windows_amd64.000000000083E477

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

I also dumped goroutine stacks by delve

Goroutine Dumps
  Goroutine 1 - User: C:/Users/hajimehoshi/ebiten/examples/test/main.go:29 main.main (0x878527) [GC assist wait]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007eaf67 in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x00000000007eaf67 in runtime.gcParkAssist
            at C:/Program Files/Go/src/runtime/mgcmark.go:737
        3  0x00000000007ea88a in runtime.gcAssistAlloc
            at C:/Program Files/Go/src/runtime/mgcmark.go:577
        4  0x00000000007dcdf4 in runtime.deductAssistCredit
            at C:/Program Files/Go/src/runtime/malloc.go:1349
        5  0x00000000008314de in runtime.mallocgc
            at C:/Program Files/Go/src/runtime/malloc.go:1037
        6  0x00000000007dcee5 in runtime.newobject
            at C:/Program Files/Go/src/runtime/malloc.go:1386
        7  0x0000000000878527 in main.main
            at C:/Users/hajimehoshi/ebiten/examples/test/main.go:29
        8  0x000000000080737d in runtime.main
            at C:/Program Files/Go/src/runtime/proc.go:272
        9  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 2 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [force gc (idle) 53406931548000]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x0000000000807698 in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x0000000000807698 in runtime.forcegchelper
            at C:/Program Files/Go/src/runtime/proc.go:337
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 3 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [GC sweep wait 53437762463500]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007f153f in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x00000000007f153f in runtime.bgsweep
            at C:/Program Files/Go/src/runtime/mgcsweep.go:317
        3  0x00000000007e5e25 in runtime.gcenable.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:203
        4  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 4 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [GC scavenge wait]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007eef29 in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x00000000007eef29 in runtime.(*scavengerState).park
            at C:/Program Files/Go/src/runtime/mgcscavenge.go:425
        3  0x00000000007ef4b9 in runtime.bgscavenge
            at C:/Program Files/Go/src/runtime/mgcscavenge.go:658
        4  0x00000000007e5dc5 in runtime.gcenable.gowrap2
            at C:/Program Files/Go/src/runtime/mgc.go:204
        5  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 5 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [finalizer wait 53406931548000]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e4ee7 in runtime.runfinq
            at C:/Program Files/Go/src/runtime/mfinal.go:193
        2  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 18 - User: C:/Users/hajimehoshi/ebiten/examples/test/main.go:22 main.main.func1 (0x8787a5) [GC assist wait]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007eaf67 in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x00000000007eaf67 in runtime.gcParkAssist
            at C:/Program Files/Go/src/runtime/mgcmark.go:737
        3  0x00000000007ea88a in runtime.gcAssistAlloc
            at C:/Program Files/Go/src/runtime/mgcmark.go:577
        4  0x00000000007dcdf4 in runtime.deductAssistCredit
            at C:/Program Files/Go/src/runtime/malloc.go:1349
        5  0x00000000008314de in runtime.mallocgc
            at C:/Program Files/Go/src/runtime/malloc.go:1037
        6  0x0000000000836609 in runtime.makeslice
            at C:/Program Files/Go/src/runtime/slice.go:116
        7  0x00000000008787a5 in main.main.func1
            at C:/Users/hajimehoshi/ebiten/examples/test/main.go:22
        8  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 19 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53406931548000]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 20 - User: :0 ??? (0x7ffdbb990424) (thread 31972) [GC mark termination 53437762463500]
        0  0x00007ffdbb990424 in ???
            at ?:-1
            error: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.
        (truncated)
  Goroutine 21 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53435067473800]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 22 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53437762463500]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 23 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53437077056700]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 24 - User: :0 ??? (0x7ffdbb9907a4) (thread 24128) [GC mark termination]
        0  0x00007ffdbb9907a4 in ???
            at ?:-1
            error: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.
        (truncated)
  Goroutine 25 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53437717344800]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 26 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
[14 goroutines]
Go source
package main

import (
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"time"
)

func main() {
	/*for _, env := range os.Environ() {
		log.Println(env)
	}*/
	/*if runtime.GOOS == "windows" && os.Getenv("SteamClientLaunch") == "1" {
		runtime.GOMAXPROCS(max(1, min(2, runtime.NumCPU()-1)))
	}*/
	log.Println(runtime.Version())
	log.Printf("PID: %d", os.Getpid())
	go func() {
		for {
			_ = make([]byte, 256*1024)
			time.Sleep(time.Millisecond)
		}
	}()

	for {
		time.Sleep(time.Second)
		var gcStats debug.GCStats
		debug.ReadGCStats(&gcStats)
		log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
	}
}

@hajimehoshi
Copy link
Member Author

My guess based on the information provided would be that the threads are stuck in some GC-related thing (spinning up GC mark workers? that only happens on the first GC if GOMAXPROCS doesn't ever change) on a note (https://cs.opensource.google/go/go/+/master:src/runtime/os_windows.go;l=676;drc=4f881115d4067bda8a236aabcae8c41cdd13b4d0). I couldn't tell you why that would be a problem, since this happens all the time.

Adding println affects the behavior apparently, so I am not sure this was a correct observation, but I saw goroutines got stuck at the stdcall2 with _WaitForSingleObject.

Alternatively, the threads are going to sleep for STW, which is also based on a note (https://cs.opensource.google/go/go/+/master:src/runtime/proc.go;l=1612;drc=f025d19e7b3f0c66242760c213cc2b54cb100f69), and perhaps something is preventing the thread stopping the world from continuing promptly. Note that the thread which is stopping-the-world spins, sleeping in 100ms increments (but is awoken early when the last thread goes to sleep). So it might be that too. We could confirm/deny this by disabling the GC but calling runtime.ReadMemStats at some regular interval. This also happens all the time without issue, though, so I also don't really have a guess as to why this would be a problem running under Steam.

STW was invoked on GOMAXPROCS in my case, but this was not related to the freeze. Freezing happens regardless of STW.

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

package main

import (
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"time"
)

func main() {
	debug.SetGCPercent(-1)
	go func() {
		for {
			println("runtime.GC start")
			runtime.GC()
			println("runtime.GC end")
			time.Sleep(5 * time.Millisecond)
		}
	}()

	log.Println(runtime.Version())
	log.Printf("PID: %d", os.Getpid())
	for i := 0; i < 5; i++ {
		go func() {
			for {
				_ = make([]byte, 256*1024)
				time.Sleep(time.Millisecond)
			}
		}()
	}

	for {
		time.Sleep(time.Second)
		var gcStats debug.GCStats
		debug.ReadGCStats(&gcStats)
		log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
	}
}

The above program manually calls runtime.GC. I run it with the Steam client and found that the goroutine for GC sometimes stops at runtime.GC. runtime.GC sometimes took more than 10 seconds. I've added printlns in runtime.GC implementation and found the locations where the goroutine got stuck seemed inconsistent. I guess the Go scheduler sometimes failed to assign a thread to the goroutine. And in the worst case, all the goroutines suspended.

@prattmic
Copy link
Member

Thanks for the details.

In the x64dbg stack traces, it looks like every thread is blocked, which seems suspicious. This thread seems particularly suspicious to me:

12836                                                                                    
                    0000009586DFEE48 00007FFDB8D9BEAD 00007FFDBB9907A4 30         System ntdll.NtReleaseMutant+14
                    0000009586DFEE78 00007FFD45D11EC0 00007FFDB8D9BEAD 70         User   kernelbase.ReleaseMutex+D
                    0000009586DFEEE8 000000000083E029 00007FFD45D11EC0 170        User   gameoverlayrenderer64.OverlayHookD3D3+27380
                    0000009586DFF058 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009586DFF088 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    0000009586DFF0B0 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009586DFF0E8 0000000000801C56 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009586DFF100 0000000000802592 0000000000801C56 560        User   innovation2007_windows_amd64.0000000000801C56
                    0000009586DFF660 0000000000813D35 0000000000802592 18         User   innovation2007_windows_amd64.0000000000802592
                    0000009586DFF678 00000000007EDC96 0000000000813D35 30         User   innovation2007_windows_amd64.0000000000813D35
                    0000009586DFF6A8 00000000007F3825 00000000007EDC96 18         User   innovation2007_windows_amd64.00000000007EDC96
                    0000009586DFF6C0 00000000007EBC87 00000000007F3825 68         User   innovation2007_windows_amd64.00000000007F3825
                    0000009586DFF728 00000000007E843E 00000000007EBC87 50         User   innovation2007_windows_amd64.00000000007EBC87
                    0000009586DFF778 000000000083A869 00000000007E843E 2A7932E7B8 User   innovation2007_windows_amd64.00000000007E843E
                    000000C00012DF30 00000000007E80CC 000000000083A869 90         User   innovation2007_windows_amd64.000000000083A869
                    000000C00012DFC0 00000000007E7EA5 00000000007E80CC 18         User   innovation2007_windows_amd64.00000000007E80CC
                    000000C00012DFD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C00012DFE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821

@qmuntal is it normal for releasing a mutex to block? Or maybe this isn't actually blocked and we got lucky. @hajimehoshi could you collect another set of stack traces to see if they look similar?

It's unfortunate that we didn't get symbol names for the Go code itself. I tried building #71242 (comment) with Go 1.23.2, but the addresses don't even come close to matching up. @hajimehoshi could you try manually symbolizing some of these? I think go tool addr2line should do the trick. My guess is that the top few frames are runtime.semasleep, runtime.notesleep, runtime.mPark.

From the goroutine stacks, a GC is clearly running. A few goroutines are waiting to help out with GC assists (which is a bit odd). Two of them we can't see:

  Goroutine 20 - User: :0 ??? (0x7ffdbb990424) (thread 31972) [GC mark termination 53437762463500]
        0  0x00007ffdbb990424 in ???
            at ?:-1
            error: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.
        (truncated)

  Goroutine 24 - User: :0 ??? (0x7ffdbb9907a4) (thread 24128) [GC mark termination]
        0  0x00007ffdbb9907a4 in ???
            at ?:-1
            error: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.
        (truncated)

These are perhaps stuck in GC mark termination, maybe in here?

@hajimehoshi
Copy link
Member Author

could you collect another set of stack traces to see if they look similar?

Go program
Stack traces by x64dbg
Thread ID          Address          To               From             Size       Party  Comment                                      
33152                                                                                   
                   00000072E57FF0A8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E57FF148 00007FFD949F1D4F 00007FFDB8D99CEE 70         User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E57FF1B8 00000000003DE029 00007FFD949F1D4F 170        User   gameoverlayrenderer64.OverlayHookD3D3+2720F
                   00000072E57FF328 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E57FF358 00000000003DC4C1 00000000003DC40D 28         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E57FF380 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E57FF3B8 00000000003A1C56 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E57FF3D0 00000000003A2592 00000000003A1C56 560        User   innovation2007_windows_amd64.00000000003A1C56
                   00000072E57FF930 00000000003B3D35 00000000003A2592 18         User   innovation2007_windows_amd64.00000000003A2592
                   00000072E57FF948 000000000038DC96 00000000003B3D35 30         User   innovation2007_windows_amd64.00000000003B3D35
                   00000072E57FF978 0000000000393825 000000000038DC96 18         User   innovation2007_windows_amd64.000000000038DC96
                   00000072E57FF990 000000000038BC87 0000000000393825 68         User   innovation2007_windows_amd64.0000000000393825
                   00000072E57FF9F8 0000000000388407 000000000038BC87 50         User   innovation2007_windows_amd64.000000000038BC87
                   00000072E57FFA48 00000000003DA869 0000000000388407 4D1ACD64E8 User   innovation2007_windows_amd64.0000000000388407
                   000000C0004D5F30 00000000003880CC 00000000003DA869 90         User   innovation2007_windows_amd64.00000000003DA869
                   000000C0004D5FC0 0000000000387EA5 00000000003880CC 18         User   innovation2007_windows_amd64.00000000003880CC
                   000000C0004D5FD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004D5FE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
26976                                                                                   
                   00000072E49FFAB8 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                   00000072E49FFD98 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                   00000072E49FFDC8 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                   00000072E49FFE48 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
9184 - Main Thread                                                                      
                   00000072E47FF928 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E47FF9C8 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E47FFB38 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E47FFB68 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E47FFB88 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E47FFBC0 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E47FFBD8 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E47FFC40 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E47FFC78 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E47FFCA8 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E47FFE20 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E47FFE58 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E47FFEB0 00000000003DA7F3 00000000003AEDCB 4D1BCD8060 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C0004D7F10 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C0004D7F30 0000000000387FC9 00000000003D572E 90         User   innovation2007_windows_amd64.00000000003D572E
                   000000C0004D7FC0 0000000000387EA5 0000000000387FC9 18         User   innovation2007_windows_amd64.0000000000387FC9
                   000000C0004D7FD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004D7FE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
7300                                                                                    
                   00000072E4BFF678 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                   00000072E4BFF958 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                   00000072E4BFF988 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                   00000072E4BFFA08 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
34092                                                                                   
                   00000072E55FF078 00007FFDB8D9BEAD 00007FFDBB9907A4 30         System ntdll.NtReleaseMutant+14
                   00000072E55FF0A8 00007FFD949F1EC0 00007FFDB8D9BEAD 70         User   kernelbase.ReleaseMutex+D
                   00000072E55FF118 00000000003DE029 00007FFD949F1EC0 170        User   gameoverlayrenderer64.OverlayHookD3D3+27380
                   00000072E55FF288 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E55FF2B8 00000000003DC4C1 00000000003DC40D 28         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E55FF2E0 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E55FF318 00000000003A1C56 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E55FF330 00000000003A2592 00000000003A1C56 560        User   innovation2007_windows_amd64.00000000003A1C56
                   00000072E55FF890 00000000003B3D35 00000000003A2592 18         User   innovation2007_windows_amd64.00000000003A2592
                   00000072E55FF8A8 000000000038DC96 00000000003B3D35 30         User   innovation2007_windows_amd64.00000000003B3D35
                   00000072E55FF8D8 0000000000393825 000000000038DC96 18         User   innovation2007_windows_amd64.000000000038DC96
                   00000072E55FF8F0 000000000038BC87 0000000000393825 68         User   innovation2007_windows_amd64.0000000000393825
                   00000072E55FF958 0000000000388407 000000000038BC87 50         User   innovation2007_windows_amd64.000000000038BC87
                   00000072E55FF9A8 00000000003DA869 0000000000388407 4D1AEDC588 User   innovation2007_windows_amd64.0000000000388407
                   000000C0004DBF30 00000000003880CC 00000000003DA869 90         User   innovation2007_windows_amd64.00000000003DA869
                   000000C0004DBFC0 0000000000387EA5 00000000003880CC 18         User   innovation2007_windows_amd64.00000000003880CC
                   000000C0004DBFD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004DBFE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
29616                                                                                   
                   00000072E4FFEC88 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E4FFED28 00007FFD949F1D4F 00007FFDB8D99CEE 70         User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E4FFED98 00000000003DE029 00007FFD949F1D4F 170        User   gameoverlayrenderer64.OverlayHookD3D3+2720F
                   00000072E4FFEF08 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E4FFEF38 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E4FFEF58 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E4FFEF90 00000000003A1C56 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E4FFEFA8 00000000003A2592 00000000003A1C56 560        User   innovation2007_windows_amd64.00000000003A1C56
                   00000072E4FFF508 00000000003B3D35 00000000003A2592 18         User   innovation2007_windows_amd64.00000000003A2592
                   00000072E4FFF520 00000000003B3A36 00000000003B3D35 60         User   innovation2007_windows_amd64.00000000003B3D35
                   00000072E4FFF580 00000000003B379A 00000000003B3A36 80         User   innovation2007_windows_amd64.00000000003B3A36
                   00000072E4FFF600 00000000003AA73D 00000000003B379A 28         User   innovation2007_windows_amd64.00000000003B379A
                   00000072E4FFF628 00000000003AA68A 00000000003AA73D 28         User   innovation2007_windows_amd64.00000000003AA73D
                   00000072E4FFF650 00000000003DA765 00000000003AA68A 8          User   innovation2007_windows_amd64.00000000003AA68A
                   00000072E4FFF658 00000000003DE477 00000000003DA765 8          User   innovation2007_windows_amd64.00000000003DA765
                   00000072E4FFF660 0000000000000000 00000000003DE477            User   innovation2007_windows_amd64.00000000003DE477
28532                                                                                   
                   00000072E51FF6F8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E51FF798 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E51FF908 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E51FF938 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E51FF958 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E51FF990 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E51FF9A8 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E51FFA10 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E51FFA48 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E51FFA78 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E51FFBF0 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E51FFC28 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E51FFC80 00000000003DA7F3 00000000003AEDCB 4D1AEC0188 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C0000BFE08 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C0000BFE28 000000000038AF67 00000000003D572E 38         User   innovation2007_windows_amd64.00000000003D572E
                   000000C0000BFE60 000000000038A88A 000000000038AF67 60         User   innovation2007_windows_amd64.000000000038AF67
                   000000C0000BFEC0 000000000037CDF4 000000000038A88A 28         User   innovation2007_windows_amd64.000000000038A88A
                   000000C0000BFEE8 00000000003D14DE 000000000037CDF4 A0         User   innovation2007_windows_amd64.000000000037CDF4
                   000000C0000BFF88 00000000003D6609 00000000003D14DE 28         User   innovation2007_windows_amd64.00000000003D14DE
                   000000C0000BFFB0 00000000004184E5 00000000003D6609 28         User   innovation2007_windows_amd64.00000000003D6609
                   000000C0000BFFD8 00000000003DC821 00000000004184E5 8          User   innovation2007_windows_amd64.00000000004184E5
                   000000C0000BFFE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
35804                                                                                   
                   00000072E53FF7E8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E53FF888 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E53FF9F8 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E53FFA28 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E53FFA48 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E53FFA80 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E53FFA98 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E53FFB00 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E53FFB38 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E53FFB68 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E53FFCE0 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E53FFD18 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E53FFD70 00000000003DA7F3 00000000003AEDCB 4D1B0DA1A0 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C0004D9F10 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C0004D9F30 0000000000387FC9 00000000003D572E 90         User   innovation2007_windows_amd64.00000000003D572E
                   000000C0004D9FC0 0000000000387EA5 0000000000387FC9 18         User   innovation2007_windows_amd64.0000000000387FC9
                   000000C0004D9FD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004D9FE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
34976                                                                                   
                   00000072E4DFF768 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                   00000072E4DFFA48 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                   00000072E4DFFA78 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                   00000072E4DFFAF8 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
17840                                                                                   
                   00000072E59FF410 00000000003A5B1E 00000000003DC3C6 78         User   innovation2007_windows_amd64.00000000003DC3C6
                   00000072E59FF488 000000000038A0A5 00000000003A5B1E 50         User   innovation2007_windows_amd64.00000000003A5B1E
                   00000072E59FF4D8 0000000000389D65 000000000038A0A5 A8         User   innovation2007_windows_amd64.000000000038A0A5
                   00000072E59FF580 000000000038BF14 0000000000389D65 68         User   innovation2007_windows_amd64.0000000000389D65
                   00000072E59FF5E8 000000000038843E 000000000038BF14 50         User   innovation2007_windows_amd64.000000000038BF14
                   00000072E59FF638 00000000003DA869 000000000038843E 4D1AACA8F8 User   innovation2007_windows_amd64.000000000038843E
                   000000C0004C9F30 00000000003880CC 00000000003DA869 90         User   innovation2007_windows_amd64.00000000003DA869
                   000000C0004C9FC0 0000000000387EA5 00000000003880CC 18         User   innovation2007_windows_amd64.00000000003880CC
                   000000C0004C9FD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004C9FE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
35732                                                                                   
                   00000072E5BFF3B8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E5BFF458 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E5BFF5C8 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E5BFF5F8 00000000003DC4C1 00000000003DC40D 28         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E5BFF620 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E5BFF658 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E5BFF670 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E5BFF6D8 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E5BFF710 00000000003ABD51 000000000037B9D2 20         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E5BFF730 00000000003AA73D 00000000003ABD51 28         User   innovation2007_windows_amd64.00000000003ABD51
                   00000072E5BFF758 00000000003AA68A 00000000003AA73D 28         User   innovation2007_windows_amd64.00000000003AA73D
                   00000072E5BFF780 00000000003DA765 00000000003AA68A 8          User   innovation2007_windows_amd64.00000000003AA68A
                   00000072E5BFF788 00000000003DE477 00000000003DA765 8          User   innovation2007_windows_amd64.00000000003DA765
                   00000072E5BFF790 0000000000000000 00000000003DE477            User   innovation2007_windows_amd64.00000000003DE477
11764                                                                                   
                   00000072E5DFF608 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E5DFF6A8 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E5DFF818 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E5DFF848 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E5DFF868 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E5DFF8A0 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E5DFF8B8 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E5DFF920 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E5DFF958 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E5DFF988 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E5DFFB00 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E5DFFB38 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E5DFFB90 00000000003DA7F3 00000000003AEDCB 4D1A27C150 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C00007BCE0 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C00007BD00 000000000038AF67 00000000003D572E 38         User   innovation2007_windows_amd64.00000000003D572E
                   000000C00007BD38 000000000038A88A 000000000038AF67 60         User   innovation2007_windows_amd64.000000000038AF67
                   000000C00007BD98 000000000037CDF4 000000000038A88A 28         User   innovation2007_windows_amd64.000000000038A88A
                   000000C00007BDC0 00000000003D14DE 000000000037CDF4 A0         User   innovation2007_windows_amd64.000000000037CDF4
                   000000C00007BE60 000000000037CEE5 00000000003D14DE 28         User   innovation2007_windows_amd64.00000000003D14DE
                   000000C00007BE88 000000000041827B 000000000037CEE5 C0         User   innovation2007_windows_amd64.000000000037CEE5
                   000000C00007BF48 00000000003A737D 000000000041827B 90         User   innovation2007_windows_amd64.000000000041827B
                   000000C00007BFD8 00000000003DC821 00000000003A737D 8          User   innovation2007_windows_amd64.00000000003A737D
                   000000C00007BFE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
27512                                                                                   
                   00000072E5FFF538 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E5FFF5D8 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E5FFF748 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E5FFF778 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E5FFF798 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E5FFF7D0 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E5FFF7E8 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E5FFF850 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E5FFF888 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E5FFF8B8 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E5FFFA30 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E5FFFA68 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E5FFFAC0 00000000003DA7F3 00000000003AEDCB 4D1A0BC348 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C0000BBE08 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C0000BBE28 000000000038AF67 00000000003D572E 38         User   innovation2007_windows_amd64.00000000003D572E
                   000000C0000BBE60 000000000038A88A 000000000038AF67 60         User   innovation2007_windows_amd64.000000000038AF67
                   000000C0000BBEC0 000000000037CDF4 000000000038A88A 28         User   innovation2007_windows_amd64.000000000038A88A
                   000000C0000BBEE8 00000000003D14DE 000000000037CDF4 A0         User   innovation2007_windows_amd64.000000000037CDF4
                   000000C0000BBF88 00000000003D6609 00000000003D14DE 28         User   innovation2007_windows_amd64.00000000003D14DE
                   000000C0000BBFB0 00000000004184E5 00000000003D6609 28         User   innovation2007_windows_amd64.00000000003D6609
                   000000C0000BBFD8 00000000003DC821 00000000004184E5 8          User   innovation2007_windows_amd64.00000000004184E5
                   000000C0000BBFE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821

I'll try addr2line soon.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
BugReport Issues describing a possible bug in the Go implementation. compiler/runtime Issues related to the Go compiler and/or runtime. NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. OS-Windows
Projects
None yet
Development

No branches or pull requests

6 participants