Software Diagnostics Library

Sometimes, when we have One-Thread Process memory dumps, it is possible to get other stack regions indirectly through the analysis of virtual memory regions. Consider, for example, this dump that has only one process exit thread:

0:000> kL
# Child-SP          RetAddr           Call Site
00 000000f1`828ff848 00007ff9`d29aa9b8 ntdll!NtTerminateProcess+0x14
01 000000f1`828ff850 00007ff9`d113cd8a ntdll!RtlExitUserProcess+0xb8
02 000000f1`828ff880 00007ff7`fbb91231 kernel32!ExitProcessImplementation+0xa
03 000000f1`828ff8b0 00007ff7`fbb9125f HiddenStack!bar1+0x41
04 000000f1`828ffa80 00007ff7`fbb91cb5 HiddenStack!foo1+0x1f
05 000000f1`828ffc40 00007ff7`fbb91b1b HiddenStack!std::_Invoker_functor::_Call<void (__cdecl*)(void)>+0x15
06 000000f1`828ffc70 00007ff7`fbb917c4 HiddenStack!std::invoke<void (__cdecl*)(void)>+0x1b
07 000000f1`828ffca0 00007ff7`fbb99728 HiddenStack!std::thread::_Invoke<std::tuple<void (__cdecl*)(void)>,0>+0x64
08 000000f1`828ffcf0 00007ff9`d1137bd4 HiddenStack!thread_start<unsigned int (__cdecl*)(void *),1>+0x50
09 000000f1`828ffd20 00007ff9`d29aced1 kernel32!BaseThreadInitThunk+0x14
0a 000000f1`828ffd50 00000000`00000000 ntdll!RtlUserThreadStart+0x21

There are no more thread stack traces:

0:000> ~
. 0 Id: 27d4.22a4 Suspend: -1 Teb: 000000f1`8266a000 Unfrozen

However, in addition to thread #0, we can find several regions having PAGE_GUARD protection:

0:000> !address
[...]
+       f1`82800000       f1`828fb000        0`000fb000 MEM_PRIVATE MEM_RESERVE                                    Stack      [~0; 27d4.22a4]
f1`828fb000       f1`828fe000        0`00003000 MEM_PRIVATE MEM_COMMIT  PAGE_READWRITE|PAGE_GUARD          Stack      [~0; 27d4.22a4]
f1`828fe000       f1`82900000        0`00002000 MEM_PRIVATE MEM_COMMIT  PAGE_READWRITE                     Stack      [~0; 27d4.22a4]
+       f1`82900000       f1`829fb000        0`000fb000 MEM_PRIVATE MEM_RESERVE                                    <unknown>
f1`829fb000       f1`829fe000        0`00003000 MEM_PRIVATE MEM_COMMIT  PAGE_READWRITE|PAGE_GUARD          <unknown>
f1`829fe000       f1`82a00000        0`00002000 MEM_PRIVATE MEM_COMMIT  PAGE_READWRITE                     <unknown>  […………….]
+       f1`82a00000       f1`82afc000        0`000fc000 MEM_PRIVATE MEM_RESERVE                                    <unknown>
f1`82afc000       f1`82aff000        0`00003000 MEM_PRIVATE MEM_COMMIT  PAGE_READWRITE|PAGE_GUARD          <unknown>
f1`82aff000       f1`82b00000        0`00001000 MEM_PRIVATE MEM_COMMIT  PAGE_READWRITE                     <unknown>  […………….]
+       f1`82b00000       f1`82bfb000        0`000fb000 MEM_PRIVATE MEM_RESERVE                                    <unknown>
f1`82bfb000       f1`82bfe000        0`00003000 MEM_PRIVATE MEM_COMMIT  PAGE_READWRITE|PAGE_GUARD          <unknown>
f1`82bfe000       f1`82c00000        0`00002000 MEM_PRIVATE MEM_COMMIT  PAGE_READWRITE                     <unknown>  […………….]
+       f1`82c00000      1fe`828f0000      10c`ffcf0000             MEM_FREE    PAGE_NOACCESS                      Free
[…]

We then can get Rough Stack Traces out of them:

0:000> .lines -d
Line number information will not be loaded

0:000> dpS f1`829fe000       f1`82a00000
00007ff9`d2986139 ntdll!RtlpFindEntry+0×4d
00007ff9`d297dbea ntdll!RtlpAllocateHeap+0xcfa
00007ff9`d297babb ntdll!RtlpAllocateHeapInternal+0×1cb
00007ff9`d297babb ntdll!RtlpAllocateHeapInternal+0×1cb
00007ff9`d2a463fa ntdll!RtlpValidateHeap+0×32
00007ff9`d2a44b25 ntdll!RtlDebugAllocateHeap+0×35d
00007ff9`d29f49d6 ntdll!RtlpAllocateHeap+0×77ae6
00007ff9`d29f49d6 ntdll!RtlpAllocateHeap+0×77ae6
00007ff9`d0070000 KERNELBASE!UrlHashW <PERF> (KERNELBASE+0×0)
00007ff9`d007b4b1 KERNELBASE!SetTEBLangID+0×2d
00007ff9`d007ac70 KERNELBASE!_KernelBaseBaseDllInitialize+0×90
00007ff9`d297babb ntdll!RtlpAllocateHeapInternal+0×1cb
00007ff9`d297babb ntdll!RtlpAllocateHeapInternal+0×1cb
00007ff9`d19e7890 msvcrt!CRTDLL_INIT
00007ff9`d29db5a3 ntdll!RTL_BINARY_ARRAY<RTLP_FLS_SLOT,8,4>::ChunkAllocate+0×67
00007ff9`d19e0000 msvcrt!`dynamic initializer for ‘__ExceptionPtr::m_badAllocExceptionPtr” <PERF> (msvcrt+0×0)
00007ff9`d29db65d ntdll!RTL_BINARY_ARRAY<RTLP_FLS_SLOT,8,4>::SetValue+0×39
00007ff9`d2964ef7 ntdll!RtlDeactivateActivationContextUnsafeFast+0xc7
00007ff9`d299439c ntdll!RtlFlsSetValue+0xec
00007ff9`d2986139 ntdll!RtlpFindEntry+0×4d
00000000`7ffe0301 SharedUserData+0×301
00007ff9`d297dbea ntdll!RtlpAllocateHeap+0xcfa
00000000`7ffe0358 SharedUserData+0×358
00007ff7`fbb923ca HiddenStack!std::chrono::duration_cast<std::chrono::duration<double,
std::ratio<1,1000000000> >,__int64,std::ratio<1,1000000000>,void>+0×4a
00000000`7ffe0358 SharedUserData+0×358
00007ff9`d294bb47 ntdll!RtlGetSystemTimePrecise+0×57
00007ff9`d00b6931 KERNELBASE!SleepEx+0xa1
00007ff9`d00d3890 KERNELBASE!GetSystemTimePreciseAsFileTime+0×10
00007ff7`fbb931b4 HiddenStack!_Thrd_sleep+0×3c
00007ff7`fbb916c5 HiddenStack!std::this_thread::sleep_until<std::chrono::steady_clock,
std::chrono::duration<__int64,std::ratio<1,1000000000> > >+0×65
00007ff7`fbb91651 HiddenStack!std::chrono::operator+<std::chrono::steady_clock,
std::chrono::duration<__int64,std::ratio<1,1000000000> >,__int64,std::ratio<1,1> >+0×41
00007ff7`fbb913fd HiddenStack!std::this_thread::sleep_for<__int64,std::ratio<1,1> >+0×2d
00007ff7`fbb912a9 HiddenStack!bar2+0×39
00007ff7`fbb912df HiddenStack!foo2+0×1f
00007ff7`fbb91cb5 HiddenStack!std::_Invoker_functor::_Call<void (__cdecl*)(void)>+0×15
00007ff7`fbb91aec HiddenStack!std::unique_ptr<std::tuple<void (__cdecl*)(void)>,std::default_delete<std::tuple<void (__cdecl*)(void)> > >::unique_ptr<std::tuple<void (__cdecl*)(void)>,std::default_delete<std::tuple<void (__cdecl*)(void)> > ><std::default_delete<std::tuple<void (__cdecl*)(void)> >,0>+0×2c
00007ff7`fbb91b1b HiddenStack!std::invoke<void (__cdecl*)(void)>+0×1b
00007ff7`fbb917c4 HiddenStack!std::thread::_Invoke<std::tuple<void (__cdecl*)(void)>,0>+0×64
00007ff7`fbb9c1d7 HiddenStack!__acrt_getptd+0xb3
00007ff7`fbb99728 HiddenStack!thread_start<unsigned int (__cdecl*)(void *),1>+0×50
00007ff9`d1137bd4 kernel32!BaseThreadInitThunk+0×14
00007ff9`d29aced1 ntdll!RtlUserThreadStart+0×21

0:000> dpS f1`82aff000       f1`82b00000
00007ff9`d2986139 ntdll!RtlpFindEntry+0×4d
00007ff9`d297dbea ntdll!RtlpAllocateHeap+0xcfa
00007ff9`d297dbea ntdll!RtlpAllocateHeap+0xcfa
00007ff9`d297babb ntdll!RtlpAllocateHeapInternal+0×1cb
00007ff9`d2a463fa ntdll!RtlpValidateHeap+0×32
00007ff9`d2a463fa ntdll!RtlpValidateHeap+0×32
00007ff9`d2a44b25 ntdll!RtlDebugAllocateHeap+0×35d
00007ff9`d29f49d6 ntdll!RtlpAllocateHeap+0×77ae6
00007ff9`d2962da8 ntdll!LdrpInitializeThread+0×40
00007ff9`d297562f ntdll!TppCallbackCheckThreadAfterCallback+0×9f
00007ff9`d29700e5 ntdll!RtlRegisterThreadWithCsrss+0×35
00007ff9`d29b18f5 ntdll!_LdrpInitialize+0×89
00007ff9`d2975394 ntdll!TppCallbackEpilog+0×144
00007ff9`d29701d6 ntdll!TppCritSetThread+0×7a
00007ff9`d2973155 ntdll!TppWorkCallbackPrologRelease+0×1c9
00007ff9`d296e2c3 ntdll!LdrpWorkCallback+0×63
00007ff9`d2aa52f0 ntdll!LdrpWorkQueue
00007ff9`d29708a2 ntdll!TppWorkpExecuteCallback+0xb2
00000000`7ffe0386 SharedUserData+0×386
00007ff9`d2974060 ntdll!TppWorkerThread+0×300
00007ff9`d1137bd4 kernel32!BaseThreadInitThunk+0×14
00007ff9`d29aced1 ntdll!RtlUserThreadStart+0×21

0:000> dpS f1`82bfe000       f1`82c00000
00007ff9`d2986139 ntdll!RtlpFindEntry+0×4d
00007ff9`d297dbea ntdll!RtlpAllocateHeap+0xcfa
00007ff9`d297dbea ntdll!RtlpAllocateHeap+0xcfa
00007ff9`d297babb ntdll!RtlpAllocateHeapInternal+0×1cb
00007ff9`d2a463fa ntdll!RtlpValidateHeap+0×32
00007ff9`d2a463fa ntdll!RtlpValidateHeap+0×32
00007ff9`d2a44b25 ntdll!RtlDebugAllocateHeap+0×35d
00007ff9`d29f49d6 ntdll!RtlpAllocateHeap+0×77ae6
00007ff9`d2962da8 ntdll!LdrpInitializeThread+0×40
00007ff9`d29700e5 ntdll!RtlRegisterThreadWithCsrss+0×35
00007ff9`d29b18f5 ntdll!_LdrpInitialize+0×89
00007ff9`d297babb ntdll!RtlpAllocateHeapInternal+0×1cb
00007ff9`d29701d6 ntdll!TppCritSetThread+0×7a
00007ff9`d2970098 ntdll!TppPoolAddWorker+0×68
00007ff9`d2974060 ntdll!TppWorkerThread+0×300
00007ff9`d1137bd4 kernel32!BaseThreadInitThunk+0×14
00007ff9`d29aced1 ntdll!RtlUserThreadStart+0×21

We call such analysis pattern Hidden Stack as another way to get Historical Information from memory dumps.

The example memory dump, the application PDB file, and source code can be downloaded from here.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Full symbol files usually include source code information for UI debugging. Debuggers, like WinDbg, may show it for each relevant stack trace frame:

0:001> k
# Child-SP          RetAddr           Call Site
00 00000074`fe5fe3c8 00007ff9`72937ff7 ntdll!NtWaitForMultipleObjects+0x14
01 00000074`fe5fe3d0 00007ff9`72937ede KERNELBASE!WaitForMultipleObjectsEx+0x107
02 00000074`fe5fe6d0 00007ff9`74a671fb KERNELBASE!WaitForMultipleObjects+0xe
03 00000074`fe5fe710 00007ff9`74a66ca8 kernel32!WerpReportFaultInternal+0x51b
04 00000074`fe5fe830 00007ff9`729df848 kernel32!WerpReportFault+0xac
05 00000074`fe5fe870 00007ff9`75304af2 KERNELBASE!UnhandledExceptionFilter+0x3b8
06 00000074`fe5fe990 00007ff9`752ec6d6 ntdll!RtlUserThreadStart$filt$0+0xa2
07 00000074`fe5fe9d0 00007ff9`753011ff ntdll!_C_specific_handler+0x96
08 00000074`fe5fea40 00007ff9`752ca289 ntdll!RtlpExecuteHandlerForException+0xf
09 00000074`fe5fea70 00007ff9`752ffe6e ntdll!RtlDispatchException+0x219
0a 00000074`fe5ff180 00007ff7`bbd51cb0 ntdll!KiUserExceptionDispatch+0x2e
0b 00000074`fe5ff930 00007ff7`bbd51983 SourceStackTrace!std::_Deque_unchecked_const_iterator<std::_Deque_val< std::_Deque_simple_types<int> > >::operator*+0x50 [C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Tools\MSVC\14.23.28105\include\deque @ 55]
0c 00000074`fe5ff970 00007ff7`bbd51883 SourceStackTrace!std::_Deque_unchecked_iterator<std::_Deque_val< std::_Deque_simple_types<int> > >::operator*+0×13 [C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Tools\MSVC\14.23.28105\include\deque @ 175]
0d 00000074`fe5ff9a0 00007ff7`bbd515f6 SourceStackTrace!std::deque<int,std::allocator<int> >::back+0×33 [C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Tools\MSVC\14.23.28105\include\deque @ 1091]
0e 00000074`fe5ff9f0 00007ff7`bbd5156c SourceStackTrace!std::stack<int,std::deque<int,std::allocator<int> > >::top+0×16 [C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Tools\MSVC\14.23.28105\include\stack @ 66]
0f 00000074`fe5ffa20 00007ff7`bbd521a3 SourceStackTrace!Info::gather+0×3c [C:\NewWork\SourceStackTrace\InfoGather.cpp @ 9]
10 00000074`fe5ffa90 00007ff7`bbd5108d SourceStackTrace!Info::scatter+0×13 [C:\NewWork\SourceStackTrace\InfoScatter.cpp @ 6]
11 00000074`fe5ffac0 00007ff7`bbd510a9 SourceStackTrace!worker_bar+0×1d [C:\NewWork\SourceStackTrace\bar.cpp @ 10]
12 00000074`fe5ffb00 00007ff7`bbd522b9 SourceStackTrace!worker_foo+0×9 [C:\NewWork\SourceStackTrace\foo.cpp @ 6]
13 00000074`fe5ffb30 00007ff7`bbd526b5 SourceStackTrace!worker+0×9 [C:\NewWork\SourceStackTrace\SourceStackTrace.cpp @ 8]
14 00000074`fe5ffb60 00007ff7`bbd5269b SourceStackTrace!std::_Invoker_functor::_Call<void (__cdecl*)(void)>+0×15 [C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Tools\MSVC\14.23.28105\include\type_traits @ 1571]
15 00000074`fe5ffb90 00007ff7`bbd525d4 SourceStackTrace!std::invoke<void (__cdecl*)(void)>+0×1b [C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Tools\MSVC\14.23.28105\include\type_traits @ 1571]
16 00000074`fe5ffbc0 00007ff7`bbd59784 SourceStackTrace!std::thread::_Invoke<std::tuple<void (__cdecl*)(void)>,0>+0×64 [C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC\Tools\MSVC\14.23.28105\include\thread @ 40]
17 00000074`fe5ffc10 00007ff9`74a17bd4 SourceStackTrace!thread_start<unsigned int (__cdecl*)(void *),1>+0×50 [minkernel\crts\ucrt\src\appcrt\startup\thread.cpp @ 97]
18 00000074`fe5ffc40 00007ff9`752cced1 kernel32!BaseThreadInitThunk+0×14
19 00000074`fe5ffc70 00000000`00000000 ntdll!RtlUserThreadStart+0×21

Such Source Stack Trace frames has a similar structure as native module!function+offset format and can be mined as well including most of traditional stack trace patterns:

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Raw stack memory region analysis is more productive with Region Clusters analysis pattern. Here we apply simple clustering techniques to organize various region values into disjoint sets with chosen semantics. For our purposes simple sort suffices to generate such clusters that can be visually inspected. We take the same stack.csv file from Region Profile analysis pattern. It’s values are sorted and the results are shown in sorted order with corresponding count of occurrences and symbolic references (we use the internal version of Narrascope written in C++, a narrative debugger, but you can use your favorite scripting language here):

0 count: 13718
1 count: 273
2 count: 23
3 count: 22
4 count: 28
5 count: 9
6 count: 5
7 count: 18
8 count: 35
9 count: 5
a count: 24
b count: 12
c count: 4
d count: 3
e count: 1
f count: 28
10 count: 14
...
c0000034 count: 2
c0000388 count: 2
c01c0001 count: 1
c0a70000 count: 1
d0908070 count: 1
dcae0fa0 count: 1
e30000e3 count: 1
f80004fc count: 2
ffff5815 count: 2
fffffed3 count: 2
fffffffd count: 2
ffffffff count: 18
100000000 count: 6
100000001 count: 4
100001f80 count: 1
100001fa0 count: 16
100001fa4 count: 2
100003033 count: 2
100010000 count: 1
...
7ff700000000 count: 1
7ff700000001 count: 2
7ff70000000d count: 1
7ff747390000 Photoshop_exe count: 1
7ff74ebd4ec0 Photoshop_exe+0x7844ec0 count: 1
7ff74ef351c7 Photoshop_exe+0x7ba51c7 count: 1
7ff74ef4e2f0 Photoshop_exe+0x7bbe2f0 count: 1
7ff74ef4e5a9 Photoshop_exe+0x7bbe5a9 count: 1
...
7fff00000000 count: 21
7fff00000001 count: 7
7fff00000002 count: 1
7fff00000003 count: 1
7fff00000004 count: 1
7fff00000011 count: 1
7fff00000020 count: 1
7fff00000040 count: 3
7fff00000102 count: 1
7fff0000029e count: 3
7fff00140000 count: 1
7fff02000002 count: 1
7fff4782c33b libcef!GetHandleVerifier+0x61d7b count: 1
7fff4782c884 libcef!GetHandleVerifier+0x622c4 count: 1
7fff493749cc libcef!cef_time_to_timet+0x1a9228 count: 2
...
7fff9a0c1e57 GdiPlus!GpGraphics::MeasureString+0x333 count: 1
7fff9a128c2a GdiPlus!FastTextImager::MeasureString+0x32 count: 1
7fff9a174e18 GdiPlus!GpFontFamily::vftable' count: 2
7fff9b6055b3 DWrite!FontFace::GetDesignGlyphAdvances+0x57 count: 1
7fffa7e6c260 comctl32!ListBox_WndProc count: 5
7fffa7e6c357 comctl32!ListBox_WndProc+0xf7 count: 2
7fffb1373c18 npmproxy!INotifyNetworkListManagerEventsProxyVtbl+0x1b8 count: 1
7fffb2c14e96 msvcp140!_Mbrtowc+0x66 [f:\dd\vctools\crt\crtw32\stdcpp\xmbtowc.c @ 156] count: 1
...
7fffc09f0359 ntdll!qsort+0x379 count: 1
7fffc09fa1e4 ntdll!woutput_s+0x8e8 count: 1
7fffc09fa297 ntdll!write_string+0x3f count: 1
7fffc09fbd30 ntdll!NtdllDefWindowProc_W count: 2
7fffc09fbf10 ntdll!NtdllDispatchHook_W count: 2
7fffc09ffc54 ntdll!KiUserCallForwarder+0x24 count: 1
7fffc09ffdb4 ntdll!KiUserCallbackDispatcherContinue count: 2
800000000000 count: 1
800000000001 count: 2
800063640000 count: 36
800066660000 count: 38
80006f6f0000 count: 2
800072720000 count: 8
800075750000 count: 1
974b00000000 count: 1
974b8118d10d count: 1
a76b00000000 count: 1
a76bb8365307 count: 1
a76bb8378c47 count: 1
a76bb8378f77 count: 1
a76bb837bfd7 count: 1
a8c300000000 count: 1
a8c311cf265f count: 1
...
30000000000000 count: 1
30000000310030 count: 1
30000300470048 count: 1
30002000100000 count: 1
3000300030007b count: 1
3000300031002d count: 1
30003000310031 count: 2
300031002d0037 count: 1
30003800390032 count: 3
31000000000000 count: 1
310000007d0036 count: 1
31002d00310037 count: 1
310032002d0035 count: 1
...
7fdf7fbd7f9c7f7b count: 2
8000800000000001 count: 1
8000800000001fa0 count: 1
8000800080000000 count: 6
8000800080008000 count: 52
80121a254b25250a count: 1
923800003f000000 count: 2
bf000000bf000000 count: 1
bff0000000000000 count: 2
e5b2a56118358cbe count: 2
ffff0072656c6c6f count: 1
fffffdb773438b57 count: 3
ffffff0000000005 count: 1
ffffff7bc010786f count: 1
ffffff7bc010787f count: 1
fffffffb00000000 count: 1
ffffffff00000000 count: 4
ffffffff00000001 count: 3
ffffffff00000005 count: 1
ffffffff00001fa0 count: 2
ffffffff4c494146 count: 2
ffffffffffffc3ce count: 1
fffffffffffffef6 count: 1
ffffffffffffff00 count: 2
ffffffffffffff01 count: 2
fffffffffffffffe count: 166
ffffffffffffffff count: 38

We can easily identify error values, module boundaries, and Regular Data. The sorting can also be done for double word or word values, for example to isolate errors or wide character values, but this will have to be seen whether it is useful.

This clustering approach can be depicted in the following idealized diagram:

The full output can be found here: stack-clusters.txt for stack.csv file.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Manual analysis of Execution Residue in stack regions can be quite daunting so some basic statistics on the distribution of address, value, and symbolic information can be useful. To automate this process we use Pandas python library and interpret preprocessed WinDbg output of dps and dpp commands as DataFrame object:

import pandas as pd
import pandas_profiling

df = pd.read_csv("stack.csv")
html_file = open("stack.html", "w")
html_file.write (pandas_profiling.ProfileReport(df).to_html())
html_file.close()

We get a rudimentary profile report: stack.html for stack.csv file. The same was also done for Address, Address/Value, Value, Symbol output of dpp command: stack4columns.html for stack4columns.csv file.

We call this analysis pattern Region Profile since any Memory Region can be used. This analysis pattern is not limited to Python/Pandas and different libraries/scripts/script languages can also be used for statistical and correlational profiling. We plan to provide more examples here in the future.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Sometimes we have complex plugins or subsystems consisting from several modules that are loaded in the same process address space (and this is also possible in kernel space as well). Usually subsystem components reside in the same persistent folder (including its possible subfolders):

0:000> lmf
start             end                 module name
...
00007fff`46ee0000 00007fff`4cde8000 libcef C:\Program Files\Adobe\Adobe Photoshop CC 2019\Required\Plug-ins\Spaces\libcef.dll
…
00007fff`7fa40000 00007fff`7fbc2000 Spaces C:\Program Files\Adobe\Adobe Photoshop CC 2019\Required\Plug-ins\Spaces\Spaces.8li
…
00007fff`8ba50000 00007fff`8bae3000 chrome_elf C:\Program Files\Adobe\Adobe Photoshop CC 2019\Required\Plug-ins\Spaces\chrome_elf.dll
…

When we know product architecture we can group modules according to the known design not only by their physical locations.

This analysis pattern, that we call Subsystem Modules, is useful for the analysis of possibles relationships of Stack Traces from Stack Trace Collections and other Historical Information during the root cause analysis of various crash and hang issues. This can also be applicable to .NET managed space analysis that includes various multi-file assemblies.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Stack Overflow (user mode) analysis pattern relied on prominent stack trace signs. However, in cases of low memory condition there may not be an opportunity to create a new guard page for the expanding stack region and the exception itself may be Hidden Exception:

0:003> kc
# Call Site
00 KERNELBASE!GetModuleHandleExW
01 EdgeContent!wil::details::GetModuleInformation
02 EdgeContent!wil::details::GetCurrentModuleName
03 EdgeContent!wil::details::LogFailure
04 EdgeContent!wil::details::ReportFailure
05 EdgeContent!wil::details::ReportFailure_Hr
06 EdgeContent!wil::details::in1diag3::FailFast_Hr
07 EdgeContent!`anonymous namespace'::MemoryLimitWatchdogThreadProc
08 kernel32!BaseThreadInitThunk
09 ntdll!RtlUserThreadStart

0:003> .exr -1
ExceptionAddress: 00007ff992f6391b (KERNELBASE!GetModuleHandleExW+0x000000000000007b)
ExceptionCode: c00000fd (Stack overflow)
ExceptionFlags: 00000000
NumberParameters: 2
Parameter[0]: 0000000000000001
Parameter[1]: 000000fe709fdfc8

0:003> r
rax=0000000000000001 rbx=0000000000000006 rcx=00007ff945e3fa80
rdx=000000fe709fe020 rsi=00007ff945e3fa80 rdi=000000fe709fe030
rip=00007ff992f6391b rsp=000000fe709fdfd0 rbp=0000000000000040
r8=000000fe709fe030  r9=0000000000000040 r10=00000fff28bc7dba
r11=0440000001000010 r12=00007ff94606be60 r13=000000000000013f
r14=0000000000000003 r15=0000000000000000
iopl=0         nv up ei pl nz na pe nc
cs=0033  ss=002b  ds=002b  es=002b  fs=0053  gs=002b             efl=00010200
KERNELBASE!GetModuleHandleExW+0x7b:
00007ff9`92f6391b ff1597f31200    call    qword ptr [KERNELBASE!_imp_RtlPcToFileHeader (00007ff9`93092cb8)] ds:00007ff9`93092cb8={ntdll!RtlPcToFileHeader (00007ff9`960d0f50)}

The stack trace is short and the exception seem happened during call so we examine the stack pointer:

0:003> !address 000000fe709fdfd0
Usage:                  Stack
Base Address:           000000fe`709fb000
End Address:            000000fe`70a00000
Region Size:            00000000`00005000 (  20.000 kB)
State:                  00001000          MEM_COMMIT
Protect:                00000004          PAGE_READWRITE
Type:                   00020000          MEM_PRIVATE
Allocation Base:        000000fe`70900000
Allocation Protect:     00000004          PAGE_READWRITE

The committed stack region is very small so we expect its surrounding from the output of !address WinDbg command (without any address parameters):

Opened log file 'c:\MemoryDumps\stack.txt'
0:003> !address 000000fe709fdfd0
[...]
+ fe`70600000 fe`706fb000 0`000fb000 MEM_PRIVATE MEM_RESERVE Stack [~1; 1f50.21d4]
fe`706fb000 fe`706fe000 0`00003000 MEM_PRIVATE MEM_COMMIT PAGE_READWRITE|PAGE_GUARD Stack [~1; 1f50.21d4]
fe`706fe000 fe`70700000 0`00002000 MEM_PRIVATE MEM_COMMIT PAGE_READWRITE Stack [~1; 1f50.21d4]
+ fe`70700000 fe`70800000 0`00100000 MEM_FREE PAGE_NOACCESS Free
+ fe`70800000 fe`708f9000 0`000f9000 MEM_PRIVATE MEM_RESERVE Stack [~2; 1f50.460c]
fe`708f9000 fe`708fc000 0`00003000 MEM_PRIVATE MEM_COMMIT PAGE_READWRITE|PAGE_GUARD Stack [~2; 1f50.460c]
fe`708fc000 fe`70900000 0`00004000 MEM_PRIVATE MEM_COMMIT PAGE_READWRITE Stack [~2; 1f50.460c]
+ fe`70900000 fe`709fb000 0`000fb000 MEM_PRIVATE MEM_RESERVE Stack [~3; 1f50.4f80]
fe`709fb000 fe`70a00000 0`00005000 MEM_PRIVATE MEM_COMMIT PAGE_READWRITE Stack [~3; 1f50.4f80]
+ fe`70a00000 fe`70afc000 0`000fc000 MEM_PRIVATE MEM_RESERVE Stack [~4; 1f50.4ff4]
fe`70afc000 fe`70aff000 0`00003000 MEM_PRIVATE MEM_COMMIT PAGE_READWRITE|PAGE_GUARD Stack [~4; 1f50.4ff4]
fe`70aff000 fe`70b00000 0`00001000 MEM_PRIVATE MEM_COMMIT PAGE_READWRITE Stack [~4; 1f50.4ff4]
+ fe`70b00000 fe`70bf9000 0`000f9000 MEM_PRIVATE MEM_RESERVE Stack [~5; 1f50.5ea0]
fe`70bf9000 fe`70bfc000 0`00003000 MEM_PRIVATE MEM_COMMIT PAGE_READWRITE|PAGE_GUARD Stack [~5; 1f50.5ea0]
fe`70bfc000 fe`70c00000 0`00004000 MEM_PRIVATE MEM_COMMIT PAGE_READWRITE Stack [~5; 1f50.5ea0]
[…]

We see that there is no guard page for the stack ~3 compared to other stacks and there is plenty of reserved space before that to create a new guard page in the normal circumstances. We also recall that the stack overflow exception is about the inability to create a guard page:

0:003> !error c00000fd
Error code: (NTSTATUS) 0xc00000fd (3221225725) - A new guard page for the stack cannot be created.

Looking at Stack Trace Collection we find the thread ~11 that was doing System Call:

0:003> ~*kc
[...]

11  Id: 1f50.2424 Suspend: 1 Teb: 000000fe`6fec8000 Unfrozen
# Call Site
00 ntdll!NtAllocateVirtualMemory
01 KERNELBASE!VirtualAlloc
02 eShims!NS_ACGLockdownTelemetry::APIHook_VirtualAlloc
03 Chakra!Memory::VirtualAllocWrapper::AllocPages
[…]

For this thread the last error value show that there was low memory condition, perhaps the page file was full to create new committed pages:

0:003> ~11s
ntdll!NtAllocateVirtualMemory+0x14:
00007ff9`9615aca4 c3              ret

0:011> !teb
TEB at 000000fe6fec8000
ExceptionList:        0000000000000000
StackBase:            000000fe71e00000
StackLimit:           000000fe7140a000
SubSystemTib:         0000000000000000
FiberData:            0000000000001e00
ArbitraryUserPointer: 0000000000000000
Self:                 000000fe6fec8000
EnvironmentPointer:   0000000000000000
ClientId:             0000000000001f50 . 0000000000002424
RpcHandle:            0000000000000000
Tls Storage:          0000023c84be2c70
PEB Address:          000000fe6fe97000
LastErrorValue:       1455
LastStatusValue:      c000012d
Count Owned Locks:    0
HardErrorMode:        0

0:011> !error c000012d
Error code: (NTSTATUS) 0xc000012d (3221225773) - {Out of Virtual Memory}  Your system is low on virtual memory. To ensure that Windows runs properly, increase the size of your virtual memory paging file. For more information, see Help.

0:011> !error 0n1455
Error code: (Win32) 0x5af (1455) - The paging file is too small for this operation to complete.

Because the original analysis you see above was done on the memory dump from the working computer we are not able to provide that memory dump for security reasons. So, to model this behavior, we created an application that has two threads: one leaks committed virtual memory and when the next allocation fails it creates a separate thread that attempts to do stack overflow. We then launched 20 copies of that process to limit the size of the individual crash dump. The example memory dump and the application PDB file can be downloaded from here.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

When we have Active Thread:

0:001> kL
# Child-SP RetAddr Call Site
00 (Inline Function) --------`-------- Windows_Web!JsonParser::ParseString+0x96
01 0000007f`56efdb00 00007fff`700198c5 Windows_Web!JsonParser::ParseValue+0x3b7
02 0000007f`56efdb80 00007fff`70018ab3 Windows_Web!JsonParser::ParseArray+0xf5
03 0000007f`56efdbd0 00007fff`70018f72 Windows_Web!JsonParser::ParseValue+0x663
04 0000007f`56efdc50 00007fff`700186fb Windows_Web!JsonParser::ParseObject+0x422
05 0000007f`56efdda0 00007fff`700198c5 Windows_Web!JsonParser::ParseValue+0x2ab
06 0000007f`56efde20 00007fff`70018ab3 Windows_Web!JsonParser::ParseArray+0xf5
07 0000007f`56efde70 00007fff`70018f72 Windows_Web!JsonParser::ParseValue+0x663
08 0000007f`56efdef0 00007fff`700186fb Windows_Web!JsonParser::ParseObject+0x422
09 0000007f`56efe040 00007fff`70018f72 Windows_Web!JsonParser::ParseValue+0x2ab
0a 0000007f`56efe0c0 00007fff`700186fb Windows_Web!JsonParser::ParseObject+0x422
0b 0000007f`56efe210 00007fff`70011a69 Windows_Web!JsonParser::ParseValue+0x2ab
0c 0000007f`56efe290 00007fff`700119d2 Windows_Web!JsonValueImpl::ParseInternal+0x75
0d 0000007f`56efe2d0 00007fff`700118e8 Windows_Web!JsonValueImpl::InitializeFromString+0x16
0e 0000007f`56efe300 00007fff`70011876 Windows_Web!JsonValueFactory::TryParseInternal+0x50
0f 0000007f`56efe330 00007fff`3a6fe48e Windows_Web!JsonValueFactory::TryParse+0x36
10 0000007f`56efe360 00007fff`3a78f95c eModel!JsonUtils::GetJsonObject+0x82
[...]

which is also Spiking Thread:

0:001> !runaway f
User Mode Time
Thread Time
1:1a2c 0 days 0:03:37.562
8:52a8 0 days 0:00:36.890
[…]

Kernel Mode Time
Thread Time
8:52a8 0 days 0:00:10.625
1:1a2c 0 days 0:00:18.078
[...]

we can look at the relative proportion of User Mode / Kernel Mode times to infer Active Space. This can be further confirmed if we have Step Dumps, for example, in our case we had an earlier memory dump with the same Active Thread and Constant Subtrace (with Variable Subtraces in the same Active Space):

0:001> kL
# Child-SP RetAddr Call Site
00 0000007f`56efd8d0 00007fff`a1109453 ntdll!RtlpHpLfhSlotAllocate+0×126
01 0000007f`56efd9c0 00007fff`a110a8db ntdll!RtlpAllocateHeapInternal+0×313
02 0000007f`56efdaa0 00007fff`9ea09960 ntdll!RtlpHpAllocWithExceptionProtection+0×3b
03 0000007f`56efdb70 00007fff`7001ba1b msvcrt!malloc+0×70
04 0000007f`56efdba0 00007fff`700187ca Windows_Web!operator new+0×23
05 (Inline Function) ——–`——– Windows_Web!StringBuilder::EnsureBufferSpace+0×18
06 (Inline Function) ——–`——– Windows_Web!JsonParser::ParseString+0×59
07 0000007f`56efdbd0 00007fff`70018f72 Windows_Web!JsonParser::ParseValue+0×37a
08 0000007f`56efdc50 00007fff`700186fb Windows_Web!JsonParser::ParseObject+0×422
09 0000007f`56efdda0 00007fff`700198c5 Windows_Web!JsonParser::ParseValue+0×2ab
0a 0000007f`56efde20 00007fff`70018ab3 Windows_Web!JsonParser::ParseArray+0xf5
0b 0000007f`56efde70 00007fff`70018f72 Windows_Web!JsonParser::ParseValue+0×663
0c 0000007f`56efdef0 00007fff`700186fb Windows_Web!JsonParser::ParseObject+0×422
0d 0000007f`56efe040 00007fff`70018f72 Windows_Web!JsonParser::ParseValue+0×2ab
0e 0000007f`56efe0c0 00007fff`700186fb Windows_Web!JsonParser::ParseObject+0×422
0f 0000007f`56efe210 00007fff`70011a69 Windows_Web!JsonParser::ParseValue+0×2ab
10 0000007f`56efe290 00007fff`700119d2 Windows_Web!JsonValueImpl::ParseInternal+0×75
11 0000007f`56efe2d0 00007fff`700118e8 Windows_Web!JsonValueImpl::InitializeFromString+0×16
12 0000007f`56efe300 00007fff`70011876 Windows_Web!JsonValueFactory::TryParseInternal+0×50
13 0000007f`56efe330 00007fff`3a6fe48e Windows_Web!JsonValueFactory::TryParse+0×36
14 0000007f`56efe360 00007fff`3a78f95c eModel!JsonUtils::GetJsonObject+0×82
[…]

Even in the absence of Active Threads in process memory dumps it is useful to look at time spent in kernel and user modes to see any anomalies, especially when threads are blocked in System Calls. Then the high value of kernel mode time should suggest to inspect kernel Activity Space (for example, via a kernel dump). If we have an indication of high values of user mode time in a kernel memory dump we may need to obtain process memory Step Dumps.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

In addition to Exception Thread analysis pattern (or several threads when we have Multiple Exceptions) that we named Exception Stack Trace there may be a separated thread that reports such exceptions (Exception Reporting Thread):

This dump file has an exception of interest stored in it.
The stored exception information can be accessed via .ecxr.
(4a0c.2ab4): Application hang - code cfffffff (first/second chance not available)
win32u!NtUserCreateWindowEx+0x14:

0:002> kc
# Call Site
00 win32u!NtUserCreateWindowEx
01 user32!VerNtUserCreateWindowEx
02 user32!CreateWindowInternal
03 user32!CreateWindowExW
04 CoreMessaging!Microsoft::CoreUI::Dispatch::UserAdapter:: InitializeWindow
05 CoreMessaging!Microsoft::CoreUI::Dispatch::UserAdapter::Initialize
06 CoreMessaging!Microsoft::CoreUI::Dispatch::EventLoop::Callback_Run
07 CoreMessaging!Microsoft::CoreUI::Messaging::MessageSession$R:: Microsoft__CoreUI__IExportMessageSession_Impl::Run
08 CoreMessaging!Microsoft::CoreUI:: IExportMessageSession$X__ExportAdapter::Run
09 CoreMessaging!Windows::System::DispatcherQueue::RunLoop
0a CoreMessaging!Windows::System::DispatcherQueueController:: DispatcherQueueThreadProc
0b kernel32!BaseThreadInitThunk
0c ntdll!RtlUserThreadStart

0:002> ~42kc
# Call Site
00 ntdll!NtWaitForMultipleObjects
01 ntdll!WerpWaitForCrashReporting
02 ntdll!RtlReportExceptionEx
03 eModel!s_ReportAsyncBoundaryHangOnSuspendedThread
04 kernel32!BaseThreadInitThunk
05 ntdll!RtlUserThreadStart

We can spot such a thread in Stack Trace Collection.

Although the example above is for application hang (Blocked Thread) and also illustrates Self-Diagnosis, similar threads may report Handled Exceptions. Usually, Exception Threads are also Exception Reporting Threads as shown in Invalid Parameter analysis pattern example.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Insufficient stack memory may not result in Stack Overflow if there are internal thread checks or external watchdog threads. In some way this is a specialization of a more general Insufficient Memory (region) analysis pattern.

Consider this thread stack trace we see when we open a crash dump in WinDbg:

0:003> kc
# Call Site
00 EdgeContent!wil::details::ReportFailure
01 EdgeContent!wil::details::ReportFailure_Hr
02 EdgeContent!wil::details::in1diag3::FailFast_Hr
03 EdgeContent!`anonymous namespace'::MemoryLimitWatchdogThreadProc
04 kernel32!BaseThreadInitThunk
05 ntdll!RtlUserThreadStart

The reported error is related to general security checks and doesn’t reveal much:

0:003> .lastevent
Last event: 10e0.25fc: Security check failure or stack buffer overrun - code c0000409 (first/second chance not available)
debugger time: Sat Sep 29 21:30:31.531 2018 (UTC + 1:00)

However, from Stack Trace Motif we may infer that the failure was related to some memory limit.

When looking at Stack Trace Collection we notice the thread #11 which has the long sequence of frames:

11 Id: 10e0.d68 Suspend: 2 Teb: 000000ca`e96e3000 Unfrozen
# Call Site
00 ntdll!NtWaitForSingleObject
01 KERNELBASE!WaitForSingleObjectEx
02 Chakra!Memory::Recycler::WaitForConcurrentThread
03 Chakra!Memory::Recycler::FinishConcurrentCollect
04 Chakra!ThreadContext::ExecuteRecyclerCollectionFunction
05 Chakra!Memory::Recycler::FinishConcurrentCollectWrapped
06 Chakra!Memory::Recycler::LargeAlloc<0>
07 Chakra!Memory::Recycler::AllocLeaf
08 Chakra!Js::CompoundString::GetSz
09 Chakra!Js::JSONStringifier::ReadProperty
0a Chakra!Js::JSONStringifier::ReadObject
0b Chakra!Js::JSONStringifier::ReadProperty
0c Chakra!Js::JSONStringifier::Stringify
0d Chakra!JSON::Stringify
0e Chakra!amd64_CallFunction
0f Chakra!Js::InterpreterStackFrame::OP_CallCommon<Js::OpLayoutDynamicProfile<
Js::OpLayoutT_CallIWithICIndex<Js::LayoutSizePolicy<0> > > >
10 Chakra!Js::InterpreterStackFrame::ProcessUnprofiled
11 Chakra!Js::InterpreterStackFrame::Process
12 Chakra!Js::InterpreterStackFrame::InterpreterHelper
13 Chakra!Js::InterpreterStackFrame::InterpreterThunk
[...]
fa Chakra!ThreadContext::ExecuteImplicitCall<<
lambda_5a46706206cf607f01fe0fb33b6e8acf> >
fb Chakra!Js::DynamicObject::CallToPrimitiveFunction
fc Chakra!Js::DynamicObject::ToPrimitiveImpl<409>
fd Chakra!Js::DynamicObject::ToPrimitive
fe Chakra!Js::JavascriptConversion::OrdinaryToPrimitive
ff Chakra!Js::JavascriptConversion::MethodCallToPrimitive

It looks like the thread stack is really long that we suspect the possibility of overflow:

0:003> ~11kc 0xffff
[...]
c473 Chakra!amd64_CallFunction
c474 Chakra!Js::JavascriptFunction::CallFunction<1>
c475 Chakra!Js::JavascriptFunction::CallRootFunctionInternal
c476 Chakra!Js::JavascriptFunction::CallRootFunction
c477 Chakra!ScriptSite::CallRootFunction
c478 Chakra!ScriptSite::Execute
c479 Chakra!ScriptEngineBase::Execute
c47a edgehtml!CMutationObserver::PerformMicrotaskCheckpoint
c47b edgehtml!CObserverManager::InvokeObserversForCheckpoint
c47c edgehtml!CObserverManager::MicroTaskExecutionCallback
c47d edgehtml!HTML5TaskScheduler::RunReadiedTask
c47e edgehtml!HTML5TaskScheduler::PerformMicrotaskCheckpoint
c47f edgehtml!CJScript9Holder::ExecuteCallback
c480 edgehtml!CScriptTimer::ExecuteTimer
c481 edgehtml!CScriptTimerManager::OnTick
c482 edgehtml!CPaintAlignedTimerManager<CPaintBeat>::ProcessTimers
c483 edgehtml!CPaintBeat::OnProcessTimersTask
c484 edgehtml!GWndAsyncTask::Run
c485 edgehtml!HTML5TaskScheduler::RunReadiedTask
c486 edgehtml!TaskSchedulerBase::RunReadiedTasksInTaskQueueWithCallback
c487 edgehtml!HTML5TaskScheduler::RunReadiedTasks
c488 edgehtml!HTML5EventLoopDriver::DriveLowPriorityTaskExecution
c489 edgehtml!GlobalWndOnPaintPriorityMethodCall
c48a edgehtml!GlobalWndProc
c48b user32!FilteredProcessRedirectingWndProcW
c48c user32!UserCallWinProcCheckWow
c48d user32!DispatchClientMessage
c48e user32!_fnDWORD
c48f ntdll!KiUserCallbackDispatcherContinue
c490 win32u!NtUserDispatchMessage
c491 user32!DispatchMessageWorker
c492 EdgeContent!CBrowserTab::_TabWindowThreadProc
c493 EdgeContent!LCIETab_ThreadProc
c494 edgeIso!_IsoThreadProc_WrapperToReleaseScope
c495 kernel32!BaseThreadInitThunk
c496 ntdll!RtlUserThreadStart

However, the current stack pointer is the way above the stack region limit:

0:003> ~11s
ntdll!NtWaitForSingleObject+0x14:
00007ffa`b0d99f74 c3              ret

0:011> r rsp
rsp=000000caeb033348

0:011> !teb
TEB at 000000cae96e3000
ExceptionList:        0000000000000000
StackBase:            000000caeba00000
StackLimit:           000000caeb00b000
SubSystemTib:         0000000000000000
FiberData:            0000000000001e00
ArbitraryUserPointer: 0000000000000000
Self:                 000000cae96e3000
EnvironmentPointer:   0000000000000000
ClientId:             00000000000010e0 . 0000000000000d68
RpcHandle:            0000000000000000
Tls Storage:          000001e915185590
PEB Address:          000000cae96a8000
LastErrorValue:       1455
LastStatusValue:      c000012d
Count Owned Locks:    0
HardErrorMode:        0

The last error code shows possible Insufficient Memory (committed memory) pattern but there were enough space in the page file and the same crashes were observed in a much more spacious memory environments:

0:011> !error c000012d
Error code: (NTSTATUS) 0xc000012d (3221225773) - {Out of Virtual Memory}  Your system is low on virtual memory. To ensure that Windows runs properly, increase the size of your virtual memory paging file. For more information, see Help.

0:011> !error 0n1455
Error code: (Win32) 0x5af (1455) - The paging file is too small for this operation to complete.

When we look at a raw stack trace region though, we see it was used to the fullest:

0:011> dps 000000caeb00b000 L30
000000ca`eb00b000 00000000`00000000
000000ca`eb00b008 00000000`00000000
000000ca`eb00b010 00000000`00000000
000000ca`eb00b018 00000000`00000000
000000ca`eb00b020 00000000`00000000
000000ca`eb00b028 00000000`00000000
000000ca`eb00b030 00000000`00000000
000000ca`eb00b038 00000000`00000000
000000ca`eb00b040 00000000`00000000
000000ca`eb00b048 00000000`00000000
000000ca`eb00b050 00000000`00000000
000000ca`eb00b058 00000000`00000000
000000ca`eb00b060 00000000`00000000
000000ca`eb00b068 00000000`00000000
000000ca`eb00b070 00000000`00000000
000000ca`eb00b078 00000000`00000000
000000ca`eb00b080 00000000`00000000
000000ca`eb00b088 00000000`00000000
000000ca`eb00b090 00000000`00000000
000000ca`eb00b098 000001e9`258805c0
000000ca`eb00b0a0 00000000`00000000
000000ca`eb00b0a8 00000000`00000000
000000ca`eb00b0b0 00000000`00000131
000000ca`eb00b0b8 00007ffa`8de137a9 Chakra!Js::DictionaryTypeHandlerBase<unsigned>::GetProperty+0×119
000000ca`eb00b0c0 00000000`00000006
000000ca`eb00b0c8 000001e9`258805c0
000000ca`eb00b0d0 000001e9`258805c0
000000ca`eb00b0d8 000001e9`283d4e70
000000ca`eb00b0e0 000000ca`eb00b210
000000ca`eb00b0e8 00000000`00000000
000000ca`eb00b0f0 00000000`00000131
000000ca`eb00b0f8 000001f1`71863c70
000000ca`eb00b100 00007ffa`8e3cd468 Chakra!Js::BuiltInPropertyRecords::number
000000ca`eb00b108 00000000`00000000
000000ca`eb00b110 000001f1`71863c70
000000ca`eb00b118 000000ca`eb00b210
000000ca`eb00b120 000001e9`258805c0
000000ca`eb00b128 00007ffa`8de103d3 Chakra!Js::DynamicObject::GetPropertyQuery+0×53
000000ca`eb00b130 000001e9`24cc65b0
000000ca`eb00b138 000001e9`258805c0
000000ca`eb00b140 000001e9`258805c0
000000ca`eb00b148 00000000`00000131
000000ca`eb00b150 000000ca`eb00b210
000000ca`eb00b158 00000000`00000000
000000ca`eb00b160 000001f1`71863c70
000000ca`eb00b168 00000000`00000000
000000ca`eb00b170 00000000`00000000
000000ca`eb00b178 00007ffa`8df79864 Chakra!Js::JavascriptOperators::GetProperty_InternalSimple+0×64

We also see Execution Residue possibly indicating catch exception processing:

0:011> dpS 000000caeb00b000 L100
00007ffa`8de137a9 Chakra!Js::DictionaryTypeHandlerBase<unsigned short>::GetProperty+0x119
00007ffa`8e3cd468 Chakra!Js::BuiltInPropertyRecords::number
00007ffa`8de103d3 Chakra!Js::DynamicObject::GetPropertyQuery+0x53
00007ffa`8df79864 Chakra!Js::JavascriptOperators::GetProperty_InternalSimple+0x64
00007ffa`b0d470d0 ntdll!RtlSetLastWin32Error+0x40
00007ffa`b0d470d0 ntdll!RtlSetLastWin32Error+0x40
00007ffa`b048a5da msvcrt!getptd_noexit+0x6e
00007ffa`8dfb36d6 Chakra!`Js::JavascriptExceptionOperators::StackTraceAccessor'::`1'::catch$8
+0x50
00007ffa`b048a553 msvcrt!getptd+0xb
00007ffa`b045aad6 msvcrt!_DestructExceptionObject+0x46
00007ffa`b0463798 msvcrt!_CxxCallCatchBlock+0x1e8
00007ffa`8ddb6d54 Chakra!Js::JavascriptExceptionOperators::StackTraceAccessor+0x2a4
00007ffa`8e324798 Chakra!ValueType::PrimitiveOrObject+0xa80
00007ffa`8dfb3686 Chakra!`Js::JavascriptExceptionOperators::StackTraceAccessor'::`1'::catch$8
00007ffa`8dd20000 Chakra!_jscriptinfo_IID_Lookup <PERF> (Chakra+0x0)
00007ffa`b04635b0 msvcrt!_CxxCallCatchBlock
00007ffa`b0d9e2b3 ntdll!RcConsolidateFrames+0x3
00007ffa`8ddb6d54 Chakra!Js::JavascriptExceptionOperators::StackTraceAccessor+0x2a4
00007ffa`8de137a9 Chakra!Js::DictionaryTypeHandlerBase<unsigned short>::GetProperty+0x119
00007ffa`8e3cd468 Chakra!Js::BuiltInPropertyRecords::number
00007ffa`8de103d3 Chakra!Js::DynamicObject::GetPropertyQuery+0x53
00007ffa`8df79864 Chakra!Js::JavascriptOperators::GetProperty_InternalSimple+0x64
00007ffa`b0d470d0 ntdll!RtlSetLastWin32Error+0x40
00007ffa`b0d470d0 ntdll!RtlSetLastWin32Error+0x40
00007ffa`b048a5da msvcrt!getptd_noexit+0x6e
00007ffa`8dfb36d6 Chakra!`Js::JavascriptExceptionOperators::StackTraceAccessor'::`1'::catch$8
+0x50
00007ffa`b048a553 msvcrt!getptd+0xb
00007ffa`b045aad6 msvcrt!_DestructExceptionObject+0x46
00007ffa`b0463798 msvcrt!_CxxCallCatchBlock+0x1e8
00007ffa`8ddb6d54 Chakra!Js::JavascriptExceptionOperators::StackTraceAccessor+0x2a4

Another similar crash on a different machine with much more memory available had Active Thread Exception Stack Trace:

0:013> kc
# Call Site
00 ntdll!NtQueryWnfStateNameInformation
01 ntdll!SignalStartWerSvc
02 ntdll!SendMessageToWERService
03 ntdll!ReportExceptionInternal
04 kernel32!WerpReportFaultInternal
05 kernel32!WerpReportFault
06 KERNELBASE!UnhandledExceptionFilter
07 Chakra!FatalExceptionFilter
08 Chakra!ReportFatalException$filt$0
09 msvcrt!_C_specific_handler
0a ntdll!RtlpExecuteHandlerForException
0b ntdll!RtlDispatchException
0c ntdll!RtlRaiseException
0d KERNELBASE!RaiseException
0e Chakra!ReportFatalException
0f Chakra!OutOfMemory_fatal_error
10 Chakra!Js::Exception::RaiseIfScriptActive
11 Chakra!Js::Throw::OutOfMemory
12 Chakra!Memory::Recycler::LargeAlloc<0>
13 Chakra!Memory::Recycler::AllocLeaf
14 Chakra!Js::LazyJSONString::GetSz
15 Chakra!Js::CompoundString::AppendGeneric<Js::CompoundString>
16 Chakra!Js::JavascriptExceptionOperators::StackTraceAccessor
17 Chakra!Js::InlineCache::TryGetProperty<1,1,1,1,0>
18 Chakra!Js::CacheOperators::TryGetProperty<1,1,1,1,1,1,1,0,0>
19 Chakra!Js::JavascriptOperators::PatchGetValue<1,Js::InlineCache>
1a js!d.toString
[...]

which actually showed the Javascript URL:

0:013> .frame 1a
1a 00000037`4671e5e0 00007ffe`1753f1a1 js!d.toString [https://yastatic.net/awaps-ad-sdk-js-bundles/1.0-889/bundles/inpage.bundle.js @ 8,36454]

The stack region was also almost full with execution residue including exception catch processing. But this time it show “stack space” hint:

0:013> !teb
TEB at 00000037456dc000
ExceptionList:        0000000000000000
StackBase:            0000003747100000
StackLimit:           000000374670a000
SubSystemTib:         0000000000000000
FiberData:            0000000000001e00
ArbitraryUserPointer: 0000000000000000
Self:                 00000037456dc000
EnvironmentPointer:   0000000000000000
ClientId:             0000000000005914 . 0000000000001150
RpcHandle:            0000000000000000
Tls Storage:          000001a5bcfd88d0
PEB Address:          00000037456bd000
LastErrorValue:       0
LastStatusValue:      c000012d
Count Owned Locks:    0
HardErrorMode:        0

0:013> s-su 000000374670a000 00000037`46720000
[…]
00000037`4670d210 “tack space”
[…]

Note: Regardless of the actual root cause and mechanism for these crash dumps, such stack region monitoring can also be used for software prognostics purposes.

The example dump can be downloaded from here.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

For completion, we introduce a managed space version of Hidden Exception in addition to user and kernel space variants.

0:000> ~*kL
[...]
13 Id: 1b70.1c2c Suspend: 0 Teb: 00446000 Unfrozen
# ChildEBP RetAddr
00 08e7ec4c 755e1cf3 ntdll!NtWaitForMultipleObjects+0xc
01 08e7ede0 6ef8bc6e KERNELBASE!WaitForMultipleObjectsEx+0x133
02 08e7ee30 6ef8b9b3 clr!WaitForMultipleObjectsEx_SO_TOLERANT+0x3c
03 08e7eebc 6ef8baa4 clr!Thread::DoAppropriateWaitWorker+0x237
04 08e7ef28 6ef8bc14 clr!Thread::DoAppropriateWait+0x64
05 08e7ef74 6eef648b clr!CLREventBase::WaitEx+0x128
06 08e7ef8c 6f0058f6 clr!CLREventBase::Wait+0x1a
07 08e7f018 6f005834 clr!AwareLock::EnterEpilogHelper+0xa8
08 08e7f060 6f005980 clr!AwareLock::EnterEpilog+0x48
09 08e7f078 6f00662c clr!AwareLock::Enter+0x4a
0a 08e7f104 08d71d79 clr!JITutil_MonEnterWorker+0x9c
WARNING: Frame IP not in any known module. Following frames may be wrong.
0b 08e7f120 6dd9608d 0x8d71d79
0c 08e7f12c 6ddc2925 mscorlib_ni+0x3c608d
0d 08e7f190 6ddc2836 mscorlib_ni+0x3f2925
0e 08e7f1a4 6ddc27f1 mscorlib_ni+0x3f2836
0f 08e7f1c0 6dd95fe8 mscorlib_ni+0x3f27f1
10 08e7f1d8 6ee6eaf6 mscorlib_ni+0x3c5fe8
11 08e7f1e4 6ee71d50 clr!CallDescrWorkerInternal+0x34
12 08e7f238 6ee77764 clr!CallDescrWorkerWithHandler+0x6b
13 08e7f2a0 6eef4d2d clr!MethodDescCallSite::CallTargetWorker+0x16a
14 08e7f414 6efae269 clr!ThreadNative::KickOffThread_Worker+0x173
15 08e7f428 6efae2d3 clr!ManagedThreadBase_DispatchInner+0x71
16 08e7f4cc 6efae3a0 clr!ManagedThreadBase_DispatchMiddle+0x7e
17 08e7f528 6ee7af05 clr!ManagedThreadBase_DispatchOuter+0x5b
18 08e7f534 6ee7aea2 clr!ManagedThreadBase_DispatchInCorrectAD+0x15
19 08e7f600 6ee7af4d clr!Thread::DoADCallBack+0x328
1a 08e7f624 6efae2d3 clr!ManagedThreadBase_DispatchInner+0x4e
1b 08e7f6c8 6efae3a0 clr!ManagedThreadBase_DispatchMiddle+0x7e
1c 08e7f724 6efae40f clr!ManagedThreadBase_DispatchOuter+0x5b
1d 08e7f748 6eef4be2 clr!ManagedThreadBase_FullTransitionWithAD+0x2f
1e 08e7f7c4 6eef62d1 clr!ThreadNative::KickOffThread+0x256
1f 08e7fbe4 76c28484 clr!Thread::intermediateThreadProc+0x55
20 08e7fbf8 77842fea kernel32!BaseThreadInitThunk+0x24
21 08e7fc40 77842fba ntdll!__RtlUserThreadStart+0x2f
22 08e7fc50 00000000 ntdll!_RtlUserThreadStart+0x1b
[...]

0:000> ~13s
eax=00000000 ebx=00000001 ecx=00000000 edx=00000000 esi=00000001 edi=00000001
eip=7784a7bc esp=08e7ec50 ebp=08e7ede0 iopl=0 nv up ei pl nz ac pe nc
cs=0023 ss=002b ds=002b es=002b fs=0053 gs=002b efl=00000216
ntdll!NtWaitForMultipleObjects+0xc:
7784a7bc c21400 ret 14h

0:013> !CLRStack
OS Thread Id: 0x1c2c (13)
Child SP IP Call Site
08e7efb4 7784a7bc [GCFrame: 08e7efb4]
08e7f094 7784a7bc [HelperMethodFrame_1OBJ: 08e7f094] System.Threading.Monitor.Enter(System.Object)
08e7f10c 08d71d79 UserQuery+ClassMain.thread_proc_1()
08e7f128 6dd9608d *** ERROR: Module load completed but symbols could not be loaded for mscorlib.ni.dll
System.Threading.ThreadHelper.ThreadStart_Context(System.Object)
08e7f134 6ddc2925 System.Threading.ExecutionContext.RunInternal(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object, Boolean)
08e7f1a0 6ddc2836 System.Threading.ExecutionContext.Run(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object, Boolean)
08e7f1b4 6ddc27f1 System.Threading.ExecutionContext.Run(System.Threading.ExecutionContext, System.Threading.ContextCallback, System.Object)
08e7f1cc 6dd95fe8 System.Threading.ThreadHelper.ThreadStart()
08e7f308 6ee6eaf6 [GCFrame: 08e7f308]
08e7f4e8 6ee6eaf6 [DebuggerU2MCatchHandlerFrame: 08e7f4e8]
08e7f554 6ee6eaf6 [ContextTransitionFrame: 08e7f554]
08e7f6e4 6ee6eaf6 [DebuggerU2MCatchHandlerFrame: 08e7f6e4]

0:013> !teb
TEB at 00446000
ExceptionList: 08e7edd0
StackBase: 08e80000
StackLimit: 08e7a000
SubSystemTib: 00000000
FiberData: 00001e00
ArbitraryUserPointer: 00000000
Self: 00446000
EnvironmentPointer: 00000000
ClientId: 00001b70 . 00001c2c
RpcHandle: 00000000
Tls Storage: 008eb8e8
PEB Address: 0040a000
LastErrorValue: 0
LastStatusValue: c0000034
Count Owned Locks: 0
HardErrorMode: 0

0:013> !DumpStackObjects 08e7a000 08e80000
OS Thread Id: 0×1c2c (13)
ESP/REG Object Name
08E7DD18 0270f714 LINQPad.ExecutionModel.OutPipe
08E7DD20 02736ca8 LINQPad.Disposable
08E7DD2C 0270f714 LINQPad.ExecutionModel.OutPipe
08E7DD3C 02736ca8 LINQPad.Disposable
08E7DD40 02736c88 System.Action
08E7DD44 02736ca8 LINQPad.Disposable
08E7DD64 0270f714 LINQPad.ExecutionModel.OutPipe
08E7DD98 02736ca8 LINQPad.Disposable
08E7DDB8 0270f714 LINQPad.ExecutionModel.OutPipe
08E7DE78 0270f9ec System.Object
08E7DE7C 0270f990 LINQPad.ObjectGraph.Formatters.HtmlWriter
08E7DEAC 0270f990 LINQPad.ObjectGraph.Formatters.HtmlWriter
08E7DEE4 0262e16c System.String
08E7DEF8 026aa9d0 System.String
08E7DF04 0270f990 LINQPad.ObjectGraph.Formatters.HtmlWriter
08E7E054 02724ecc System.Threading.ThreadHelper
08E7E058 026fad7c System.Threading.ContextCallback
08E7E06C 026fad7c System.Threading.ContextCallback
08E7E074 02724ecc System.Threading.ThreadHelper
08E7E0A8 0272fb68 System.NullReferenceException
08E7E0AC 026fad7c System.Threading.ContextCallback
08E7E0B8 02724ecc System.Threading.ThreadHelper
08E7E340 0272fcc0 System.Runtime.CompilerServices.RuntimeHelpers+TryCode
08E7E344 0272fce0 System.Runtime.CompilerServices.RuntimeHelpers+CleanupCode
08E7E348 0272fca4 System.Environment+ResourceHelper+GetResourceStringUserData
08E7E35C 0272fce0 System.Runtime.CompilerServices.RuntimeHelpers+CleanupCode
08E7E378 0272fca4 System.Environment+ResourceHelper+GetResourceStringUserData
08E7E37C 0272fc0c System.Environment+ResourceHelper
08E7E964 0272fb68 System.NullReferenceException
08E7EB3C 02724ecc System.Threading.ThreadHelper
08E7ECCC 02724ecc System.Threading.ThreadHelper
08E7ECD0 026fad7c System.Threading.ContextCallback
08E7ECD8 0272fa88 System.String critical section 1
08E7EFE8 0272fabc System.String critical section 2
08E7F034 026fad7c System.Threading.ContextCallback
08E7F088 02724ecc System.Threading.ThreadHelper
08E7F08C 026fad7c System.Threading.ContextCallback
08E7F0B8 02724ecc System.Threading.ThreadHelper
08E7F0C0 026fad7c System.Threading.ContextCallback
08E7F0F0 0272fabc System.String critical section 2
08E7F11C 026fad7c System.Threading.ContextCallback
08E7F128 02724f00 System.Threading.ExecutionContext
08E7F134 02724e98 System.Threading.Thread
08E7F144 02724e98 System.Threading.Thread
08E7F188 02724f00 System.Threading.ExecutionContext
08E7F18C 026fad7c System.Threading.ContextCallback
08E7F19C 02724ecc System.Threading.ThreadHelper
08E7F1B0 02724ecc System.Threading.ThreadHelper
08E7F1B8 02724ecc System.Threading.ThreadHelper
08E7F1BC 02724f00 System.Threading.ExecutionContext
08E7F1C8 02724ecc System.Threading.ThreadHelper
08E7F244 02724ee0 System.Threading.ThreadStart
08E7F2C4 02724ee0 System.Threading.ThreadStart
08E7F2D8 02724ee0 System.Threading.ThreadStart

The example dump can be downloaded from here.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Backwards disassembling used in memory analysis patterns such as Coincidental Symbolic Information may be ambiguous and can show Wild Code output. This may also be debugger disassembling algorithm dependent. For example, default 8-instruction backwards disassembly shows this code:

0:011> ub 00007ff8`cdc9b4bf
00007ff8`cdc9b4ab 855948 test dword ptr [rcx+48h],ebx
00007ff8`cdc9b4ae b988bf03a8 mov ecx,0A803BF88h
00007ff8`cdc9b4b3 f4 hlt
00007ff8`cdc9b4b4 0100 add dword ptr [rax],eax
00007ff8`cdc9b4b6 00488b add byte ptr [rax-75h],cl
00007ff8`cdc9b4b9 09e8 or eax,ebp
00007ff8`cdc9b4bb 117236 adc dword ptr [rdx+36h],esi
00007ff8`cdc9b4be 5f pop rdi

However, if we specify the number of instructions to disassemble except 7 and 8 we get a different result (which is more correct from the forward code execution view since we disassembled the saved return address from the stack region):

0:011> ub 00007ff8`cdc9b4bf L1
00007ff8`cdc9b4ba e81172365f call clr!JIT_MonEnter (00007ff9`2d0026d0)

0:011> ub 00007ff8`cdc9b4bf L2
00007ff8`cdc9b4b7 488b09 mov rcx,qword ptr [rcx]
00007ff8`cdc9b4ba e81172365f call clr!JIT_MonEnter (00007ff9`2d0026d0)

0:011> k L10
# Child-SP RetAddr Call Site
00 0000002a`fc23e308 00007ff9`53d06099 ntdll!NtWaitForMultipleObjects+0x14
01 0000002a`fc23e310 00007ff9`2d1a96be KERNELBASE!WaitForMultipleObjectsEx+0xf9
02 0000002a`fc23e610 00007ff9`2d1a951c clr!WaitForMultipleObjectsEx_SO_TOLERANT+0x62
03 0000002a`fc23e670 00007ff9`2d1a9315 clr!Thread::DoAppropriateWaitWorker+0x1e4
04 0000002a`fc23e770 00007ff9`2d0c2b7f clr!Thread::DoAppropriateWait+0x7d
05 0000002a`fc23e7f0 00007ff9`2d1aa491 clr!CLREventBase::WaitEx+0xc4
06 0000002a`fc23e880 00007ff9`2d1aa39e clr!AwareLock::EnterEpilogHelper+0xc2
07 0000002a`fc23e940 00007ff9`2d1c1a92 clr!AwareLock::EnterEpilog+0x62
08 0000002a`fc23e9a0 00007ff8`cdc9b4bf clr!JITutil_MonEnterWorker+0xe2
09 0000002a`fc23eb40 00007ff9`275231d3 0×00007ff8`cdc9b4bf
0a 0000002a`fc23eb80 00007ff9`27523064 mscorlib_ni+0×5031d3
0b 0000002a`fc23ec50 00007ff9`27523032 mscorlib_ni+0×503064
0c 0000002a`fc23ec80 00007ff9`2751c812 mscorlib_ni+0×503032
0d 0000002a`fc23ecd0 00007ff9`2d006bb3 mscorlib_ni+0×4fc812
0e 0000002a`fc23ed10 00007ff9`2d006a70 clr!CallDescrWorkerInternal+0×83
0f 0000002a`fc23ed50 00007ff9`2d00735d clr!CallDescrWorkerWithHandler+0×4e

We call this analysis pattern Disassembly Ambiguity. The example dump can be downloaded from here.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

A virtual memory may contain regions that are memories of some other processes or systems. We do not consider the ordinary case of memory-mapped regions here but the case of type 2 hypervisor. In such a case, memory regions may be “physical memories” of Virtualized Systems. For example, we discovered such a region in crashed vmware-vmx.exe process memory dump:

0:007> !address -summary

--- Usage Summary ---------------- RgnCount ----------- Total Size -------- %ofBusy %ofTotal
Free 231 7ffe`d009b000 ( 127.995 TB) 100.00%
<unknown> 518 1`2508e000 ( 4.579 GB) 96.41% 0.00%
Image 547 0`07056000 ( 112.336 MB) 2.31% 0.00%
Heap 73 0`0216a000 ( 33.414 MB) 0.69% 0.00%
Stack 81 0`01b00000 ( 27.000 MB) 0.56% 0.00%
Other 11 0`001d0000 ( 1.813 MB) 0.04% 0.00%
TEB 27 0`00036000 ( 216.000 kB) 0.00% 0.00%
PEB 1 0`00001000 ( 4.000 kB) 0.00% 0.00%

--- Type Summary (for busy) ------ RgnCount ----------- Total Size -------- %ofBusy %ofTotal
MEM_MAPPED 88 1`0e25a000 ( 4.221 GB) 88.88% 0.00%
MEM_PRIVATE 623 0`1aca5000 ( 428.645 MB) 8.81% 0.00%
MEM_IMAGE 547 0`07056000 ( 112.336 MB) 2.31% 0.00%

--- State Summary ---------------- RgnCount ----------- Total Size -------- %ofBusy %ofTotal
MEM_FREE 231 7ffe`d009b000 ( 127.995 TB) 100.00%
MEM_COMMIT 1185 1`27657000 ( 4.616 GB) 97.18% 0.00%
MEM_RESERVE 73 0`088fe000 ( 136.992 MB) 2.82% 0.00%

--- Protect Summary (for commit) - RgnCount ----------- Total Size -------- %ofBusy %ofTotal
PAGE_READWRITE 473 1`1f38b000 ( 4.488 GB) 94.49% 0.00%
PAGE_READONLY 400 0`04a05000 ( 74.020 MB) 1.52% 0.00%
PAGE_EXECUTE_READ 196 0`0367a000 ( 54.477 MB) 1.12% 0.00%
PAGE_WRITECOPY 59 0`001de000 ( 1.867 MB) 0.04% 0.00%
PAGE_READWRITE|PAGE_GUARD 27 0`00051000 ( 324.000 kB) 0.01% 0.00%
PAGE_NOACCESS 27 0`0001b000 ( 108.000 kB) 0.00% 0.00%
PAGE_EXECUTE_READWRITE 3 0`00003000 ( 12.000 kB) 0.00% 0.00%

--- Largest Region by Usage ----------- Base Address -------- Region Size ----------
Free 290`ffe50000 7d66`9b210000 ( 125.401 TB)
<unknown> 28f`f8f90000 1`00000000 ( 4.000 GB)
Image 7ffa`9969f000 0`00e47000 ( 14.277 MB)
Heap 28f`95c7b000 0`00ae4000 ( 10.891 MB)
Stack b8`f7b00000 0`000fc000 (1008.000 kB)
Other 28f`f2050000 0`00181000 ( 1.504 MB)
TEB b8`f7147000 0`00002000 ( 8.000 kB)
PEB b8`f7146000 0`00001000 ( 4.000 kB)

The size of the region is 4 GB which coincides with the size of Windows VM:

We assume that the whole VM physical space was placed there and we had an instance of a physical memory dump inside a process memory dump. Whatever is such a physical memory dump internal organization, most likely the pages correspond to 4 Kb memory chunks inside. We can employ WinDbg commands that allow the address parameter. For example, we can look for Hidden Modules:

0:007> .imgscan /r 28f`f8f90000 L?1`00000000
[…]
MZ at 00000290`f5867000 - size 7f000
Name: HAL.dll
[…]
MZ at 00000290`a089b000 - size 3000
Name: TDI.SYS
[…]

0:007> !dh 00000290`a089b000

File Type: DLL
FILE HEADER VALUES
14C machine (i386)
2 number of sections
592AD310 time date stamp Sun May 28 06:39:28 2017

0 file pointer to symbol table
0 number of symbols
E0 size of optional header
2122 characteristics
Executable
App can handle >2gb addresses
32 bit word machine
DLL

OPTIONAL HEADER VALUES
10B magic #
9.00 linker version
A00 size of code
400 size of initialized data
0 size of uninitialized data
0 address of entry point
1000 base of code
----- new -----
ffffffff8a7d0000 image base
1000 section alignment
200 file alignment
3 subsystem (Windows CUI)
10.00 operating system version
10.00 image version
5.01 subsystem version
3000 size of image
400 size of headers
10F33 checksum
0000000000040000 size of stack reserve
0000000000001000 size of stack commit
0000000000100000 size of heap reserve
0000000000001000 size of heap commit
540 DLL characteristics
Dynamic base
NX compatible
No structured exception handler
1140 [ 73A] address [size] of Export Directory
0 [ 0] address [size] of Import Directory
2000 [ 3E8] address [size] of Resource Directory
0 [ 0] address [size] of Exception Directory
0 [ 0] address [size] of Security Directory
0 [ 0] address [size] of Base Relocation Directory
1000 [ 1C] address [size] of Debug Directory
0 [ 0] address [size] of Description Directory
0 [ 0] address [size] of Special Directory
0 [ 0] address [size] of Thread Storage Directory
0 [ 0] address [size] of Load Configuration Directory
0 [ 0] address [size] of Bound Import Directory
0 [ 0] address [size] of Import Address Table Directory
0 [ 0] address [size] of Delay Import Directory
0 [ 0] address [size] of COR20 Header Directory
0 [ 0] address [size] of Reserved Directory

SECTION HEADER #1
.text name
87A virtual size
1000 virtual address
A00 size of raw data
400 file pointer to raw data
0 file pointer to relocation table
0 file pointer to line numbers
0 number of relocations
0 number of line numbers
60000020 flags
Code
(no align specified)
Execute Read

Debug Directories(1)
Type Size Address Pointer
cv 20 101c 41c Format: RSDS, guid, 1, tdi.pdb

SECTION HEADER #2
.rsrc name
3E8 virtual size
2000 virtual address
400 size of raw data
E00 file pointer to raw data
0 file pointer to relocation table
0 file pointer to line numbers
0 number of relocations
0 number of line numbers
40000040 flags
Initialized Data
(no align specified)
Read Only
[...]

We call such pattern Hyperdump:

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Sometimes we can diagnose potential data races by looking at Stack Trace Collection (from all processes and threads or from CPUs). For example, if we have one Exception Stack Trace and a very similar stack trace, for example, with Constant Subtrace and the same Stack Trace Motif, it may point to such a situation with potential diagnosis of related components. Here’s an example from a kernel memory dump where it was suggested as a workaround to close ProcessA before hibernating a system (the problem repeatedly happened on wake up):

0: kd> !analyze -v

[...]

KERNEL_SECURITY_CHECK_FAILURE (139)
A kernel component has corrupted a critical data structure. The corruption
could potentially allow a malicious user to gain control of this machine.
Arguments:
Arg1: 0000000000000003, A LIST_ENTRY has been corrupted (i.e. double remove).
Arg2: ffff8900ea8fae40, Address of the trap frame for the exception that caused the bugcheck
Arg3: ffff8900ea8fad98, Address of the exception record for the exception that caused the bugcheck
Arg4: 0000000000000000, Reserved

[...]

EXCEPTION_RECORD: ffff8900ea8fad98 -- (.exr 0xffff8900ea8fad98)
ExceptionAddress: fffff8034a9e2e93 (nt!CmpFreeKeyControlBlock+0x0000000000106a73)
ExceptionCode: c0000409 (Security check failure or stack buffer overrun)
ExceptionFlags: 00000001
NumberParameters: 1
Parameter[0]: 0000000000000003
Subcode: 0x3 FAST_FAIL_CORRUPT_LIST_ENTRY

[...]

0: kd> k
# Child-SP RetAddr Call Site
00 ffff8900`ea8fab18 fffff803`4a576ba9 nt!KeBugCheckEx
01 ffff8900`ea8fab20 fffff803`4a576f50 nt!KiBugCheckDispatch+0x69
02 ffff8900`ea8fac60 fffff803`4a575286 nt!KiFastFailDispatch+0xd0
03 ffff8900`ea8fae40 fffff803`4a9e2e93 nt!KiRaiseSecurityCheckFailure+0x3c6
04 ffff8900`ea8fafd0 fffff803`4a8c444b nt!CmpFreeKeyControlBlock+0×106a73
05 ffff8900`ea8fb000 fffff803`4a8e1392 nt!CmpDoParseKey+0×2adb
06 ffff8900`ea8fb3d0 fffff803`4a8bdd61 nt!CmpParseKey+0×302
07 ffff8900`ea8fb570 fffff803`4a8d3a2d nt!ObpLookupObjectName+0xb71
08 ffff8900`ea8fb740 fffff803`4a8d370d nt!ObOpenObjectByNameEx+0×1dd
09 ffff8900`ea8fb880 fffff803`4a8cfc5f nt!CmOpenKey+0×29d
0a ffff8900`ea8fba40 fffff803`4a576683 nt!NtOpenKeyEx+0xf
0b ffff8900`ea8fba80 00007ffc`ac3b82b4 nt!KiSystemServiceCopyEnd+0×13
0c 00000001`2c0feb28 00000000`00000000 0×00007ffc`ac3b82b4

0: kd> !stacks 2 nt!Cmp

[...]

[ffffaa0a35ccb2c0 winlogon.exe]
13dc.002668 ffffaa0a37e02080 fff4e8f4 RUNNING nt!KeBugCheckEx
nt!KiBugCheckDispatch+0×69
nt!KiFastFailDispatch+0xd0
nt!KiRaiseSecurityCheckFailure+0×3c6
nt!CmpFreeKeyControlBlock+0×106a73
nt!CmpDoParseKey+0×2adb
nt!CmpParseKey+0×302
nt!ObpLookupObjectName+0xb71
nt!ObOpenObjectByNameEx+0×1dd
nt!CmOpenKey+0×29d
nt!NtOpenKeyEx+0xf
nt!KiSystemServiceCopyEnd+0×13
+0×7ffcac3b82b4

[...]

[ffffaa0a35bda780 ProcessA.exe]
1794.0017e4 ffffaa0a35c06080 fff4e8f4 RUNNING nt!CmpLockKcbStackShared+0×2
nt!CmpWalkOneLevel+0×93
nt!CmpDoParseKey+0×1c36
nt!CmpParseKey+0×302
nt!ObpLookupObjectName+0xb71
nt!ObOpenObjectByNameEx+0×1dd
nt!CmOpenKey+0×29d
nt!NtOpenKeyEx+0xf
nt!KiSystemServiceCopyEnd+0×13
+0×7ffcac3b82b4

[...]

[ffffaa0a35bb8780 svchost.exe]
1754.001fec ffffaa0a3636a7c0 fff4e8f4 READY nt!KxDispatchInterrupt+0x122
nt!KiDpcInterrupt+0x3a6
nt!KeAbPreAcquire+0xd7
nt!ExfAcquirePushLockExclusiveEx+0x10b
nt!CmpDoParseKey+0×294e
nt!CmpParseKey+0×302
nt!ObpLookupObjectName+0xb71
nt!ObOpenObjectByNameEx+0×1dd
nt!CmOpenKey+0×29d
nt!NtOpenKeyEx+0xf
nt!KiSystemServiceCopyEnd+0×13
+0×7ffcac3b82b4

[...]

We call this pattern Stack Trace Race.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

This is a variant of Data Correlation (function parameters) analysis pattern where we look at correlations across memory structures. Simple arithmetical ratios may link such structures and correlate corresponding behavioral processes. Here we look at a recent instance of calc.exe consuming a lot of CPU. Upon the discovery of that process we were curious and saved its full process memory dump via Task Manager. In the dump we discovered 4 Spiking Threads:

0:000> !runaway f
User Mode Time
Thread       Time
13:1b68      0 days 1:51:39.906
10:23a8      0 days 1:51:37.796
11:1b98      0 days 0:00:09.890
14:88c       0 days 0:00:09.828
1:2eb4      0 days 0:00:00.390
18:2a44      0 days 0:00:00.015
19:28f0      0 days 0:00:00.000
17:22c0      0 days 0:00:00.000
16:232c      0 days 0:00:00.000
15:2008      0 days 0:00:00.000
12:2880      0 days 0:00:00.000
9:2f38      0 days 0:00:00.000
8:1a98      0 days 0:00:00.000
7:1dcc      0 days 0:00:00.000
6:c58       0 days 0:00:00.000
5:1550      0 days 0:00:00.000
4:2938      0 days 0:00:00.000
3:2b64      0 days 0:00:00.000
2:2f90      0 days 0:00:00.000
0:dc4       0 days 0:00:00.000
[…]

We see that 10/13 approx. equals 1 and #11/#14 too, or #10/#11 approx. equals #13/#14 in user mode CPU consumption. If we look at kernel times we see the same ratios:

[...]
Kernel Mode Time
Thread Time
10:23a8 0 days 0:10:36.718
13:1b68 0 days 0:10:32.968
14:88c 0 days 0:00:23.859
11:1b98 0 days 0:00:23.812
1:2eb4 0 days 0:00:00.218
2:2f90 0 days 0:00:00.015
0:dc4 0 days 0:00:00.015
19:28f0 0 days 0:00:00.000
18:2a44 0 days 0:00:00.000
17:22c0 0 days 0:00:00.000
16:232c 0 days 0:00:00.000
15:2008 0 days 0:00:00.000
12:2880 0 days 0:00:00.000
9:2f38 0 days 0:00:00.000
8:1a98 0 days 0:00:00.000
7:1dcc 0 days 0:00:00.000
6:c58 0 days 0:00:00.000
5:1550 0 days 0:00:00.000
4:2938 0 days 0:00:00.000
3:2b64 0 days 0:00:00.000
[…]

Elapsed times are also correlated and we see that correlated threads were created in pairs {#10, #11} and {#13, #14}:

[...]
Elapsed Time
Thread Time
0:dc4 0 days 18:20:55.778
1:2eb4 0 days 18:20:55.731
2:2f90 0 days 18:20:55.725
3:2b64 0 days 18:20:55.721
4:2938 0 days 18:20:55.715
5:1550 0 days 18:20:55.582
6:c58 0 days 18:20:55.522
7:1dcc 0 days 18:20:55.522
8:1a98 0 days 18:20:55.522
9:2f38 0 days 18:20:55.522
10:23a8 0 days 16:12:52.330
11:1b98 0 days 16:12:52.329
12:2880 0 days 16:12:52.195
13:1b68 0 days 16:11:44.822
14:88c 0 days 16:11:44.821
15:2008 0 days 16:11:44.693
16:232c 0 days 2:09:35.021
17:22c0 0 days 2:05:13.038
18:2a44 0 days 0:23:38.000
19:28f0 0 days 0:00:24.261

This suggests that the threads are related. We call such analysis pattern variant Data Correlation (CPU times). It may also help in finding weak Coupled Processes.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Usually threads are Blocked waiting for synchronization objects or Active running threads. There is a different category of threads that appear blocked in user space but in fact may be doing a lot of invisible work (not visible from the dump type) in kernel space. We call such an analysis pattern System Call. Compare these thread Stack Traces from Stack Trace Collection:

# Call Site
00 ntdll!NtWaitForMultipleObjects
01 KERNELBASE!WaitForMultipleObjectsEx
02 user32!MsgWaitForMultipleObjectsEx
03 combase!ASTAWaitContext::KernelWait
04 combase!ASTAWaitContext::Wait
05 combase!CoMsgWaitInProcessEvents
06 Windows_UI!Windows::UI::Core::CDispatcher::WaitAndProcessMessages
07 Windows_UI!Windows::UI::Core::CDispatcher::ProcessEvents
08 Windows_UI_Xaml!CJupiterWindow::RunCoreWindowMessageLoop
09 Windows_UI_Xaml!CJupiterControl::RunMessageLoop
0a Windows_UI_Xaml!DirectUI::DXamlCore::RunMessageLoop
0b twinapi_appcore!Windows::ApplicationModel::Core::CoreApplicationView::Run
[...]
0e kernel32!BaseThreadInitThunk
0f ntdll!RtlUserThreadStart

# Call Site
00 user32!NtUserCallNoParam
01 user32!MsgWaitForMultipleObjectsEx
02 combase!ASTAWaitContext::KernelWait
03 combase!ASTAWaitContext::Wait
04 combase!CoMsgWaitInProcessEvents
05 Windows_UI!Windows::UI::Core::CDispatcher::WaitAndProcessMessages
06 Windows_UI!Windows::UI::Core::CDispatcher::ProcessEvents
07 Windows_UI_Xaml!CJupiterWindow::RunCoreWindowMessageLoop
08 Windows_UI_Xaml!CJupiterControl::RunMessageLoop
09 Windows_UI_Xaml!DirectUI::DXamlCore::RunMessageLoop
0a twinapi_appcore!Windows::ApplicationModel::Core::CoreApplicationView::Run
[…]
0d kernel32!BaseThreadInitThunk
0e ntdll!RtlUserThreadStart

The first is traditionally waiting, but the other was waked up to process an input message and may be other blocked in kernel or doing some work there. In case of a doubt we can double check the last executed instruction (from the stack trace perspective):

# Child-SP          RetAddr           Call Site
00 000000b9`358bf068 00007ffd`3de0a3be user32!NtUserCallNoParam+0xa
01 000000b9`358bf070 00007ffd`3dbdc106 user32!MsgWaitForMultipleObjectsEx+0×15e
[…]

0:000> ub user32!NtUserCallNoParam+0xa
user32!NtUserInvalidateRect:
00007ffd`3de1fc90 4c8bd1          mov     r10,rcx
00007ffd`3de1fc93 b807100000      mov     eax,1007h
00007ffd`3de1fc98 0f05            syscall
00007ffd`3de1fc9a c3              ret
00007ffd`3de1fc9b 0f1f440000      nop     dword ptr [rax+rax]
user32!NtUserCallNoParam:
00007ffd`3de1fca0 4c8bd1          mov     r10,rcx
00007ffd`3de1fca3 b808100000      mov     eax,1008h
00007ffd`3de1fca8 0f05            syscall

System Calls (especially discovered in unmanaged Execution Residue data and from Past Stack Traces reconstruction from raw stack regions and Historical Information) may aid in memory forensics (broadly speaking as reconstruction of past behaviour) in addition to System Objects.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Some Stack Traces reveal a functional purpose, for example, painting, querying a database, doing HTML or JavaScript processing, loading a file, printing, or something else. Such traces from various Stack Trace Collections (unmanaged, managed, predicate, CPUs) may be compared for similarity and help with analysis patterns, such as examples in Wait Chain (C++11 condition variable, SRW lock), finding semantically Coupled Processes, and many others where we look at the meaning of stack trace frame sequences to relate them to each other. We call this analysis pattern Stack Trace Motif by analogy with Motif trace and log analysis pattern. Longer stack traces may contain several Motives and also Technology-Specific Subtraces (for example, COM interface invocation frames).

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

Slim Reader/Writer locks are available from Windows Vista and sometimes we can see them in Stack Trace Collection. Although, at the time of this writing, there is no WinDbg support for them, the techniques described in the example for C++11 condition variable Wait Chain analysis pattern can be used. Here is an example where we used semantics of stack trace similarity such as “rendering” to find the stack trace of the possible owner thread:

0:000> ~*k

[...]

4  Id: be4.2af4 Suspend: 0 Teb: 7efa1000 Unfrozen
# ChildEBP RetAddr
00 0414bfe8 776e2157 ntdll_77670000!NtWaitForKeyedEvent+0×15
01 0414c054 5c9c3f8e ntdll_77670000!RtlAcquireSRWLockExclusive+0×12e
02 0414c06c 5ca2ad83 mshtml!TSmartResource<CDXRenderLock>::Acquire<enum DXLock::Type>+0×5e
03 0414c09c 5cdffa63 mshtml!CDXSystem::CheckAndHandleDeviceReset+0×43
04 0414c0b0 5cdff9eb mshtml!CDoc::CheckAndHandleDeviceReset+0×23
05 0414c130 5cabe82e mshtml!CDoc::OnPaint+0xcb
06 0414c164 5c9d632b mshtml!CServer::OnWindowMessage+0xfb
07 0414c324 5c9bbcb0 mshtml!CDoc::OnWindowMessage+0×24c
08 0414c354 76ea62fa mshtml!CServer::WndProc+0×58
09 0414c380 76ea731e user32!InternalCallWinProc+0×23
0a 0414c3f8 76ea6ded user32!UserCallWinProcCheckWow+0xd8
0b 0414c458 76ea6e4c user32!DispatchClientMessage+0xec
0c 0414c494 7768011a user32!__fnDWORD+0×2b
0d 0414c4cc 76eb1342 ntdll_77670000!KiUserCallbackDispatcher+0×2e
0e 0414c514 76ea789a user32!DispatchMessageWorker+0×442
0f 0414c524 5f69a97c user32!DispatchMessageW+0xf
10 0414f6f4 5f6dc648 ieframe!CTabWindow::_TabWindowThreadProc+0×464
11 0414f7b4 7543dbfc ieframe!LCIETab_ThreadProc+0×3e7
12 0414f7cc 6b303a31 iertutil!_IsoThreadProc_WrapperToReleaseScope+0×1c
13 0414f804 7696338a IEShims!NS_CreateThread::DesktopIE_ThreadProc+0×94
14 0414f810 776a9902 kernel32!BaseThreadInitThunk+0xe
15 0414f850 776a98d5 ntdll_77670000!__RtlUserThreadStart+0×70
16 0414f868 00000000 ntdll_77670000!_RtlUserThreadStart+0×1b

[...]

23  Id: be4.b88 Suspend: 0 Teb: 7ef4a000 Unfrozen
# ChildEBP RetAddr
00 0f3cbb60 76a815ce ntdll_77670000!NtWaitForSingleObject+0x15
01 0f3cbbcc 76961194 KERNELBASE!WaitForSingleObjectEx+0x98
02 0f3cbbe4 76961148 kernel32!WaitForSingleObjectExImplementation+0x75
03 0f3cbbf8 746a5aa5 kernel32!WaitForSingleObject+0x12
04 0f3cbcac 6aef2f6c dwmapi!DwmpDxGetWindowSharedSurface+0x374
05 0f3cc198 6aef1c2e dxgi!CDXGISwapChain::PresentImpl+0x6fa
06 0f3cc1f0 5d59339f dxgi!CDXGISwapChain::Present+0x5d
07 0f3cc214 5cea42bc mshtml!CDXSwapChainTargetSurface::OnPresent+0x2f
08 0f3cc240 5cea360d mshtml!CDXRenderTarget::Present+0×5c
09 0f3cc5c4 5cdbf5e1 mshtml!CPaintHandler::RenderInternal+0xad3
0a 0f3cc5f0 5ca1da8e mshtml!CPaintController::Render+0×39
0b 0f3cc630 5ce8692b mshtml!CRenderTaskDrawInPlace::Execute+0xc1
0c 0f3cc66c 5ca1c4c9 mshtml!CRenderTaskQueue::AddRenderTask+0xd2
0d 0f3cc6e0 5ca1a466 mshtml!CView::RenderInPlace+0×3cd
0e 0f3cc718 5ca1a4d4 mshtml!CDoc::PaintWorker+0×24d
0f 0f3cc738 5ca2983b mshtml!CDoc::PaintInPlace+0×40
10 0f3cc76c 5ca2978a mshtml!CPaintController::RunRenderingLoop+0×68
11 0f3cc790 5ca1180c mshtml!CPaintController::OnUpdateBeat+0×66
12 0f3cc7cc 5ca2a7af mshtml!CPaintBeat::OnBeat+0×234
13 0f3cc7f8 5c9bd27b mshtml!CPaintBeat::OnVSyncMethodCall+0×86
14 0f3cc840 5c9bc99c mshtml!GlobalWndOnMethodCall+0×17b
15 0f3cc894 76ea62fa mshtml!GlobalWndProc+0×103
16 0f3cc8c0 76ea6d3a user32!InternalCallWinProc+0×23
17 0f3cc938 76ea77d3 user32!UserCallWinProcCheckWow+0×109
18 0f3cc99c 76ea789a user32!DispatchMessageWorker+0×3cb
19 0f3cc9ac 5f69a97c user32!DispatchMessageW+0xf
1a 0f3cfb7c 5f6dc648 ieframe!CTabWindow::_TabWindowThreadProc+0×464
1b 0f3cfc3c 7543dbfc ieframe!LCIETab_ThreadProc+0×3e7
1c 0f3cfc54 6b303a31 iertutil!_IsoThreadProc_WrapperToReleaseScope+0×1c
1d 0f3cfc8c 7696338a IEShims!NS_CreateThread::DesktopIE_ThreadProc+0×94
1e 0f3cfc98 776a9902 kernel32!BaseThreadInitThunk+0xe
1f 0f3cfcd8 776a98d5 ntdll_77670000!__RtlUserThreadStart+0×70
20 0f3cfcf0 00000000 ntdll_77670000!_RtlUserThreadStart+0×1b

[...]

28  Id: be4.17c8 Suspend: 0 Teb: 7ef3d000 Unfrozen
# ChildEBP RetAddr
00 0ee2c988 776e2157 ntdll_77670000!NtWaitForKeyedEvent+0×15
01 0ee2c9f4 5c9c3f8e ntdll_77670000!RtlAcquireSRWLockExclusive+0×12e
02 0ee2ca0c 5cc69e25 mshtml!TSmartResource<CDXRenderLock>::Acquire<enum DXLock::Type>+0×5e
03 0ee2ca3c 5cc71743 mshtml!CDXSystemShared::PurgeResourceCaches+0×29
04 0ee2ca50 5cc7170d mshtml!CDXSystem::~CDXSystem+0×1d
05 0ee2ca5c 5cc716e6 mshtml!RefCounted<CDXSystem,SingleThreadedRefCount>::`vector deleting destructor’+0xd
06 0ee2ca6c 5c9bea0d mshtml!RefCounted<CDXSystem,SingleThreadedRefCount>::Release+0×2d
07 0ee2ca7c 5cc6a734 mshtml!TSmartPointer<Windows::Foundation::IAsyncOperation<Windows::Foundation:: Collections::IMapView<HSTRING__ *,Windows::Storage::Streams::RandomAccessStreamReference *> *> >::~TSmartPointer<Windows::Foundation::IAsyncOperation<Windows::Foundation:: Collections::IMapView<HSTRING__ *,Windows::Storage::Streams::RandomAccessStreamReference *> *> >+0×1d
08 0ee2ca84 5cc6a6e0 mshtml!DeinitRenderSystem+0×14
09 0ee2caa0 5cc6a63a mshtml!DeinitThreadStateStruct<void (__stdcall*)(EXTENDEDTHREADSTATE *),EXTENDEDTHREADSTATE>+0×22
0a 0ee2cab0 5d272f1d mshtml!DeinitExtendedThreadState+0×24
0b 0ee2cacc 5cabd85e mshtml!_DecrementObjectCount+0×79
0c 0ee2cad8 5f78b662 mshtml!CBaseLockCF::LockServer+0×4a
0d 0ee2cafc 5f797c4d ieframe!CBaseBrowser2::~CBaseBrowser2+0×265
0e 0ee2cb08 5f797c2e ieframe!CShellBrowser2::`vector deleting destructor’+0xd
0f 0ee2cb24 5f78e7bf ieframe!CShellBrowser2::Release+0×47
10 0ee2fcf4 5f6dc648 ieframe!CTabWindow::_TabWindowThreadProc+0×83e
11 0ee2fdb4 7543dbfc ieframe!LCIETab_ThreadProc+0×3e7
12 0ee2fdcc 6b303a31 iertutil!_IsoThreadProc_WrapperToReleaseScope+0×1c
13 0ee2fe04 7696338a IEShims!NS_CreateThread::DesktopIE_ThreadProc+0×94
14 0ee2fe10 776a9902 kernel32!BaseThreadInitThunk+0xe
15 0ee2fe50 776a98d5 ntdll_77670000!__RtlUserThreadStart+0×70
16 0ee2fe68 00000000 ntdll_77670000!_RtlUserThreadStart+0×1b

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

This is another variant of Invalid Parameter pattern where an invalid parameter is passed to either statically or dynamically linked C runtime function. Exception codes may vary, for example, c000000d and c0000417 as the following examples illustrate:

0:000> k
Child-SP          RetAddr           Call Site
00000000`0289e858 00007ff8`cf4f13ed ntdll!NtWaitForMultipleObjects+0xa
00000000`0289e860 00007ff8`d1157d51 KERNELBASE!WaitForMultipleObjectsEx+0xe1
00000000`0289eb40 00007ff8`d1157773 kernel32!WerpReportFaultInternal+0x581
00000000`0289f0b0 00007ff8`cf5d1cdf kernel32!WerpReportFault+0x83
00000000`0289f0e0 00007ff6`f0225095 KERNELBASE!UnhandledExceptionFilter+0×23f
00000000`0289f1d0 00007ff6`f021ec89 ModuleA!_invalid_parameter+0xc5
00000000`0289f790 00007ff6`f01a2636 ModuleA!vswprintf_s+0×79
[…]
00000000`0289fdc0 00007ff8`d20b5454 kernel32!BaseThreadInitThunk+0×22
00000000`0289fdf0 00000000`00000000 ntdll!RtlUserThreadStart+0×34

0:000> !analyze -v
[...]
EXCEPTION_RECORD:  ffffffffffffffff -- (.exr 0xffffffffffffffff)
ExceptionAddress: 00007ff6f021ec89 (ModuleA!vswprintf_s+0x0000000000000079)
ExceptionCode: c000000d
ExceptionFlags: 00000000
NumberParameters: 0
[…]
EXCEPTION_CODE: (NTSTATUS) 0xc000000d - An invalid parameter was passed to a service or function.

0:000> k
# ChildEBP RetAddr
00 002e6de8 74a01606 ntdll!NtWaitForMultipleObjects+0xc
01 002e6f7c 74a014b8 KERNELBASE!WaitForMultipleObjectsEx+0x136
02 002e6f98 7431a02a KERNELBASE!WaitForMultipleObjects+0x18
03 002e73d4 74319ac6 kernel32!WerpReportFaultInternal+0x545
04 002e73e4 742fcf09 kernel32!WerpReportFault+0x7a
05 002e73ec 74a1a013 kernel32!BasepReportFault+0x19
06 002e7480 73edcc2c KERNELBASE!UnhandledExceptionFilter+0×1b3
07 002e77b8 73ea75b4 msvcr90!_invoke_watson+0xf9
08 002e77e0 5f29a294 msvcr90!wcsncpy_s+0×41
09 002e77fc 5f434563 ModuleB+0×6a294
[…]
44 002efda8 76f19e54 kernel32!BaseThreadInitThunk+0×24
45 002efdf0 76f19e1f ntdll!__RtlUserThreadStart+0×2f
46 002efe00 00000000 ntdll!_RtlUserThreadStart+0×1b

0:000> !analyze -v
[...]
EXCEPTION_RECORD:  (.exr -1)
ExceptionAddress: 73ea75b4 (msvcr90!wcsncpy_s+0x00000041)
ExceptionCode: c0000417
ExceptionFlags: 00000001
NumberParameters: 0
[…]
EXCEPTION_CODE: (NTSTATUS) 0xc0000417 - An invalid parameter was passed to a C runtime function.

We can inspect parameters passed to the function where it is possible. In 32-bit memory dumps we may see the first 3 parameters in the output of kv WinDbg command. More than 3 function parameters require the analysis of the raw stack region. 64-bit memory dumps may require disassembled code analysis of the caller (via ub WinDbg command).

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

In one .NET process memory dump we noticed Managed Stack Trace pointing that the thread was waiting for .NET Finalizer to finish its work:

0:000> !CLRStack
OS Thread Id: 0x60b0 (0)
ESP       EIP
00a585d8 773dca2c [HelperMethodFrame: 00a585d8] System.GC.WaitForPendingFinalizers()
00a58628 10105bbb ComponentA.FreeMemory()

The unmanaged stack trace involved Windows forms and COM object modal loops:

0:000> k
ChildEBP RetAddr
00a58034 7644112f ntdll!NtWaitForMultipleObjects+0xc
00a581c0 75ecd433 KERNELBASE!WaitForMultipleObjectsEx+0xcc
00a58224 7655fab4 user32!MsgWaitForMultipleObjectsEx+0x163
00a5825c 7656097c combase!CCliModalLoop::BlockFn+0×111
00a58314 765591fa combase!ClassicSTAThreadWaitForHandles+0×9e
00a58344 6e59103d combase!CoWaitForMultipleHandles+0×84
00a58364 6e590f9e mscorwks!NT5WaitRoutine+0×51
00a583d0 6e590f02 mscorwks!MsgWaitHelper+0xa5
00a583f0 6e6665e8 mscorwks!Thread::DoAppropriateAptStateWait+0×28
00a58474 6e66667d mscorwks!Thread::DoAppropriateWaitWorker+0×13c
00a584c4 6e666801 mscorwks!Thread::DoAppropriateWait+0×40
00a58520 6e5056b8 mscorwks!CLREvent::WaitEx+0xf7
00a58534 6e59eea3 mscorwks!CLREvent::Wait+0×17
00a58584 6e59ec19 mscorwks!WKS::GCHeap::FinalizerThreadWait+0xfb
00a58620 10105bbb mscorwks!GCInterface::RunFinalizers+0×99
00a58684 6cd8683b 0×10105bbb
00a586a0 6c4f5a52 System_Windows_Forms_ni+0xa5683b
00a586e4 6cd8e8f4 System_Windows_Forms_ni+0×1c5a52
00a586f8 6c529a00 System_Windows_Forms_ni+0xa5e8f4
00a58700 6c529981 System_Windows_Forms_ni+0×1f9a00
00a58714 6c52985a System_Windows_Forms_ni+0×1f9981
00a58778 75ec8e71 System_Windows_Forms_ni+0×1f985a
00a587a4 75ec90d1 user32!_InternalCallWinProc+0×2b
00a58838 75ec932c user32!UserCallWinProcCheckWow+0×18e
00a58898 75ec9529 user32!DispatchClientMessage+0xdc
00a588d8 773e07d6 user32!__fnDWORD+0×49
00a588ec 01293a30 ntdll!KiUserCallbackDispatcher+0×36
00a58998 75ecb0f8 0×1293a30
00a589e8 75ec8e71 user32!DefWindowProcW+0×118
00a58a14 75ec90d1 user32!_InternalCallWinProc+0×2b
00a58aa8 75ecddd5 user32!UserCallWinProcCheckWow+0×18e
00a58ae4 6c52a033 user32!CallWindowProcW+0×8d
00a58b3c 6c5331a9 System_Windows_Forms_ni+0×1fa033
00a58b84 6c529b1e System_Windows_Forms_ni+0×2031a9
00a58bdc 6c53310a System_Windows_Forms_ni+0×1f9b1e
00a58be8 6c5330c0 System_Windows_Forms_ni+0×20310a
00a58bf0 6c532d80 System_Windows_Forms_ni+0×2030c0
00a58c04 6c529a00 System_Windows_Forms_ni+0×202d80
00a58c0c 6c529981 System_Windows_Forms_ni+0×1f9a00
00a58c20 6c52985a System_Windows_Forms_ni+0×1f9981
00a58c84 75ec8e71 System_Windows_Forms_ni+0×1f985a
00a58cb0 75ec90d1 user32!_InternalCallWinProc+0×2b
00a58d44 75ec932c user32!UserCallWinProcCheckWow+0×18e
00a58da4 75ec9529 user32!DispatchClientMessage+0xdc
00a58de4 773e07d6 user32!__fnDWORD+0×49
00a58df8 011efaa0 ntdll!KiUserCallbackDispatcher+0×36
00a58e48 75eca989 0×11efaa0
00a58e7c 765603c3 user32!PeekMessageW+0×135
00a58ed0 76560436 combase!CCliModalLoop::MyPeekMessage+0×31
00a58f10 76560630 combase!CCliModalLoop::PeekRPCAndDDEMessage+0×31
00a58f3c 76560506 combase!CCliModalLoop::FindMessage+0×2b
00a58fa0 765605fb combase!CCliModalLoop::HandleWakeForMsg+0×44
00a58fc4 7656097c combase!CCliModalLoop::BlockFn+0×19d
00a5907c 765591fa combase!ClassicSTAThreadWaitForHandles+0×9e
00a590ac 6e59103d combase!CoWaitForMultipleHandles+0×84
00a590cc 6e590f9e mscorwks!NT5WaitRoutine+0×51
00a59138 6e590f02 mscorwks!MsgWaitHelper+0xa5
00a59158 6e6665e8 mscorwks!Thread::DoAppropriateAptStateWait+0×28
00a591dc 6e66667d mscorwks!Thread::DoAppropriateWaitWorker+0×13c
00a5922c 6e666801 mscorwks!Thread::DoAppropriateWait+0×40
00a59288 6e5056b8 mscorwks!CLREvent::WaitEx+0xf7
00a5929c 6e59eea3 mscorwks!CLREvent::Wait+0×17
00a592ec 6e59ec19 mscorwks!WKS::GCHeap::FinalizerThreadWait+0xfb
00a59388 10105bbb mscorwks!GCInterface::RunFinalizers+0×99
00a593ec 6cd8683b 0×10105bbb
00a59408 6ca863bd System_Windows_Forms_ni+0xa5683b
00a59438 6cd90b7e System_Windows_Forms_ni+0×7563bd
00a59480 6c539eee System_Windows_Forms_ni+0xa60b7e
00a5951c 6c539bf7 System_Windows_Forms_ni+0×209eee
00a59574 6c539a41 System_Windows_Forms_ni+0×209bf7
00a595a4 6ca4fc97 System_Windows_Forms_ni+0×209a41
00a595bc 6ca84598 System_Windows_Forms_ni+0×71fc97
00a59648 6ca847af System_Windows_Forms_ni+0×754598
00a59bc8 0996d41c System_Windows_Forms_ni+0×7547af
00a59c2c 0bc02e6d 0×996d41c
00a59c5c 0996b842 0xbc02e6d
00a59fb8 0fa8fa87 0×996b842
00a5a00c 0fa8f205 0xfa8fa87
00a5a040 0fa8f10b 0xfa8f205
00a5a058 02676666 0xfa8f10b
00a5a094 0fa8f09f 0×2676666
00a5a0a8 0fa8ed0c 0xfa8f09f
00a5a1b8 6dc355dc 0xfa8ed0c
00a5a238 6ca59703 mscorlib_ni+0×2355dc
00a5a2bc 6cd8c376 System_Windows_Forms_ni+0×729703
00a5a31c 6c529a00 System_Windows_Forms_ni+0xa5c376
00a5a324 6c529981 System_Windows_Forms_ni+0×1f9a00
00a5a338 6c52985a System_Windows_Forms_ni+0×1f9981
00a5a378 6cd8c587 System_Windows_Forms_ni+0×1f985a
00a5a3c8 6c511125 System_Windows_Forms_ni+0xa5c587
00a5a3f0 6c52985a System_Windows_Forms_ni+0×1e1125
00a5a454 75ec8e71 System_Windows_Forms_ni+0×1f985a
00a5a480 75ec90d1 user32!_InternalCallWinProc+0×2b
00a5a514 75eca66f user32!UserCallWinProcCheckWow+0×18e
00a5a580 75eca6e0 user32!DispatchMessageWorker+0×208
00a5a58c 0f1c6f4e user32!DispatchMessageW+0×10
00a5a5c0 0f1c58ed ModuleA+0×13c4e
[…]

We found Finalizer Special Thread:

0:002> k
ChildEBP RetAddr
0316ecc8 76432cc7 ntdll!NtWaitForSingleObject+0xc
0316ed3c 76432c02 KERNELBASE!WaitForSingleObjectEx+0x99
0316ed50 765b9839 KERNELBASE!WaitForSingleObject+0x12
(Inline) -------- combase!MTAThreadWaitForCall+0x43
0316ed80 7665d524 combase!MTAThreadDispatchCrossApartmentCall+0x1ed
(Inline) -------- combase!CRpcChannelBuffer::SwitchAptAndDispatchCall+0x33a2
0316eed4 7653caea combase!CRpcChannelBuffer::SendReceive2+0×62d
(Inline) ——– combase!ClientCallRetryContext::SendReceiveWithRetry+0×2e
(Inline) ——– combase!CAptRpcChnl::SendReceiveInRetryContext+0×81
0316ef34 76581789 combase!DefaultSendReceive+0×9e
(Inline) ——– combase!CAptRpcChnl::SendReceive+0×38
0316ef98 7665a010 combase!CCtxComChnl::SendReceive+0×248
0316efbc 76f85769 combase!NdrExtpProxySendReceive+0×5c
0316efd4 77006c1b rpcrt4!NdrpProxySendReceive+0×29
0316f400 76659e1e rpcrt4!NdrClientCall2+0×22b
0316f420 7653c46f combase!ObjectStublessClient+0×6c
0316f430 76581c07 combase!ObjectStubless+0xf
0316f4c4 76583024 combase!CObjectContext::InternalContextCallback+0×1e4
0316f518 6e6fcd4b combase!CObjectContext::ContextCallback+0xbc
0316f564 6e6fd936 mscorwks!CtxEntry::EnterContextOle32BugAware+0×2b
0316f684 6e7329cf mscorwks!CtxEntry::EnterContext+0×325
0316f6b8 6e732a74 mscorwks!RCWCleanupList::ReleaseRCWListInCorrectCtx+0xc4
0316f708 6e5908e3 mscorwks!RCWCleanupList::CleanupAllWrappers+0xdb
0316f74c 6e5907f3 mscorwks!SyncBlockCache::CleanupSyncBlocks+0xec
0316f910 6e589b7c mscorwks!Thread::DoExtraWorkForFinalizer+0×40
0316f920 6e54547f mscorwks!WKS::GCHeap::FinalizerThreadWorker+0×9a
0316f934 6e54541b mscorwks!Thread::DoADCallBack+0×32a
0316f9c8 6e54533a mscorwks!Thread::ShouldChangeAbortToUnload+0xe3
0316fa04 6e5b2763 mscorwks!Thread::ShouldChangeAbortToUnload+0×30a
0316fa2c 6e5b2774 mscorwks!ManagedThreadBase_NoADTransition+0×32
0316fa3c 6e5cb816 mscorwks!ManagedThreadBase::FinalizerBase+0xd
0316fa74 6e660ae9 mscorwks!WKS::GCHeap::FinalizerThreadStart+0xbb
0316fb10 76307c04 mscorwks!Thread::intermediateThreadProc+0×49
0316fb24 773fad1f kernel32!BaseThreadInitThunk+0×24
0316fb6c 773facea ntdll!__RtlUserThreadStart+0×2f
0316fb7c 00000000 ntdll!_RtlUserThreadStart+0×1b

We saw that it was blocked in an RPC call. Using the technique from In Search of Lost CID article we found the destination thread:

0:002> kvL
ChildEBP RetAddr  Args to Child
0316ecc8 76432cc7 0000096c 00000000 00000000 ntdll!NtWaitForSingleObject+0xc
0316ed3c 76432c02 0000096c ffffffff 00000000 KERNELBASE!WaitForSingleObjectEx+0x99
0316ed50 765b9839 0000096c ffffffff 00ddfb88 KERNELBASE!WaitForSingleObject+0x12
(Inline) -------- -------- -------- -------- combase!MTAThreadWaitForCall+0x43
0316ed80 7665d524 00ddf298 0629fa10 0316efec combase!MTAThreadDispatchCrossApartmentCall+0x1ed
(Inline) -------- -------- -------- -------- combase!CRpcChannelBuffer::SwitchAptAndDispatchCall+0x33a2
0316eed4 7653caea 00ddfb88 0316efec 0316efb4 combase!CRpcChannelBuffer::SendReceive2+0×62d
(Inline) ——– ——– ——– ——– combase!ClientCallRetryContext::SendReceiveWithRetry+0×2e
(Inline) ——– ——– ——– ——– combase!CAptRpcChnl::SendReceiveInRetryContext+0×81
0316ef34 76581789 00ddfb88 0316efec 0316efb4 combase!DefaultSendReceive+0×9e
(Inline) ——– ——– ——– ——– combase!CAptRpcChnl::SendReceive+0×38
0316ef98 7665a010 00ddfb88 0316efec 0316efb4 combase!CCtxComChnl::SendReceive+0×248
0316efbc 76f85769 0628365c 0316f018 76f85740 combase!NdrExtpProxySendReceive+0×5c
0316efd4 77006c1b 6d6b47ca 0628365c 0316f438 rpcrt4!NdrpProxySendReceive+0×29
0316f400 76659e1e 7652d8f8 76532ff0 0316f438 rpcrt4!NdrClientCall2+0×22b
[…]

0:002> dpp 00ddfb88
00ddfb88  7652d400 7665bff0 combase!CRpcChannelBuffer::QueryInterface
00ddfb8c  76541e2c 76664690 combase![thunk]:CRpcChannelBuffer::QueryInterface`adjustor{4}’
00ddfb90  7653a65c 766646a0 combase![thunk]:CRpcChannelBuffer::QueryInterface`adjustor{8}’
00ddfb94  adb1682c
00ddfb98  429b3908
00ddfb9c  2b59b182
00ddfba0  e6a936e1
00ddfba4  00000003
00ddfba8  0000002a
00ddfbac  06268d68 00000000
00ddfbb0  00000000
00ddfbb4  00ddf298 00ddf330
00ddfbb8  00ddfe70 00000000
00ddfbbc  062d7660 00000044
00ddfbc0  00deb5f0 7652d280 combase!CStdIdentity::`vftable’
00ddfbc4  7652b2e0 76568760 combase!CDestObject::QueryInterface
00ddfbc8  00070005
00ddfbcc  00000003
00ddfbd0  000060b0
00ddfbd4  00000000
00ddfbd8  00000000
00ddfbdc  7652d400 7665bff0 combase!CRpcChannelBuffer::QueryInterface
00ddfbe0  76541e2c 76664690 combase![thunk]:CRpcChannelBuffer::QueryInterface`adjustor{4}’
00ddfbe4  7653a65c 766646a0 combase![thunk]:CRpcChannelBuffer::QueryInterface`adjustor{8}’
00ddfbe8  adb1682c
00ddfbec  429b3908

0:002> dd 00ddf298
00ddf298  00ddf330 7666ddd0 000070b8 000060b0
00ddf2a8  056aadfb 14bb2b53 a45b843e 2611fcc9
00ddf2b8  a45b843e 2611fcc9 00001800 60b070b8
00ddf2c8  66fb7794 88242aca 04000203 00020052
00ddf2d8  00dbfb20 00000000 00000000 00000000
00ddf2e8  00000001 ffffffff 00de5e48 0628365c
00ddf2f8  00000005 00000000 00000001 00000000
00ddf308  00000000 00070005 00000000 00000000

We saw that the finalizer thread #2 was waiting for the thread #0 that was waiting for the finalizer thread #2:

0:002> ~
#  0  Id: 70b8.60b0 Suspend: 0 Teb: 7ecaf000 Unfrozen
1  Id: 70b8.26e8 Suspend: 0 Teb: 7ecac000 Unfrozen
.  2  Id: 70b8.5d2c Suspend: 0 Teb: 7eca9000 Unfrozen
3  Id: 70b8.551c Suspend: 0 Teb: 7eb76000 Unfrozen
4  Id: 70b8.7194 Suspend: 0 Teb: 7eb73000 Unfrozen
5  Id: 70b8.465c Suspend: 0 Teb: 7eb0f000 Unfrozen
6  Id: 70b8.64ac Suspend: 0 Teb: 7eb0c000 Unfrozen
7  Id: 70b8.6200 Suspend: 0 Teb: 7eb09000 Unfrozen
8  Id: 70b8.5790 Suspend: 0 Teb: 7eafa000 Unfrozen
9  Id: 70b8.6c3c Suspend: 0 Teb: 7eaf4000 Unfrozen
10  Id: 70b8.7320 Suspend: 0 Teb: 7eaeb000 Unfrozen
11  Id: 70b8.1ebc Suspend: 0 Teb: 7eb06000 Unfrozen
12  Id: 70b8.459c Suspend: 0 Teb: 7eb00000 Unfrozen
13  Id: 70b8.74e8 Suspend: 0 Teb: 7eb7f000 Unfrozen
14  Id: 70b8.6758 Suspend: 0 Teb: 7eafd000 Unfrozen
15  Id: 70b8.72e8 Suspend: 0 Teb: 7eaf1000 Unfrozen
16  Id: 70b8.5eec Suspend: 0 Teb: 7eaee000 Unfrozen
17  Id: 70b8.4a74 Suspend: 0 Teb: 7e83f000 Unfrozen
18  Id: 70b8.61b0 Suspend: 0 Teb: 7e83c000 Unfrozen
19  Id: 70b8.3cc4 Suspend: 0 Teb: 7e839000 Unfrozen
20  Id: 70b8.6554 Suspend: 0 Teb: 7e836000 Unfrozen
21  Id: 70b8.5b5c Suspend: 0 Teb: 7e833000 Unfrozen
22  Id: 70b8.6c48 Suspend: 0 Teb: 7e7ff000 Unfrozen
23  Id: 70b8.12dc Suspend: 0 Teb: 7e7fc000 Unfrozen
24  Id: 70b8.3a98 Suspend: 0 Teb: 7e7f9000 Unfrozen
25  Id: 70b8.1cb4 Suspend: 0 Teb: 7e7f6000 Unfrozen
26  Id: 70b8.5df8 Suspend: 0 Teb: 7e7f3000 Unfrozen
27  Id: 70b8.287c Suspend: 0 Teb: 7e69f000 Unfrozen
28  Id: 70b8.69b4 Suspend: 0 Teb: 7e69c000 Unfrozen
29  Id: 70b8.159c Suspend: 0 Teb: 7e699000 Unfrozen
30  Id: 70b8.1678 Suspend: 0 Teb: 7e696000 Unfrozen
31  Id: 70b8.8a0 Suspend: 0 Teb: 7e693000 Unfrozen
32  Id: 70b8.5984 Suspend: 0 Teb: 7e64f000 Unfrozen
33  Id: 70b8.256c Suspend: 0 Teb: 7e649000 Unfrozen

This is a variant of Deadlock memory analysis pattern.

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -

When the same API function is executed it may leave the same data structure in its Execution Residue inside raw stack regions of different threads. We call this analysis pattern Frame Regularity. It may be used to find and correctly interpret stack frame values from thread stack traces or regions based on values found in some other thread stack trace or region where we already know their meaning.

Consider the critical section Wait Chain below. We want to know what critical section the thread F34 is waiting for. The backwards disassembly of 00007ffd`41894e14 return address doesn’t show the critical section address explicitly. However, 00007ffd`43827e40 return address disassembly from a different critical section thread owner D7C shows critical section address 00007ffd`4391d8a8 which we can find as a 3rd “argument” to ntdll!RtlpWaitOnCriticalSection (which is just a value picked up from the stack region and not the real function argument passed by stack). Using Frame Regularity we try to see if the 3rd value for the same function from the thread F34 under question is also a critical section address 00000071`3f08c488. Indeed, we find that it is the address of the critical section owned by the thread B18.

0:000> !cs -l -o -s
-----------------------------------------
DebugInfo          = 0x00007ffd4391dbe8
Critical section   = 0x00007ffd4391d8a8 (ntdll!LdrpLoaderLock+0x0)
LOCKED
LockCount          = 0x13
WaiterWoken        = No
OwningThread       = 0×0000000000000f34
RecursionCount     = 0×1
LockSemaphore      = 0×8B0
SpinCount          = 0×0000000004000000
OwningThread DbgId = ~14s
OwningThread Stack =
Child-SP          RetAddr           : Args to Child                                                           : Call Site
00000071`40d4c618 00007ffd`43829a85 : 00000071`3f08c488 00000000`00000002 00000000`bd000200 00000000`00000000 : ntdll!NtWaitForSingleObject+0xa
00000071`40d4c620 00007ffd`43827f44 : 00000000`a90002ab 00007ffd`40d52eba 00000071`3f08c488 00000000`00000000 : ntdll!RtlpWaitOnCriticalSection+0xe1
00000071`40d4c6f0 00007ffd`41894e14 : 00000071`3f08c488 00007ffd`4229b3a8 00000000`00000000 00000071`3f08c470 : ntdll!RtlpEnterCriticalSectionContended+0xa4
00000071`40d4c730 00007ffd`41895d8d : 00000071`40c52ab0 00000000`00000000 00230022`00210020 00270026`00250024 : shell32!kfapi::CFolderPathResolver::GetPath+0×168
00000071`40d4dd50 00007ffd`41892d50 : 00000000`00000080 00000000`000000ce 00000000`00000104 00000000`000003ff : shell32!kfapi::CFolderCache::GetPath+0×772
00000071`40d4e0c0 00007ffd`41893540 : 00000071`40d4e258 00000000`20000020 00000000`000000ce 00000071`42141080 : shell32!kfapi::CKFFacade::GetFolderPath+0×120
00000071`40d4e220 00007ffd`4189368c : 00000000`00008023 00000000`000000ce 00000000`5307cc38 00000000`000000ce : shell32!SHGetFolderPathEx+0×80
00000071`40d4e290 00007ffd`40d61939 : 00000000`00000000 00000000`00000000 00000000`00000037 00000000`00008023 : shell32!SHGetFolderPathWWorker+0xf9
00000071`40d4e2f0 00007ffd`41a80fba : 00000000`00000000 00000000`00000000 00000071`40d4e400 00000071`42010000 : KERNELBASE!SHGetFolderPathW+0×49
00000071`40d4e330 00007ffd`40d7e60f : 00000071`00000001 00000000`00000020 00000000`00000035 00000071`42141080 : shell32!SHGetSpecialFolderPathWWorker+0×23
00000071`40d4e370 00000000`5317704d : 00000071`40d4e7d0 00000071`40d4ee68 00000000`00000000 00000000`53062b30 : KERNELBASE!SHGetSpecialFolderPathW+0×3f
00000071`40d4e3a0 00000000`530e72f1 : 00000000`5333fa10 00000000`00000001 00000000`5333fa10 00000000`530ab470 : ModuleA+0xccded
[…]
00000071`40d4eba0 00007ffd`4380e82e : 00000071`40cc6780 00000000`53060000 00000071`00000001 00007ffd`40892421 : ntdll!LdrpCallInitRoutine+0×4c
00000071`40d4ec00 00007ffd`4380e624 : 00000071`40cb2790 00007ffd`4384ee00 00000071`40cb2790 00000000`53224dec : ntdll!LdrpInitializeNode+0×176
ntdll!RtlpStackTraceDataBase is NULL. Probably the stack traces are not enabled.
—————————————–
DebugInfo          = 0×0000007140c52a70
Critical section   = 0×00007ffd31f41a88 (localspl!g_csBidiAccess+0×8)
LOCKED
LockCount          = 0×4
WaiterWoken        = No
OwningThread       = 0×0000000000000d7c
RecursionCount     = 0×1
LockSemaphore      = 0×1410
SpinCount          = 0×00000000020007d0
OwningThread DbgId = ~7s
OwningThread Stack =
Child-SP          RetAddr           : Args to Child                                                           : Call Site
00000071`4029e708 00007ffd`43829a85 : 0000a498`9a4b3f83 00000071`4029ec00 00000071`4029e880 00000071`4029e858 : ntdll!NtWaitForSingleObject+0xa
00000071`4029e710 00007ffd`43827f44 : 00000000`00000002 00007ffd`00000000 00007ffd`4391d8a8 00000071`4029e9d0 : ntdll!RtlpWaitOnCriticalSection+0xe1
00000071`4029e7e0 00007ffd`43827e40 : 00007ffd`41613750 00000071`3efa2a10 00007ffd`416b4d58 00007ffd`41686000 : ntdll!RtlpEnterCriticalSectionContended+0xa4
00000071`4029e820 00007ffd`4380ad5a : 00007ffd`41686000 00000071`4029e8e0 00007ffd`41613750 00000071`4029eeb8 : ntdll!LdrpAcquireLoaderLock+0×2c
00000071`4029e860 00007ffd`4386785f : 00000000`00000000 00007ffd`41685c64 00000000`00000000 00000071`4029e980 : ntdll!LdrpGetDelayloadExportDll+0×7e
00000071`4029e920 00007ffd`43826c8e : 00000000`00000000 00007ffd`41685c64 00000000`00000000 00007ffd`41613750 : ntdll!LdrpHandleProtectedDelayload+0×3f
00000071`4029ede0 00007ffd`4153763b : 00000071`4029ef50 00000000`00000000 00007ffd`2f4d3be0 00000000`00000003 : ntdll!LdrResolveDelayLoadedAPI+0×8e
00000071`4029ee40 00007ffd`415376df : 00000071`3ffe3020 00007ffd`2f4d2482 00000071`3ffe3020 00000071`3ffcb078 : combase!__delayLoadHelper2+0×2b
00000071`4029ee80 00007ffd`415239b6 : a4e9da62`d9130000 00000071`4029eed0 00000000`00000000 00000000`0000006c : combase!_tailMerge_OLEAUT32_dll+0×3f
00000071`4029eef0 00007ffd`41523883 : 00000071`40c22230 00000071`4029f029 00000000`80070015 00000071`3ffcf598 : combase!wCoUninitialize+0xb6
00000071`4029ef20 00007ffd`31e74e1a : 00000071`40c22230 00000000`00000000 00000000`80070015 00007ffd`2f4d3be0 : combase!CoUninitialize+0xb3
00000071`4029ef50 00007ffd`31e7459c : 00000000`00000000 00000071`4029f1d8 00007ffd`31f416d8 00000000`00000001 : localspl!GetBidiStrings+0×3be
00000071`4029f090 00007ffd`31e72150 : 00000000`00000000 00000071`4029f240 00007ffd`31f416d8 00000000`00000000 : localspl!GetBidiPropertiesForDevnode+0×9c
00000071`4029f140 00007ffd`31e7b03b : 00000000`00000008 00000071`403588e0 00000071`40a83130 00000000`00000001 : localspl!GetPropertiesFromPrinter+0×1f4
00000071`4029f750 00007ffd`31e7ae90 : 00000071`40aa6e40 00000000`00000000 00000000`00000001 00007ffd`ffff4b5e : localspl!CreatePrintQueueDevnodeWorker+0xab
00000071`4029f8a0 00007ffd`31eb475e : 00000000`00000000 00000000`00000000 00000071`3f05f470 00000000`00000000 : localspl!CreatePrintQueueDevnode+0xb4
00000071`4029f920 00007ffd`31eb46eb : 00000071`3ffe35c0 00000071`3f05f470 00007ffd`31eb4710 00000000`00000000 : localspl!DevnodeCreationWorkItem::Run+0×4e
00000071`4029f990 00007ffd`4386247e : 00007ffd`31eb46b0 00000071`3f05f3c0 00000071`4029fb78 00000071`3f05f470 : localspl!NThreadingLibrary::TWorkCrew::tpSimpleCallback+0×3b
00000071`4029f9c0 00007ffd`43824a5d : 00000071`3f05f470 00000071`3efd23f0 00000000`00000000 00000071`3f01e3e8 : ntdll!TppSimplepExecuteCallback+0×7e
00000071`4029fa00 00007ffd`410313d2 : 00000000`00000000 00007ffd`438245e0 00000071`3f01bf10 00000000`00000000 : ntdll!TppWorkerThread+0×47d
ntdll!RtlpStackTraceDataBase is NULL. Probably the stack traces are not enabled.
—————————————–
DebugInfo          = 0×0000007140c52970
Critical section   = 0×0000000001017600 (ModuleB+0×38F80)
LOCKED
LockCount          = 0×1
WaiterWoken        = No
OwningThread       = 0×0000000000000fe8
RecursionCount     = 0×1
LockSemaphore      = 0×14A0
SpinCount          = 0×00000000020007d0
OwningThread DbgId = ~22s
OwningThread Stack =
Child-SP          RetAddr           : Args to Child                                                           : Call Site
00000071`4102e6a8 00007ffd`43829a85 : 00000000`00000050 00007ffd`4381c937 00000071`3f110000 00000000`00000000 : ntdll!NtWaitForSingleObject+0xa
00000071`4102e6b0 00007ffd`43827f44 : 00000000`00000000 00000000`00000000 00007ffd`4391d8a8 00000000`00000000 : ntdll!RtlpWaitOnCriticalSection+0xe1
00000071`4102e780 00007ffd`43827e40 : 00000071`4102ebe8 00000000`00000000 00000071`4102ebb8 00000071`40cc5a00 : ntdll!RtlpEnterCriticalSectionContended+0xa4
00000071`4102e7c0 00007ffd`43818998 : 00007ff7`acadf000 00000071`4102e830 00000071`4102e830 00000071`4102e830 : ntdll!LdrpAcquireLoaderLock+0×2c
00000071`4102e800 00007ffd`4380b229 : 00000000`00000000 00000071`4102ee40 00007ffd`437f4d38 00000000`00000000 : ntdll!LdrpFindOrMapDll+0×4f8
00000071`4102eb50 00007ffd`4380af29 : 00000000`00000000 00000000`00000000 00000071`4102ee40 00000071`4102ee40 : ntdll!LdrpLoadDll+0×295
00000071`4102ed80 00007ffd`40d58e4a : 00000000`00000000 00000071`4102ee50 00000000`010154a0 00000000`00000002 : ntdll!LdrLoadDll+0×99
00000071`4102ee00 00000000`00fd36ec : 00000000`00000000 00000000`010020a0 00000000`010154a0 00000000`00000000 : KERNELBASE!LoadLibraryExW+0xca
00000071`4102ee70 00000000`00fd3aa6 : 00000071`41147420 00000000`028424c0 00000000`00000001 00000071`411473dc : ModuleB+0×1f1c
[…]
00000071`4102f5b0 00007ffd`3a9b5804 : 00000071`40c33ea8 00007ff7`00000008 00000000`00000008 00000071`3ffff190 : winspool!SpoolerPrinterEventNative+0xb1
00000071`4102f620 00007ffd`31a5566d : 00000071`40c33ea8 00000071`4102f719 00000000`00000008 00000000`00000000 : winspool!SpoolerPrinterEvent+0×54
00000071`4102f660 00007ffd`31e76682 : 00007ffd`31a55480 00000071`40c33ea8 00000071`4102f7c0 00000071`40aabcc0 : PrintIsolationProxy!sandbox::PrintSandboxObject::SandboxDriverEvent+0×1ed
00000071`4102f760 00007ffd`31e7659a : 00000071`3ffaf750 00007ffd`31e76610 00000071`40aabcc0 00000000`00000008 : localspl!sandbox::SandboxObserver::SandboxDriverEvent+0×72
00000071`4102f7b0 00007ffd`31e7621b : 00000071`40c33ea8 00000000`00000000 00000071`40c33ea8 00000000`00000000 : localspl!sandbox::SandboxPrinterDriverEvent+0xda
00000071`4102f820 00007ffd`31e76169 : 00000071`40aa6e40 00000000`00000001 00000000`00000000 00007ffd`31f40001 : localspl!SplDriverEvent+0×5b
00000071`4102f890 00007ffd`31f11c4b : 00000000`00000000 00000071`4102f940 00000071`40aa6e40 00007ffd`31f400c8 : localspl!PrinterDriverEvent+0xbd
ntdll!RtlpStackTraceDataBase is NULL. Probably the stack traces are not enabled.
—————————————–
DebugInfo          = 0×0000007140c52630
Critical section   = 0×000000713f08c488 (+0×713F08C488)
LOCKED
LockCount          = 0×1
WaiterWoken        = No
OwningThread       = 0×0000000000000b18
RecursionCount     = 0×1
LockSemaphore      = 0×14AC
SpinCount          = 0×00000000020007d0
OwningThread DbgId = ~1s
OwningThread Stack =
Child-SP          RetAddr           : Args to Child                                                           : Call Site
00000071`3fe4b5e8 00007ffd`43829a85 : 00000001`00000000 00000000`00000000 00000071`41927fd0 00007ffd`43821067 : ntdll!NtWaitForSingleObject+0xa
00000071`3fe4b5f0 00007ffd`43827f44 : 00000000`00000000 00000000`0000017b 00007ffd`4391d8a8 00000000`00000000 : ntdll!RtlpWaitOnCriticalSection+0xe1
00000071`3fe4b6c0 00007ffd`43827e40 : 00007ffd`41c57850 00000071`3efe3700 00007ffd`4229bf90 00007ffd`42203380 : ntdll!RtlpEnterCriticalSectionContended+0xa4
00000071`3fe4b700 00007ffd`4380ad5a : 00007ffd`42203380 00000071`3fe4b7c0 00007ffd`41c57850 00007ffd`4381609c : ntdll!LdrpAcquireLoaderLock+0×2c
00000071`3fe4b740 00007ffd`4386785f : 00000000`00000000 00007ffd`422026f4 00000071`3ef01790 00000071`3fe4b860 : ntdll!LdrpGetDelayloadExportDll+0×7e
00000071`3fe4b800 00007ffd`43826c8e : 00000000`00000000 00007ffd`422026f4 00000000`00000000 00007ffd`41c57850 : ntdll!LdrpHandleProtectedDelayload+0×3f
00000071`3fe4bcc0 00007ffd`41a818eb : 00000071`3fe4c910 00000071`3fe4c8a0 00000000`00000000 ffffffff`ffffffff : ntdll!LdrResolveDelayLoadedAPI+0×8e
00000071`3fe4bd20 00007ffd`41896d9f : 00000071`40c52670 ffffffff`ffffffff 00000000`00000000 00000071`3fe4e201 : shell32!_delayLoadHelper2+0×2b
00000071`3fe4bd60 00007ffd`41894374 : 00000071`3fe4c8a0 00000071`3fe4bf10 00000000`00000027 00000071`3fe4bed0 : shell32!_tailMerge_api_ms_win_core_com_l1_1_1_dll+0×3f
00000071`3fe4bdd0 00007ffd`4189409a : 00000000`00000000 00000000`00000000 00000071`3fe4c910 00000000`00000000 : shell32!kfapi::CFolderDefinitionStorage::_LoadRegistry+0xb4
00000071`3fe4c440 00007ffd`41896943 : 00000000`00000031 00000000`00000000 00000071`3fe4c698 00000071`40c52670 : shell32!kfapi::CFolderDefinitionStorage::Load+0×66
00000071`3fe4c6a0 00007ffd`4189796a : 00000000`00000000 00000071`40c539a0 00000071`40c37700 00007ffd`41a80f90 : shell32!kfapi::CFolderPathResolver::GetPath+0×242
00000071`3fe4dcc0 00007ffd`41892d50 : 00000071`3fe4e050 00000071`3fe4e038 00000071`3fe4e038 00000000`00000200 : shell32!kfapi::CFolderCache::GetPath+0xb74
00000071`3fe4e030 00007ffd`41893540 : 00000071`3fe4e1c8 00000071`40cb16f0 00000071`40cb16f0 00000000`00000000 : shell32!kfapi::CKFFacade::GetFolderPath+0×120
00000071`3fe4e190 00007ffd`4189368c : 00000000`00008023 00007ffd`4382c0ae 00000000`00000000 00000071`3ef92bd0 : shell32!SHGetFolderPathEx+0×80
00000071`3fe4e200 00007ffd`40d61939 : 00000000`00000000 00000000`00000000 00007ffd`40dc3f80 00000000`00008023 : shell32!SHGetFolderPathWWorker+0xf9
00000071`3fe4e260 00007ffd`41a80fba : 00000000`00000000 00000000`00000000 00000071`3fe4e370 00000000`00000000 : KERNELBASE!SHGetFolderPathW+0×49
00000071`3fe4e2a0 00007ffd`40d7e60f : 00000000`00000000 00000071`3fe4e370 00000000`00000023 00000000`00000001 : shell32!SHGetSpecialFolderPathWWorker+0×23
00000071`3fe4e2e0 00000000`540b036d : 00000071`3fe4e630 00000000`00000001 00000000`00000000 00000071`3fe4e6b0 : KERNELBASE!SHGetSpecialFolderPathW+0×3f
00000071`3fe4e310 00000000`54096f45 : 00000071`3fe4e708 00000000`00000001 00000071`3fe4e6d8 00000000`53fd1903 : ModuleC+0xdca2d
ntdll!RtlpStackTraceDataBase is NULL. Probably the stack traces are not enabled.

0:000> ub 00007ffd`41894e14
shell32!kfapi::CFolderPathResolver::GetPath+0×145:
00007ffd`41894df1 498d5d30        lea     rbx,[r13+30h]
00007ffd`41894df5 48895c2468      mov     qword ptr [rsp+68h],rbx
00007ffd`41894dfa 4c897c2470      mov     qword ptr [rsp+70h],r15
00007ffd`41894dff 498bff          mov     rdi,r15
00007ffd`41894e02 4c897c2470      mov     qword ptr [rsp+70h],r15
00007ffd`41894e07 488975e0        mov     qword ptr [rbp-20h],rsi
00007ffd`41894e0b 488bce          mov     rcx,rsi
00007ffd`41894e0e ff159cbaa900    call    qword ptr [shell32!_imp_EnterCriticalSection (00007ffd`423308b0)]

0:000> ub 00007ffd`43827e40
ntdll!LdrpAcquireLoaderLock:
00007ffd`43827e14 4053            push    rbx
00007ffd`43827e16 4883ec30        sub     rsp,30h
00007ffd`43827e1a 488d0d4f8d0f00  lea     rcx,[ntdll!LdrpModuleEnumLock (00007ffd`43920b70)]
00007ffd`43827e21 e8ca040000      call    ntdll!RtlAcquireSRWLockShared (00007ffd`438282f0)
00007ffd`43827e26 803c258403fe7f00 cmp     byte ptr [SharedUserData+0×384 (00000000`7ffe0384)],0
00007ffd`43827e2e 0f85242c0700    jne     ntdll!LdrpAcquireLoaderLock+0×72c44 (00007ffd`4389aa58)
00007ffd`43827e34 488d0d6d5a0f00  lea     rcx,[ntdll!LdrpLoaderLock (00007ffd`4391d8a8)]
00007ffd`43827e3b e820000000      call    ntdll!RtlEnterCriticalSection (00007ffd`43827e60)

- Dmitry Vostokov @ DumpAnalysis.org + TraceAnalysis.org -