test: heappriv.h Source File

00001 /*++ 00002 00003 Copyright (c) 1992 Microsoft Corporation 00004 00005 Module Name: 00006 00007 heappriv.h 00008 00009 Abstract: 00010 00011 Private include file used by heap allocator (heap.c, heapdll.c and 00012 heapdbg.c) 00013 00014 Author: 00015 00016 Steve Wood (stevewo) 25-Oct-1994 00017 00018 Revision History: 00019 00020 --*/ 00021 00022 #ifndef _RTL_HEAP_PRIVATE_ 00023 #define _RTL_HEAP_PRIVATE_ 00024 00025 #include "heappage.h" 00026 00027 // 00028 // Disable FPO optimization so even retail builds get somewhat reasonable 00029 // stack backtraces 00030 // 00031 00032 #if i386 00033 // #pragma optimize("y",off) 00034 #endif 00035 00036 #if DBG 00037 #define HEAPASSERT(exp) if (!(exp)) RtlAssert( #exp, __FILE__, __LINE__, NULL ) 00038 #else 00039 #define HEAPASSERT(exp) 00040 #endif 00041 00042 // 00043 // This variable contains the fill pattern used for heap tail checking 00044 // 00045 00046 UCHAR CheckHeapFillPattern[ CHECK_HEAP_TAIL_SIZE ]; 00047 00048 00049 // 00050 // Here are the locking routines for the heap (kernel and user) 00051 // 00052 00053 #ifdef NTOS_KERNEL_RUNTIME 00054 00055 // 00056 // Kernel mode heap uses the kernel resource package for locking 00057 // 00058 00059 #define RtlInitializeLockRoutine(L) ExInitializeResource((PERESOURCE)(L)) 00060 #define RtlAcquireLockRoutine(L) ExAcquireResourceExclusive((PERESOURCE)(L),TRUE) 00061 #define RtlReleaseLockRoutine(L) ExReleaseResource((PERESOURCE)(L)) 00062 #define RtlDeleteLockRoutine(L) ExDeleteResource((PERESOURCE)(L)) 00063 #define RtlOkayToLockRoutine(L) ExOkayToLockRoutine((PERESOURCE)(L)) 00064 00065 #else // #ifdef NTOS_KERNEL_ROUTINE 00066 00067 // 00068 // User mode heap uses the critical section package for locking 00069 // 00070 00071 #ifndef PREALLOCATE_EVENT_MASK 00072 00073 #define PREALLOCATE_EVENT_MASK 0x80000000 // **** defined only in dll\resource.c 00074 00075 #endif // PREALLOCATE_EVENT_MASK 00076 00077 #define RtlInitializeLockRoutine(L) RtlInitializeCriticalSectionAndSpinCount((PRTL_CRITICAL_SECTION)(L),(PREALLOCATE_EVENT_MASK | 4000)) 00078 #define RtlAcquireLockRoutine(L) RtlEnterCriticalSection((PRTL_CRITICAL_SECTION)(L)) 00079 #define RtlReleaseLockRoutine(L) RtlLeaveCriticalSection((PRTL_CRITICAL_SECTION)(L)) 00080 #define RtlDeleteLockRoutine(L) RtlDeleteCriticalSection((PRTL_CRITICAL_SECTION)(L)) 00081 #define RtlOkayToLockRoutine(L) NtdllOkayToLockRoutine((PVOID)(L)) 00082 00083 #endif // #ifdef NTOS_KERNEL_RUNTIME 00084 00085 00086 // 00087 // Here are some debugging macros for the heap 00088 // 00089 00090 #ifdef NTOS_KERNEL_RUNTIME 00091 00092 #define HEAP_DEBUG_FLAGS 0 00093 #define DEBUG_HEAP(F) FALSE 00094 #define SET_LAST_STATUS(S) NOTHING; 00095 00096 #else // #ifdef NTOS_KERNEL_ROUTINE 00097 00098 #define HEAP_DEBUG_FLAGS (HEAP_VALIDATE_PARAMETERS_ENABLED | \ 00099 HEAP_VALIDATE_ALL_ENABLED | \ 00100 HEAP_CAPTURE_STACK_BACKTRACES | \ 00101 HEAP_CREATE_ENABLE_TRACING | \ 00102 HEAP_FLAG_PAGE_ALLOCS) 00103 #define DEBUG_HEAP(F) ((F & HEAP_DEBUG_FLAGS) && !(F & HEAP_SKIP_VALIDATION_CHECKS)) 00104 #define SET_LAST_STATUS(S) {NtCurrentTeb()->LastErrorValue = RtlNtStatusToDosError( NtCurrentTeb()->LastStatusValue = (ULONG)(S) );} 00105 00106 #endif // #ifdef NTOS_KERNEL_RUNTIME 00107 00108 00109 // 00110 // Here are the macros used for debug printing and breakpoints 00111 // 00112 00113 #ifdef NTOS_KERNEL_RUNTIME 00114 00115 #define HeapDebugPrint( _x_ ) {DbgPrint _x_;} 00116 00117 #define HeapDebugBreak( _x_ ) {if (KdDebuggerEnabled) DbgBreakPoint();} 00118 00119 #else // #ifdef NTOS_KERNEL_ROUTINE 00120 00121 #define HeapDebugPrint( _x_ ) \ 00122 { \ 00123 PLIST_ENTRY _Module; \ 00124 PLDR_DATA_TABLE_ENTRY _Entry; \ 00125 \ 00126 _Module = NtCurrentPeb()->Ldr->InLoadOrderModuleList.Flink; \ 00127 _Entry = CONTAINING_RECORD( _Module, \ 00128 LDR_DATA_TABLE_ENTRY, \ 00129 InLoadOrderLinks); \ 00130 DbgPrint("HEAP[%wZ]: ", &_Entry->BaseDllName); \ 00131 DbgPrint _x_; \ 00132 } 00133 00134 #define HeapDebugBreak( _x_ ) \ 00135 { \ 00136 VOID RtlpBreakPointHeap( PVOID BadAddress ); \ 00137 \ 00138 RtlpBreakPointHeap( (_x_) ); \ 00139 } 00140 00141 #endif // #ifdef NTOS_KERNEL_RUNTIME 00142 00143 00144 // 00145 // Implemented in heap.c 00146 // 00147 00148 BOOLEAN 00149 RtlpInitializeHeapSegment ( 00150 IN PHEAP Heap, 00151 IN PHEAP_SEGMENT Segment, 00152 IN UCHAR SegmentIndex, 00153 IN ULONG Flags, 00154 IN PVOID BaseAddress, 00155 IN PVOID UnCommittedAddress, 00156 IN PVOID CommitLimitAddress 00157 ); 00158 00159 PHEAP_FREE_ENTRY 00160 RtlpCoalesceFreeBlocks ( 00161 IN PHEAP Heap, 00162 IN PHEAP_FREE_ENTRY FreeBlock, 00163 IN OUT PSIZE_T FreeSize, 00164 IN BOOLEAN RemoveFromFreeList 00165 ); 00166 00167 VOID 00168 RtlpDeCommitFreeBlock ( 00169 IN PHEAP Heap, 00170 IN PHEAP_FREE_ENTRY FreeBlock, 00171 IN SIZE_T FreeSize 00172 ); 00173 00174 VOID 00175 RtlpInsertFreeBlock ( 00176 IN PHEAP Heap, 00177 IN PHEAP_FREE_ENTRY FreeBlock, 00178 IN SIZE_T FreeSize 00179 ); 00180 00181 PHEAP_FREE_ENTRY 00182 RtlpFindAndCommitPages ( 00183 IN PHEAP Heap, 00184 IN PHEAP_SEGMENT Segment, 00185 IN OUT PSIZE_T Size, 00186 IN PVOID AddressWanted OPTIONAL 00187 ); 00188 00189 PVOID 00190 RtlAllocateHeapSlowly ( 00191 IN PVOID HeapHandle, 00192 IN ULONG Flags, 00193 IN SIZE_T Size 00194 ); 00195 00196 BOOLEAN 00197 RtlFreeHeapSlowly ( 00198 IN PVOID HeapHandle, 00199 IN ULONG Flags, 00200 IN PVOID BaseAddress 00201 ); 00202 00203 SIZE_T 00204 RtlpGetSizeOfBigBlock ( 00205 IN PHEAP_ENTRY BusyBlock 00206 ); 00207 00208 PHEAP_ENTRY_EXTRA 00209 RtlpGetExtraStuffPointer ( 00210 PHEAP_ENTRY BusyBlock 00211 ); 00212 00213 BOOLEAN 00214 RtlpCheckBusyBlockTail ( 00215 IN PHEAP_ENTRY BusyBlock 00216 ); 00217 00218 00219 // 00220 // Implemented in heapdll.c 00221 // 00222 00223 VOID 00224 RtlpAddHeapToProcessList ( 00225 IN PHEAP Heap 00226 ); 00227 00228 VOID 00229 RtlpRemoveHeapFromProcessList ( 00230 IN PHEAP Heap 00231 ); 00232 00233 PHEAP_FREE_ENTRY 00234 RtlpCoalesceHeap ( 00235 IN PHEAP Heap 00236 ); 00237 00238 BOOLEAN 00239 RtlpCheckHeapSignature ( 00240 IN PHEAP Heap, 00241 IN PCHAR Caller 00242 ); 00243 00244 00245 // 00246 // Implemented in heapdbg.c 00247 // 00248 00249 BOOLEAN 00250 RtlpValidateHeapEntry ( 00251 IN PHEAP Heap, 00252 IN PHEAP_ENTRY BusyBlock, 00253 IN PCHAR Reason 00254 ); 00255 00256 BOOLEAN 00257 RtlpValidateHeap ( 00258 IN PHEAP Heap, 00259 IN BOOLEAN AlwaysValidate 00260 ); 00261 00262 VOID 00263 RtlpUpdateHeapListIndex ( 00264 USHORT OldIndex, 00265 USHORT NewIndex 00266 ); 00267 00268 BOOLEAN 00269 RtlpValidateHeapHeaders( 00270 IN PHEAP Heap, 00271 IN BOOLEAN Recompute 00272 ); 00273 00274 00275 // 00276 // Macro for setting a bit in the freelist vector to indicate entries are 00277 // present. 00278 // 00279 00280 #define SET_FREELIST_BIT( H, FB ) \ 00281 { \ 00282 ULONG _Index_; \ 00283 ULONG _Bit_; \ 00284 \ 00285 HEAPASSERT((FB)->Size < HEAP_MAXIMUM_FREELISTS); \ 00286 \ 00287 _Index_ = (FB)->Size >> 3; \ 00288 _Bit_ = (1 << ((FB)->Size & 7)); \ 00289 \ 00290 HEAPASSERT(((H)->u.FreeListsInUseBytes[ _Index_ ] & _Bit_) == 0); \ 00291 \ 00292 (H)->u.FreeListsInUseBytes[ _Index_ ] |= _Bit_; \ 00293 } 00294 00295 // 00296 // Macro for clearing a bit in the freelist vector to indicate entries are 00297 // not present. 00298 // 00299 00300 #define CLEAR_FREELIST_BIT( H, FB ) \ 00301 { \ 00302 ULONG _Index_; \ 00303 ULONG _Bit_; \ 00304 \ 00305 HEAPASSERT((FB)->Size < HEAP_MAXIMUM_FREELISTS); \ 00306 \ 00307 _Index_ = (FB)->Size >> 3; \ 00308 _Bit_ = (1 << ((FB)->Size & 7)); \ 00309 \ 00310 HEAPASSERT((H)->u.FreeListsInUseBytes[ _Index_ ] & _Bit_); \ 00311 HEAPASSERT(IsListEmpty(&(H)->FreeLists[ (FB)->Size ])); \ 00312 \ 00313 (H)->u.FreeListsInUseBytes[ _Index_ ] ^= _Bit_; \ 00314 } 00315 00316 00317 // 00318 // This macro inserts a free block into the appropriate free list including 00319 // the [0] index list with entry filling if necessary 00320 // 00321 00322 #define RtlpInsertFreeBlockDirect( H, FB, SIZE ) \ 00323 { \ 00324 PLIST_ENTRY _HEAD, _NEXT; \ 00325 PHEAP_FREE_ENTRY _FB1; \ 00326 \ 00327 HEAPASSERT((FB)->Size == (SIZE)); \ 00328 (FB)->Flags &= ~(HEAP_ENTRY_FILL_PATTERN | \ 00329 HEAP_ENTRY_EXTRA_PRESENT | \ 00330 HEAP_ENTRY_BUSY); \ 00331 \ 00332 if ((H)->Flags & HEAP_FREE_CHECKING_ENABLED) { \ 00333 \ 00334 RtlFillMemoryUlong( (PCHAR)((FB) + 1), \ 00335 ((SIZE) << HEAP_GRANULARITY_SHIFT) - \ 00336 sizeof( *(FB) ), \ 00337 FREE_HEAP_FILL ); \ 00338 \ 00339 (FB)->Flags |= HEAP_ENTRY_FILL_PATTERN; \ 00340 } \ 00341 \ 00342 if ((SIZE) < HEAP_MAXIMUM_FREELISTS) { \ 00343 \ 00344 _HEAD = &(H)->FreeLists[ (SIZE) ]; \ 00345 \ 00346 if (IsListEmpty(_HEAD)) { \ 00347 \ 00348 SET_FREELIST_BIT( H, FB ); \ 00349 } \ 00350 \ 00351 } else { \ 00352 \ 00353 _HEAD = &(H)->FreeLists[ 0 ]; \ 00354 _NEXT = _HEAD->Flink; \ 00355 \ 00356 while (_HEAD != _NEXT) { \ 00357 \ 00358 _FB1 = CONTAINING_RECORD( _NEXT, HEAP_FREE_ENTRY, FreeList ); \ 00359 \ 00360 if ((SIZE) <= _FB1->Size) { \ 00361 \ 00362 break; \ 00363 \ 00364 } else { \ 00365 \ 00366 _NEXT = _NEXT->Flink; \ 00367 } \ 00368 } \ 00369 \ 00370 _HEAD = _NEXT; \ 00371 } \ 00372 \ 00373 InsertTailList( _HEAD, &(FB)->FreeList ); \ 00374 } 00375 00376 // 00377 // This version of RtlpInsertFreeBlockDirect does no filling. 00378 // 00379 00380 #define RtlpFastInsertFreeBlockDirect( H, FB, SIZE ) \ 00381 { \ 00382 if ((SIZE) < HEAP_MAXIMUM_FREELISTS) { \ 00383 \ 00384 RtlpFastInsertDedicatedFreeBlockDirect( H, FB, SIZE ); \ 00385 \ 00386 } else { \ 00387 \ 00388 RtlpFastInsertNonDedicatedFreeBlockDirect( H, FB, SIZE ); \ 00389 } \ 00390 } 00391 00392 // 00393 // This version of RtlpInsertFreeBlockDirect only works for dedicated free 00394 // lists and doesn't do any filling. 00395 // 00396 00397 #define RtlpFastInsertDedicatedFreeBlockDirect( H, FB, SIZE ) \ 00398 { \ 00399 PLIST_ENTRY _HEAD; \ 00400 \ 00401 HEAPASSERT((FB)->Size == (SIZE)); \ 00402 \ 00403 if (!((FB)->Flags & HEAP_ENTRY_LAST_ENTRY)) { \ 00404 \ 00405 HEAPASSERT(((PHEAP_ENTRY)(FB) + (SIZE))->PreviousSize == (SIZE)); \ 00406 } \ 00407 \ 00408 (FB)->Flags &= HEAP_ENTRY_LAST_ENTRY; \ 00409 \ 00410 _HEAD = &(H)->FreeLists[ (SIZE) ]; \ 00411 \ 00412 if (IsListEmpty(_HEAD)) { \ 00413 \ 00414 SET_FREELIST_BIT( H, FB ); \ 00415 } \ 00416 \ 00417 InsertTailList( _HEAD, &(FB)->FreeList ); \ 00418 } 00419 00420 // 00421 // This version of RtlpInsertFreeBlockDirect only works for nondedicated free 00422 // lists and doesn't do any filling. 00423 // 00424 00425 #define RtlpFastInsertNonDedicatedFreeBlockDirect( H, FB, SIZE ) \ 00426 { \ 00427 PLIST_ENTRY _HEAD, _NEXT; \ 00428 PHEAP_FREE_ENTRY _FB1; \ 00429 \ 00430 HEAPASSERT((FB)->Size == (SIZE)); \ 00431 \ 00432 if (!((FB)->Flags & HEAP_ENTRY_LAST_ENTRY)) { \ 00433 \ 00434 HEAPASSERT(((PHEAP_ENTRY)(FB) + (SIZE))->PreviousSize == (SIZE)); \ 00435 } \ 00436 \ 00437 (FB)->Flags &= (HEAP_ENTRY_LAST_ENTRY); \ 00438 \ 00439 _HEAD = &(H)->FreeLists[ 0 ]; \ 00440 _NEXT = _HEAD->Flink; \ 00441 \ 00442 while (_HEAD != _NEXT) { \ 00443 \ 00444 _FB1 = CONTAINING_RECORD( _NEXT, HEAP_FREE_ENTRY, FreeList ); \ 00445 \ 00446 if ((SIZE) <= _FB1->Size) { \ 00447 \ 00448 break; \ 00449 \ 00450 } else { \ 00451 \ 00452 _NEXT = _NEXT->Flink; \ 00453 } \ 00454 } \ 00455 \ 00456 InsertTailList( _NEXT, &(FB)->FreeList ); \ 00457 } 00458 00459 00460 // 00461 // This macro removes a block from its free list with fill checking if 00462 // necessary 00463 // 00464 00465 #define RtlpRemoveFreeBlock( H, FB ) \ 00466 { \ 00467 RtlpFastRemoveFreeBlock( H, FB ) \ 00468 \ 00469 if ((FB)->Flags & HEAP_ENTRY_FILL_PATTERN) { \ 00470 \ 00471 SIZE_T cb, cbEqual; \ 00472 PVOID p; \ 00473 \ 00474 cb = ((FB)->Size << HEAP_GRANULARITY_SHIFT) - sizeof( *(FB) ); \ 00475 \ 00476 if ((FB)->Flags & HEAP_ENTRY_EXTRA_PRESENT && \ 00477 cb > sizeof( HEAP_FREE_ENTRY_EXTRA )) { \ 00478 \ 00479 cb -= sizeof( HEAP_FREE_ENTRY_EXTRA ); \ 00480 } \ 00481 \ 00482 cbEqual = RtlCompareMemoryUlong( (PCHAR)((FB) + 1), \ 00483 cb, \ 00484 FREE_HEAP_FILL ); \ 00485 \ 00486 if (cbEqual != cb) { \ 00487 \ 00488 HeapDebugPrint(( \ 00489 "HEAP: Free Heap block %lx modified at %lx after it was freed\n", \ 00490 (FB), \ 00491 (PCHAR)((FB) + 1) + cbEqual )); \ 00492 \ 00493 HeapDebugBreak((FB)); \ 00494 } \ 00495 } \ 00496 } 00497 00498 // 00499 // This version of RtlpRemoveFreeBlock does no fill checking 00500 // 00501 00502 #define RtlpFastRemoveFreeBlock( H, FB ) \ 00503 { \ 00504 PLIST_ENTRY _EX_Blink; \ 00505 PLIST_ENTRY _EX_Flink; \ 00506 \ 00507 _EX_Flink = (FB)->FreeList.Flink; \ 00508 _EX_Blink = (FB)->FreeList.Blink; \ 00509 \ 00510 _EX_Blink->Flink = _EX_Flink; \ 00511 _EX_Flink->Blink = _EX_Blink; \ 00512 \ 00513 if ((_EX_Flink == _EX_Blink) && \ 00514 ((FB)->Size < HEAP_MAXIMUM_FREELISTS)) { \ 00515 \ 00516 CLEAR_FREELIST_BIT( H, FB ); \ 00517 } \ 00518 } 00519 00520 // 00521 // This version of RtlpRemoveFreeBlock only works for dedicated free lists 00522 // (where we know that (FB)->Mask != 0) and doesn't do any fill checking 00523 // 00524 00525 #define RtlpFastRemoveDedicatedFreeBlock( H, FB ) \ 00526 { \ 00527 PLIST_ENTRY _EX_Blink; \ 00528 PLIST_ENTRY _EX_Flink; \ 00529 \ 00530 _EX_Flink = (FB)->FreeList.Flink; \ 00531 _EX_Blink = (FB)->FreeList.Blink; \ 00532 \ 00533 _EX_Blink->Flink = _EX_Flink; \ 00534 _EX_Flink->Blink = _EX_Blink; \ 00535 \ 00536 if (_EX_Flink == _EX_Blink) { \ 00537 \ 00538 CLEAR_FREELIST_BIT( H, FB ); \ 00539 } \ 00540 } 00541 00542 // 00543 // This version of RtlpRemoveFreeBlock only works for dedicated free lists 00544 // (where we know that (FB)->Mask == 0) and doesn't do any fill checking 00545 // 00546 00547 #define RtlpFastRemoveNonDedicatedFreeBlock( H, FB ) \ 00548 { \ 00549 RemoveEntryList(&(FB)->FreeList) \ 00550 } 00551 00552 00553 // 00554 // Heap tagging routines implemented in heapdll.c 00555 // 00556 00557 #if DBG 00558 00559 #define IS_HEAP_TAGGING_ENABLED() (TRUE) 00560 00561 #else 00562 00563 #define IS_HEAP_TAGGING_ENABLED() (RtlGetNtGlobalFlags() & FLG_HEAP_ENABLE_TAGGING) 00564 00565 #endif // DBG 00566 00567 // 00568 // ORDER IS IMPORTANT HERE...SEE RtlpUpdateTagEntry sources 00569 // 00570 00571 typedef enum _HEAP_TAG_ACTION { 00572 00573 AllocationAction, 00574 VirtualAllocationAction, 00575 FreeAction, 00576 VirtualFreeAction, 00577 ReAllocationAction, 00578 VirtualReAllocationAction 00579 00580 } HEAP_TAG_ACTION; 00581 00582 PWSTR 00583 RtlpGetTagName ( 00584 PHEAP Heap, 00585 USHORT TagIndex 00586 ); 00587 00588 USHORT 00589 RtlpUpdateTagEntry ( 00590 PHEAP Heap, 00591 USHORT TagIndex, 00592 SIZE_T OldSize, // Only valid for ReAllocation and Free actions 00593 SIZE_T NewSize, // Only valid for ReAllocation and Allocation actions 00594 HEAP_TAG_ACTION Action 00595 ); 00596 00597 VOID 00598 RtlpResetTags ( 00599 PHEAP Heap 00600 ); 00601 00602 VOID 00603 RtlpDestroyTags ( 00604 PHEAP Heap 00605 ); 00606 00607 00608 // 00609 // Define heap lookaside list allocation functions. 00610 // 00611 00612 typedef struct _HEAP_LOOKASIDE { 00613 SLIST_HEADER ListHead; 00614 00615 USHORT Depth; 00616 USHORT MaximumDepth; 00617 00618 ULONG TotalAllocates; 00619 ULONG AllocateMisses; 00620 ULONG TotalFrees; 00621 ULONG FreeMisses; 00622 00623 ULONG LastTotalAllocates; 00624 ULONG LastAllocateMisses; 00625 00626 ULONG Future[2]; 00627 00628 } HEAP_LOOKASIDE, *PHEAP_LOOKASIDE; 00629 00630 NTKERNELAPI 00631 VOID 00632 RtlpInitializeHeapLookaside ( 00633 IN PHEAP_LOOKASIDE Lookaside, 00634 IN USHORT Depth 00635 ); 00636 00637 NTKERNELAPI 00638 VOID 00639 RtlpDeleteHeapLookaside ( 00640 IN PHEAP_LOOKASIDE Lookaside 00641 ); 00642 00643 VOID 00644 RtlpAdjustHeapLookasideDepth ( 00645 IN PHEAP_LOOKASIDE Lookaside 00646 ); 00647 00648 NTKERNELAPI 00649 PVOID 00650 RtlpAllocateFromHeapLookaside ( 00651 IN PHEAP_LOOKASIDE Lookaside 00652 ); 00653 00654 NTKERNELAPI 00655 BOOLEAN 00656 RtlpFreeToHeapLookaside ( 00657 IN PHEAP_LOOKASIDE Lookaside, 00658 IN PVOID Entry 00659 ); 00660 00661 #endif // _RTL_HEAP_PRIVATE_