physical.c File Reference

#include "mi.h"

Go to the source code of this file.

Defines
#define	COPY_STACK_SIZE   256
#define	BITS_IN_ULONG   ((sizeof (ULONG)) * 8)
#define	LOWEST_USABLE_PHYSICAL_ADDRESS   (16 * 1024 * 1024)
#define	LOWEST_USABLE_PHYSICAL_PAGE   (LOWEST_USABLE_PHYSICAL_ADDRESS >> PAGE_SHIFT)
#define	LOWEST_BITMAP_PHYSICAL_PAGE   0
#define	MI_FRAME_TO_BITMAP_INDEX(x)   ((ULONG)(x))
#define	MI_BITMAP_INDEX_TO_FRAME(x)   ((ULONG)(x))
Functions
VOID	MiFlushUserPhysicalPteList (IN PMMPTE_FLUSH_LIST PteFlushList)
NTSTATUS	NtMapUserPhysicalPages (IN PVOID VirtualAddress, IN ULONG_PTR NumberOfPages, IN PULONG_PTR UserPfnArray OPTIONAL)
NTSTATUS	NtMapUserPhysicalPagesScatter (IN PVOID *VirtualAddresses, IN ULONG_PTR NumberOfPages, IN PULONG_PTR UserPfnArray OPTIONAL)
NTSTATUS	NtAllocateUserPhysicalPages (IN HANDLE ProcessHandle, IN OUT PULONG_PTR NumberOfPages, OUT PULONG_PTR UserPfnArray)
NTSTATUS	NtFreeUserPhysicalPages (IN HANDLE ProcessHandle, IN OUT PULONG_PTR NumberOfPages, IN PULONG_PTR UserPfnArray)
VOID	MiRemoveUserPhysicalPagesVad (IN PMMVAD_SHORT Vad)
VOID	MiUpdateVadPhysicalPages (IN ULONG_PTR TotalFreedPages)
VOID	MiCleanPhysicalProcessPages (IN PEPROCESS Process)
Variables
ULONG_PTR	MmVadPhysicalPages

---

Define Documentation

#define BITS_IN_ULONG   ((sizeof (ULONG)) * 8)

Definition at line 67 of file physical.c.

#define COPY_STACK_SIZE   256

Definition at line 65 of file physical.c. Referenced by MiCleanPhysicalProcessPages(), MiDoPoolCopy(), NtFreeUserPhysicalPages(), NtMapUserPhysicalPages(), and NtMapUserPhysicalPagesScatter().

#define LOWEST_BITMAP_PHYSICAL_PAGE   0

Definition at line 72 of file physical.c.

#define LOWEST_USABLE_PHYSICAL_ADDRESS   (16 * 1024 * 1024)

Definition at line 69 of file physical.c. Referenced by NtAllocateUserPhysicalPages().

#define LOWEST_USABLE_PHYSICAL_PAGE   (LOWEST_USABLE_PHYSICAL_ADDRESS >> PAGE_SHIFT)

Definition at line 70 of file physical.c. Referenced by MiCleanPhysicalProcessPages(), MiRemoveUserPhysicalPagesVad(), NtAllocateUserPhysicalPages(), NtFreeUserPhysicalPages(), NtMapUserPhysicalPages(), and NtMapUserPhysicalPagesScatter().

#define MI_BITMAP_INDEX_TO_FRAME (  x   )     ((ULONG)(x))

Definition at line 74 of file physical.c. Referenced by MiCleanPhysicalProcessPages().

#define MI_FRAME_TO_BITMAP_INDEX (  x   )     ((ULONG)(x))

Definition at line 73 of file physical.c. Referenced by NtAllocateUserPhysicalPages(), NtFreeUserPhysicalPages(), NtMapUserPhysicalPages(), and NtMapUserPhysicalPagesScatter().

---

Function Documentation

VOID MiCleanPhysicalProcessPages (  IN PEPROCESS  Process  )

Definition at line 2313 of file physical.c. References APC_LEVEL, ASSERT, BitMap, COPY_STACK_SIZE, ExFreePool(), LOWEST_USABLE_PHYSICAL_PAGE, MI_BITMAP_INDEX_TO_FRAME, MI_PFN_ELEMENT, MI_PFN_IS_AWE, MI_SET_PFN_DELETED, MiUpdateVadPhysicalPages(), MmFreePagesFromMdl(), MmHighestPossiblePhysicalPage, MmInitializeMdl, NonPagedPool, NULL, PAGE_SHIFT, PsReturnPoolQuota(), _MMPFN::PteAddress, RtlFindSetBits(), TRUE, and _MMPFN::u2. Referenced by MmCleanProcessAddressSpace(). 02319 : 02320 02321 This routine frees the VadPhysicalBitMap, any remaining physical pages (as 02322 they may not have been currently mapped into any Vads) and returns the 02323 bitmap quota. 02324 02325 Arguments: 02326 02327 Process - Supplies the process to clean. 02328 02329 Return Value: 02330 02331 None. 02332 02333 Environment: 02334 02335 Kernel mode, APC level, working set mutex held. Called only on process 02336 exit, so the AWE lock is not needed here. 02337 02338 --*/ 02339 02340 { 02341 PMMPFN Pfn1; 02342 ULONG BitMapSize; 02343 ULONG BitMapIndex; 02344 ULONG BitMapHint; 02345 PRTL_BITMAP BitMap; 02346 PPFN_NUMBER MdlPage; 02347 PFN_NUMBER MdlHack[(sizeof(MDL) / sizeof(PFN_NUMBER)) + COPY_STACK_SIZE]; 02348 ULONG_PTR MdlPages; 02349 ULONG_PTR NumberOfPages; 02350 ULONG_PTR TotalFreedPages; 02351 PMDL MemoryDescriptorList; 02352 PFN_NUMBER PageFrameIndex; 02353 #if DBG 02354 ULONG_PTR ActualPages = 0; 02355 ULONG_PTR ExpectedPages = 0; 02356 #endif 02357 02358 ASSERT (KeGetCurrentIrql() == APC_LEVEL); 02359 02360 #if DBG 02361 ExpectedPages = Process->VadPhysicalPages; 02362 #else 02363 if (Process->VadPhysicalPages == 0) { 02364 return; 02365 } 02366 #endif 02367 02368 TotalFreedPages = 0; 02369 BitMap = Process->VadPhysicalPagesBitMap; 02370 02371 if (BitMap != NULL) { 02372 02373 MdlPages = COPY_STACK_SIZE; 02374 MemoryDescriptorList = (PMDL)&MdlHack[0]; 02375 02376 MdlPage = (PPFN_NUMBER)(MemoryDescriptorList + 1); 02377 NumberOfPages = 0; 02378 02379 BitMapHint = 0; 02380 02381 while (TRUE) { 02382 02383 BitMapIndex = RtlFindSetBits (BitMap, 1, BitMapHint); 02384 02385 if (BitMapIndex < BitMapHint) { 02386 break; 02387 } 02388 02389 if (BitMapIndex == 0xFFFFFFFF) { 02390 break; 02391 } 02392 02393 PageFrameIndex = MI_BITMAP_INDEX_TO_FRAME(BitMapIndex); 02394 02395 #if defined (_WIN64) 02396 02397 // 02398 // This may become a problem for 64-bit systems with > 32tb 02399 // of physical memory as the 3rd parameter to RtlFindSetBits is 02400 // a ULONG. 02401 // 02402 02403 ASSERT (PageFrameIndex < 0x100000000); 02404 #endif 02405 02406 // 02407 // The bitmap search wraps, so handle it here. 02408 // Note PFN 0 is illegal. 02409 // 02410 02411 ASSERT (PageFrameIndex != 0); 02412 ASSERT (PageFrameIndex >= LOWEST_USABLE_PHYSICAL_PAGE); 02413 02414 ASSERT (ExpectedPages != 0); 02415 Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); 02416 ASSERT (Pfn1->u2.ShareCount == 1); 02417 ASSERT (Pfn1->PteAddress == (PMMPTE)0); 02418 02419 ASSERT (MI_PFN_IS_AWE (Pfn1)); 02420 02421 MI_SET_PFN_DELETED(Pfn1); 02422 02423 *MdlPage = PageFrameIndex; 02424 MdlPage += 1; 02425 NumberOfPages += 1; 02426 #if DBG 02427 ActualPages += 1; 02428 #endif 02429 02430 if (NumberOfPages == COPY_STACK_SIZE) { 02431 02432 // 02433 // Free the pages in the full MDL. 02434 // 02435 02436 MmInitializeMdl (MemoryDescriptorList, 02437 0, 02438 NumberOfPages << PAGE_SHIFT); 02439 02440 MmFreePagesFromMdl (MemoryDescriptorList); 02441 02442 MdlPage = (PPFN_NUMBER)(MemoryDescriptorList + 1); 02443 Process->VadPhysicalPages -= NumberOfPages; 02444 TotalFreedPages += NumberOfPages; 02445 NumberOfPages = 0; 02446 } 02447 02448 BitMapHint = BitMapIndex + 1; 02449 if (BitMapHint >= BitMap->SizeOfBitMap) { 02450 break; 02451 } 02452 } 02453 02454 // 02455 // Free any straggling MDL pages here. 02456 // 02457 02458 if (NumberOfPages != 0) { 02459 MmInitializeMdl (MemoryDescriptorList, 02460 0, 02461 NumberOfPages << PAGE_SHIFT); 02462 02463 MmFreePagesFromMdl (MemoryDescriptorList); 02464 Process->VadPhysicalPages -= NumberOfPages; 02465 TotalFreedPages += NumberOfPages; 02466 } 02467 02468 ASSERT (ExpectedPages == ActualPages); 02469 02470 BitMapSize = sizeof(RTL_BITMAP) + (ULONG)((((MmHighestPossiblePhysicalPage + 1) + 31) / 32) * 4); 02471 02472 Process->VadPhysicalPagesBitMap = NULL; 02473 ExFreePool (BitMap); 02474 PsReturnPoolQuota (Process, NonPagedPool, BitMapSize); 02475 } 02476 02477 ASSERT (ExpectedPages == ActualPages); 02478 ASSERT (Process->VadPhysicalPages == 0); 02479 02480 if (TotalFreedPages != 0) { 02481 MiUpdateVadPhysicalPages (TotalFreedPages); 02482 } 02483 02484 return; 02485 }

VOID MiFlushUserPhysicalPteList (  IN PMMPTE_FLUSH_LIST  PteFlushList  )

Definition at line 2488 of file physical.c. References FALSE, KeFlushEntireTb(), KeFlushMultipleTb(), KeFlushSingleTb(), MM_MAXIMUM_FLUSH_COUNT, NULL, TRUE, _MMPTE::u, and ZeroPte. Referenced by MiRemoveUserPhysicalPagesVad(), NtAllocateUserPhysicalPages(), NtFreeUserPhysicalPages(), NtMapUserPhysicalPages(), and NtMapUserPhysicalPagesScatter(). 02494 : 02495 02496 This routine flushes all the PTEs in the PTE flush list. 02497 If the list has overflowed, the entire TB is flushed. 02498 02499 N.B. The intent was for this routine to NEVER write the PTEs and have 02500 the caller do this instead. There is no such export from Ke, so 02501 the flush of a single TB just reuses the PTE. 02502 02503 Arguments: 02504 02505 PteFlushList - Supplies an optional pointer to the list to be flushed. 02506 02507 Return Value: 02508 02509 None. 02510 02511 Environment: 02512 02513 Kernel mode, PFN lock NOT held. 02514 02515 --*/ 02516 02517 { 02518 ULONG count; 02519 02520 count = PteFlushList->Count; 02521 02522 if (count == 0) { 02523 return; 02524 } 02525 02526 if (count != 1) { 02527 if (count < MM_MAXIMUM_FLUSH_COUNT) { 02528 KeFlushMultipleTb (count, 02529 &PteFlushList->FlushVa[0], 02530 TRUE, 02531 FALSE, 02532 NULL, 02533 ZeroPte.u.Flush); 02534 } 02535 else { 02536 02537 // 02538 // Array has overflowed, flush the entire TB. 02539 // 02540 02541 KeFlushEntireTb (TRUE, FALSE); 02542 } 02543 } 02544 else { 02545 02546 // 02547 // This always writes the (same) value into the PTE. 02548 // 02549 02550 KeFlushSingleTb (PteFlushList->FlushVa[0], 02551 TRUE, 02552 FALSE, 02553 (PHARDWARE_PTE)PteFlushList->FlushPte[0], 02554 *(PHARDWARE_PTE)PteFlushList->FlushPte[0]); 02555 } 02556 02557 PteFlushList->Count = 0; 02558 return; 02559 } }

VOID MiRemoveUserPhysicalPagesVad (  IN PMMVAD_SHORT  Vad  )

Definition at line 2110 of file physical.c. References APC_LEVEL, ASSERT, _MMPTE_FLUSH_LIST::Count, DbgPrint, _MMPTE_FLUSH_LIST::FlushPte, _MMPTE_FLUSH_LIST::FlushVa, LOCK_AWE, LOCK_PFN2, LOWEST_USABLE_PHYSICAL_PAGE, MI_GET_PAGE_FRAME_FROM_PTE, MI_PFN_ELEMENT, MI_PFN_IS_AWE, MI_VPN_TO_VA, MI_VPN_TO_VA_ENDING, MI_WRITE_INVALID_PTE, MiFlushUserPhysicalPteList(), MiGetPteAddress, MiGetVirtualAddressMappedByPte, MM_MAXIMUM_FLUSH_COUNT, NULL, _EPROCESS::PhysicalVadList, PsGetCurrentProcess, _MMPTE::u, UNLOCK_AWE, UNLOCK_PFN2, _MI_PHYSICAL_VIEW::Vad, _EPROCESS::VadPhysicalPages, _EPROCESS::VadPhysicalPagesBitMap, and ZeroPte. Referenced by MmCleanProcessAddressSpace(), and NtFreeVirtualMemory(). 02116 : 02117 02118 This function removes the user-physical-pages mapped region from the 02119 current process's address space. This mapped region is private memory. 02120 02121 The physical pages of this Vad are unmapped here, but not freed. 02122 02123 Pagetable pages are freed and their use/commitment counts/quotas are 02124 managed by our caller. 02125 02126 Arguments: 02127 02128 Vad - Supplies the VAD which manages the address space. 02129 02130 Return Value: 02131 02132 None. 02133 02134 Environment: 02135 02136 APC level, working set mutex and address creation mutex held. 02137 02138 --*/ 02139 02140 { 02141 PMMPFN Pfn1; 02142 PEPROCESS Process; 02143 PFN_NUMBER PageFrameIndex; 02144 MMPTE_FLUSH_LIST PteFlushList; 02145 PMMPTE PointerPte; 02146 MMPTE PteContents; 02147 PMMPTE EndingPte; 02148 #if DBG 02149 KIRQL OldIrql; 02150 KIRQL OldIrql2; 02151 ULONG_PTR ActualPages; 02152 ULONG_PTR ExpectedPages; 02153 PLIST_ENTRY NextEntry; 02154 PMI_PHYSICAL_VIEW PhysicalView; 02155 #endif 02156 02157 ASSERT (KeGetCurrentIrql() == APC_LEVEL); 02158 02159 ASSERT (Vad->u.VadFlags.UserPhysicalPages == 1); 02160 02161 Process = PsGetCurrentProcess(); 02162 02163 // 02164 // If the physical pages count is zero, nothing needs to be done. 02165 // On checked systems, verify the list anyway. 02166 // 02167 02168 #if DBG 02169 ActualPages = 0; 02170 ExpectedPages = Process->VadPhysicalPages; 02171 #else 02172 if (Process->VadPhysicalPages == 0) { 02173 return; 02174 } 02175 #endif 02176 02177 // 02178 // The caller must have removed this Vad from the physical view list, 02179 // otherwise another thread could immediately remap pages back into the Vad. 02180 // 02181 // This allows us to proceed without acquiring the AWE or PFN locks - 02182 // everything can be done under the WS lock which is already held. 02183 // 02184 02185 #if DBG 02186 LOCK_AWE (Process, OldIrql); 02187 02188 LOCK_PFN2 (OldIrql2); 02189 02190 NextEntry = Process->PhysicalVadList.Flink; 02191 while (NextEntry != &Process->PhysicalVadList) { 02192 02193 PhysicalView = CONTAINING_RECORD(NextEntry, 02194 MI_PHYSICAL_VIEW, 02195 ListEntry); 02196 02197 if (PhysicalView->Vad == (PMMVAD)Vad) { 02198 DbgPrint ("MiRemoveUserPhysicalPagesVad : Vad %p still in list!\n", 02199 Vad); 02200 DbgBreakPoint (); 02201 } 02202 02203 NextEntry = NextEntry->Flink; 02204 } 02205 02206 UNLOCK_PFN2 (OldIrql2); 02207 UNLOCK_AWE (Process, OldIrql); 02208 #endif 02209 02210 // 02211 // If the physical pages bitmap doesn't exist, nothing needs to be done. 02212 // 02213 02214 if (Process->VadPhysicalPagesBitMap == NULL) { 02215 ASSERT (ExpectedPages == 0); 02216 return; 02217 } 02218 02219 PointerPte = MiGetPteAddress (MI_VPN_TO_VA (Vad->StartingVpn)); 02220 EndingPte = MiGetPteAddress (MI_VPN_TO_VA_ENDING (Vad->EndingVpn)); 02221 02222 PteFlushList.Count = 0; 02223 02224 while (PointerPte <= EndingPte) { 02225 PteContents = *PointerPte; 02226 if (PteContents.u.Hard.Valid == 0) { 02227 PointerPte += 1; 02228 continue; 02229 } 02230 02231 PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte); 02232 02233 ASSERT (PageFrameIndex >= LOWEST_USABLE_PHYSICAL_PAGE); 02234 ASSERT (ExpectedPages != 0); 02235 02236 Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); 02237 02238 ASSERT (MI_PFN_IS_AWE (Pfn1)); 02239 ASSERT (Pfn1->u2.ShareCount == 2); 02240 ASSERT (Pfn1->PteAddress == PointerPte); 02241 02242 // 02243 // The frame is currently mapped in this Vad so the PTE must 02244 // be cleared and the TB entry flushed. 02245 // 02246 02247 Pfn1->u2.ShareCount -= 1; 02248 Pfn1->PteAddress = (PMMPTE)0; 02249 02250 if (PteFlushList.Count != MM_MAXIMUM_FLUSH_COUNT) { 02251 PteFlushList.FlushVa[PteFlushList.Count] = 02252 MiGetVirtualAddressMappedByPte (PointerPte); 02253 PteFlushList.FlushPte[PteFlushList.Count] = PointerPte; 02254 PteFlushList.Count += 1; 02255 } 02256 02257 MI_WRITE_INVALID_PTE (PointerPte, ZeroPte); 02258 02259 PointerPte += 1; 02260 #if DBG 02261 ActualPages += 1; 02262 #endif 02263 ASSERT (ActualPages <= ExpectedPages); 02264 } 02265 02266 // 02267 // Flush the TB entries for these pages. Note ZeroPte is only used 02268 // when the FlushPte[0] field is nonzero or if only a single PTE is 02269 // being flushed. 02270 // 02271 02272 MiFlushUserPhysicalPteList (&PteFlushList); 02273 02274 return; 02275 }

VOID MiUpdateVadPhysicalPages (  IN ULONG_PTR  TotalFreedPages  )

Definition at line 2278 of file physical.c. References LOCK_PFN, MmVadPhysicalPages, and UNLOCK_PFN. Referenced by MiCleanPhysicalProcessPages(). 02284 : 02285 02286 Nonpaged helper routine to update the VadPhysicalPages count. 02287 02288 Arguments: 02289 02290 TotalFreedPages - Supplies the number of pages just freed. 02291 02292 Return Value: 02293 02294 None. 02295 02296 Environment: 02297 02298 Kernel mode, APC level or below. 02299 02300 --*/ 02301 02302 { 02303 KIRQL OldIrql; 02304 02305 LOCK_PFN (OldIrql); 02306 MmVadPhysicalPages -= TotalFreedPages; 02307 UNLOCK_PFN (OldIrql); 02308 02309 return; 02310 }

NTSTATUS NtAllocateUserPhysicalPages (  IN HANDLE  ProcessHandle, IN OUT PULONG_PTR  NumberOfPages, OUT PULONG_PTR  UserPfnArray )

Definition at line 970 of file physical.c. References _EPROCESS::AddressSpaceDeleted, ASSERT, BitMap, _MDL::ByteCount, _MMPTE_FLUSH_LIST::Count, ExAllocatePoolWithTag, EXCEPTION_EXECUTE_HANDLER, ExFreePool(), ExSystemExceptionFilter(), FALSE, _MMPTE_FLUSH_LIST::FlushPte, _MMPTE_FLUSH_LIST::FlushVa, KeAttachProcess(), KeDetachProcess(), KernelMode, KPROCESSOR_MODE, L, LOCK_AWE, LOCK_PFN, LOCK_WS, LOWEST_USABLE_PHYSICAL_ADDRESS, LOWEST_USABLE_PHYSICAL_PAGE, MI_FRAME_TO_BITMAP_INDEX, MI_PFN_ELEMENT, MI_PFN_IS_AWE, MI_SET_PFN_DELETED, MI_WRITE_INVALID_PTE, MiFlushUserPhysicalPteList(), MiGetVirtualAddressMappedByPte, MM_MAXIMUM_FLUSH_COUNT, MmAllocatePagesForMdl(), MmFreePagesFromMdl(), MmHighestPossiblePhysicalPage, MmVadPhysicalPages, _MDL::Next, NonPagedPool, NT_SUCCESS, NTSTATUS(), NULL, ObDereferenceObject, ObReferenceObjectByHandle(), PAGE_SHIFT, PAGE_SIZE, PASSIVE_LEVEL, _EPROCESS::Pcb, ProbeForWrite(), ProbeForWritePointer, PsChargePoolQuota(), PsGetCurrentProcess, PsProcessType, PsReturnPoolQuota(), RtlClearAllBits(), RtlClearBits(), RtlFindSetBits(), RtlInitializeBitMap(), RtlSetBits(), SeLockMemoryPrivilege, SeSinglePrivilegeCheck(), Status, TRUE, _MMPTE::u, UNLOCK_AWE, UNLOCK_PFN, UNLOCK_WS, _EPROCESS::VadPhysicalPages, _EPROCESS::VadPhysicalPagesBitMap, and ZeroPte. : This function allocates nonpaged physical pages for the specified subject process. No WSLEs are maintained for this range. The caller must check the NumberOfPages returned to determine how many pages were actually allocated (this number may be less than the requested amount). On success, the user array is filled with the allocated physical page frame numbers (only up to the returned NumberOfPages is filled in). No PTEs are filled here - this gives the application the flexibility to order the address space with no metadata structure imposed by the Mm. Applications do this via NtMapUserPhysicalPages - ie: - Each physical page allocated is set in the process's bitmap. This provides remap, free and unmap a way to validate and rundown these frames. Unmaps may result in a walk of the entire bitmap, but that's ok as unmaps should be less frequent. The win is it saves us from using up system virtual address space to manage these frames. - Note that the same physical frame may NOT be mapped at two different virtual addresses in the process. This makes frees and unmaps substantially faster as no checks for aliasing need be performed. Arguments: ProcessHandle - Supplies an open handle to a process object. NumberOfPages - Supplies a pointer to a variable that supplies the desired size in pages of the allocation. This is filled with the actual number of pages allocated. UserPfnArray - Supplies a pointer to user memory to store the allocated frame numbers into. Return Value: Various NTSTATUS codes. --*/ { ULONG i; KIRQL OldIrql; KIRQL OldIrqlPfn; PEPROCESS Process; KPROCESSOR_MODE PreviousMode; NTSTATUS Status; LOGICAL Attached; LOGICAL WsHeld; ULONG_PTR CapturedNumberOfPages; ULONG_PTR AllocatedPages; ULONG_PTR MdlRequestInPages; ULONG_PTR TotalAllocatedPages; PMDL MemoryDescriptorList; PMDL MemoryDescriptorList2; PMDL MemoryDescriptorHead; PPFN_NUMBER MdlPage; PRTL_BITMAP BitMap; ULONG BitMapSize; ULONG BitMapIndex; PMMPFN Pfn1; PHYSICAL_ADDRESS LowAddress; PHYSICAL_ADDRESS MdlLowAddress; PHYSICAL_ADDRESS HighAddress; PHYSICAL_ADDRESS SkipBytes; PMMPTE PointerPte; MMPTE OldPteContents; MMPTE_FLUSH_LIST PteFlushList; ULONG SizeOfBitMap; ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL); Attached = FALSE; WsHeld = FALSE; // // Check the allocation type field. // PreviousMode = KeGetPreviousMode(); // // Establish an exception handler, probe the specified addresses // for write access and capture the initial values. // try { // // Capture the number of pages. // if (PreviousMode != KernelMode) { ProbeForWritePointer (NumberOfPages); CapturedNumberOfPages = *NumberOfPages; if (CapturedNumberOfPages == 0) { return STATUS_SUCCESS; } if (CapturedNumberOfPages > (MAXULONG_PTR / sizeof(ULONG_PTR))) { return STATUS_INVALID_PARAMETER_2; } ProbeForWrite (UserPfnArray, (ULONG)(CapturedNumberOfPages * sizeof (ULONG_PTR)), sizeof(PULONG_PTR)); } else { CapturedNumberOfPages = *NumberOfPages; } } except (ExSystemExceptionFilter()) { // // If an exception occurs during the probe or capture // of the initial values, then handle the exception and // return the exception code as the status value. // return GetExceptionCode(); } // // Reference the specified process handle for VM_OPERATION access. // if (ProcessHandle == NtCurrentProcess()) { Process = PsGetCurrentProcess(); } else { Status = ObReferenceObjectByHandle ( ProcessHandle, PROCESS_VM_OPERATION, PsProcessType, PreviousMode, (PVOID *)&Process, NULL ); if (!NT_SUCCESS(Status)) { return Status; } } if (!SeSinglePrivilegeCheck (SeLockMemoryPrivilege, PreviousMode)) { if (ProcessHandle != NtCurrentProcess()) { ObDereferenceObject (Process); } return STATUS_PRIVILEGE_NOT_HELD; } // // If the specified process is not the current process, attach // to the specified process. // if (PsGetCurrentProcess() != Process) { KeAttachProcess (&Process->Pcb); Attached = TRUE; } BitMapSize = 0; // // Get the working set mutex to synchronize. This also blocks APCs so // an APC which takes a page fault does not corrupt various structures. // LOCK_WS (Process); WsHeld = TRUE; // // Make sure the address space was not deleted, If so, return an error. // if (Process->AddressSpaceDeleted != 0) { Status = STATUS_PROCESS_IS_TERMINATING; goto ErrorReturn; } // // Create the physical pages bitmap if it does not already exist. // LockMemory privilege is required. // BitMap = Process->VadPhysicalPagesBitMap; if (BitMap == NULL) { BitMapSize = sizeof(RTL_BITMAP) + (ULONG)((((MmHighestPossiblePhysicalPage + 1) + 31) / 32) * 4); BitMap = ExAllocatePoolWithTag (NonPagedPool, BitMapSize, 'LdaV'); if (BitMap == NULL) { Status = STATUS_INSUFFICIENT_RESOURCES; goto ErrorReturn; } RtlInitializeBitMap (BitMap, (PULONG)(BitMap + 1), (ULONG)(MmHighestPossiblePhysicalPage + 1)); RtlClearAllBits (BitMap); try { // // Charge quota for the nonpaged pool for the bitmap. This is // done here rather than by using ExAllocatePoolWithQuota // so the process object is not referenced by the quota charge. // PsChargePoolQuota (Process, NonPagedPool, BitMapSize); } except (EXCEPTION_EXECUTE_HANDLER) { Status = GetExceptionCode(); ExFreePool (BitMap); goto ErrorReturn; } SizeOfBitMap = BitMap->SizeOfBitMap; } else { // // It's ok to snap this without a lock. // SizeOfBitMap = Process->VadPhysicalPagesBitMap->SizeOfBitMap; } AllocatedPages = 0; TotalAllocatedPages = 0; MemoryDescriptorHead = NULL; SkipBytes.QuadPart = 0; // // Allocate from the top of memory going down to preserve low pages // for 32/24-bit device drivers. Just under 4gb is the maximum allocation // per MDL so the ByteCount field does not overflow. // HighAddress.QuadPart = ((ULONGLONG)(SizeOfBitMap - 1)) << PAGE_SHIFT; if (HighAddress.QuadPart > (ULONGLONG)0x100000000) { LowAddress.QuadPart = (ULONGLONG)0x100000000; } else { LowAddress.QuadPart = LOWEST_USABLE_PHYSICAL_ADDRESS; if (LowAddress.QuadPart >= HighAddress.QuadPart) { if (BitMapSize) { ExFreePool (BitMap); PsReturnPoolQuota (Process, NonPagedPool, BitMapSize); } Status = STATUS_INSUFFICIENT_RESOURCES; goto ErrorReturn; } } MdlLowAddress = LowAddress; do { MdlRequestInPages = CapturedNumberOfPages - TotalAllocatedPages; if (MdlRequestInPages > (ULONG_PTR)((MAXULONG - PAGE_SIZE) >> PAGE_SHIFT)) { MdlRequestInPages = (ULONG_PTR)((MAXULONG - PAGE_SIZE) >> PAGE_SHIFT); } // // Note this allocation returns zeroed pages. // MemoryDescriptorList = MmAllocatePagesForMdl (MdlLowAddress, HighAddress, SkipBytes, MdlRequestInPages << PAGE_SHIFT); if (MemoryDescriptorList != NULL) { MemoryDescriptorList->Next = MemoryDescriptorHead; MemoryDescriptorHead = MemoryDescriptorList; MdlPage = (PPFN_NUMBER)(MemoryDescriptorList + 1); AllocatedPages = MemoryDescriptorList->ByteCount >> PAGE_SHIFT; TotalAllocatedPages += AllocatedPages; LOCK_PFN (OldIrqlPfn); MmVadPhysicalPages += AllocatedPages; UNLOCK_PFN (OldIrqlPfn); // // The per-process WS lock guards updates to // Process->VadPhysicalPages. // Process->VadPhysicalPages += AllocatedPages; #if defined(_ALPHA_) && !defined(_AXP64_) if (BitMapSize == 0) { LOCK_AWE (Process, OldIrql); } #endif // // Update the allocation bitmap for each allocated frame. // Note the PFN lock is not needed to modify the PteAddress below. // In fact, even the AWE lock is not needed (except on Alpha32 due // to word tearing in an already existing bitmap) as these pages // are brand new. // for (i = 0; i < AllocatedPages; i += 1) { ASSERT (*MdlPage >= LOWEST_USABLE_PHYSICAL_PAGE); BitMapIndex = MI_FRAME_TO_BITMAP_INDEX(*MdlPage); ASSERT (BitMapIndex < BitMap->SizeOfBitMap); ASSERT (RtlCheckBit (BitMap, BitMapIndex) == 0); #if defined (_WIN64) // // This may become a problem for 64-bit systems with > 32tb // of physical memory as the 2nd parameter to RtlSetBits is // a ULONG. // ASSERT (*MdlPage < 0x100000000); #endif Pfn1 = MI_PFN_ELEMENT (*MdlPage); ASSERT (MI_PFN_IS_AWE (Pfn1)); Pfn1->PteAddress = (PMMPTE)0; ASSERT (Pfn1->u2.ShareCount == 1); RtlSetBits (BitMap, BitMapIndex, 1L); MdlPage += 1; } #if defined(_ALPHA_) && !defined(_AXP64_) if (BitMapSize == 0) { UNLOCK_AWE (Process, OldIrql); } #endif ASSERT (TotalAllocatedPages <= CapturedNumberOfPages); if (TotalAllocatedPages == CapturedNumberOfPages) { break; } // // Try the same memory range again - there might be more pages // left in it that can be claimed as a truncated MDL had to be // used for the last request. // continue; } if (LowAddress.QuadPart == LOWEST_USABLE_PHYSICAL_ADDRESS) { // // No (more) pages available. If this becomes a common situation, // all the working sets could be flushed here. // if (TotalAllocatedPages == 0) { if (BitMapSize) { ExFreePool (BitMap); PsReturnPoolQuota (Process, NonPagedPool, BitMapSize); } Status = STATUS_INSUFFICIENT_RESOURCES; goto ErrorReturn; } // // Make do with what we've gotten so far. // break; } ASSERT (HighAddress.QuadPart > (ULONGLONG)0x100000000); HighAddress.QuadPart = (ULONGLONG)0x100000000 - 1; LowAddress.QuadPart = LOWEST_USABLE_PHYSICAL_ADDRESS; MdlLowAddress = LowAddress; } while (TRUE); ASSERT (TotalAllocatedPages != 0); if (BitMapSize != 0) { // // If this API resulted in the creation of the bitmap, then set it // in the process structure now. No need for locking around this. // Process->VadPhysicalPagesBitMap = BitMap; } UNLOCK_WS (Process); WsHeld = FALSE; if (Attached == TRUE) { KeDetachProcess(); Attached = FALSE; } // // Establish an exception handler and carefully write out the // number of pages and the frame numbers. // try { ASSERT (TotalAllocatedPages <= CapturedNumberOfPages); *NumberOfPages = TotalAllocatedPages; MemoryDescriptorList = MemoryDescriptorHead; while (MemoryDescriptorList != NULL) { MdlPage = (PPFN_NUMBER)(MemoryDescriptorList + 1); AllocatedPages = MemoryDescriptorList->ByteCount >> PAGE_SHIFT; for (i = 0; i < AllocatedPages; i += 1) { *UserPfnArray = *(PULONG_PTR)MdlPage; ASSERT (MI_PFN_ELEMENT(*MdlPage)->u2.ShareCount == 1); UserPfnArray += 1; MdlPage += 1; } MemoryDescriptorList = MemoryDescriptorList->Next; } Status = STATUS_SUCCESS; } except (ExSystemExceptionFilter()) { // // If anything went wrong communicating the pages back to the user // then the entire system service is rolled back. // Status = GetExceptionCode(); MemoryDescriptorList = MemoryDescriptorHead; PteFlushList.Count = 0; if (PsGetCurrentProcess() != Process) { KeAttachProcess (&Process->Pcb); Attached = TRUE; } LOCK_WS (Process); WsHeld = TRUE; if (Process->AddressSpaceDeleted != 0) { Status = STATUS_PROCESS_IS_TERMINATING; goto ErrorReturn; } // // AWE lock protection is needed here to prevent the malicious app // that is mapping these pages between our allocation and our free // below. // LOCK_AWE (Process, OldIrql); while (MemoryDescriptorList != NULL) { AllocatedPages = MemoryDescriptorList->ByteCount >> PAGE_SHIFT; MdlPage = (PPFN_NUMBER)(MemoryDescriptorList + 1); for (i = 0; i < AllocatedPages; i += 1) { BitMapIndex = MI_FRAME_TO_BITMAP_INDEX(*MdlPage); ASSERT (BitMapIndex < BitMap->SizeOfBitMap); ASSERT (RtlCheckBit (BitMap, BitMapIndex) == 1); #if defined (_WIN64) // // This may become a problem for 64-bit systems with > 32tb // of physical memory as the 2nd parameter to RtlSetBits is // a ULONG. // ASSERT (*MdlPage < 0x100000000); #endif RtlClearBits (BitMap, BitMapIndex, 1L); // // Note the PFN lock is not needed for the operations below. // Pfn1 = MI_PFN_ELEMENT (*MdlPage); ASSERT (MI_PFN_IS_AWE (Pfn1)); // // The frame cannot be currently mapped in any Vad unless a // malicious app is trying random pages in an attempt to // corrupt the system. Prevent this behavior by checking // the sharecount and handling it properly here. // if (Pfn1->u2.ShareCount != 1) { ASSERT (Pfn1->u2.ShareCount == 2); Pfn1->u2.ShareCount -= 1; PointerPte = Pfn1->PteAddress; Pfn1->PteAddress = (PMMPTE)0; OldPteContents = *PointerPte; ASSERT (OldPteContents.u.Hard.Valid == 1); if (PteFlushList.Count != MM_MAXIMUM_FLUSH_COUNT) { PteFlushList.FlushVa[PteFlushList.Count] = MiGetVirtualAddressMappedByPte (PointerPte); PteFlushList.FlushPte[PteFlushList.Count] = PointerPte; PteFlushList.Count += 1; } MI_WRITE_INVALID_PTE (PointerPte, ZeroPte); } ASSERT (Pfn1->u2.ShareCount == 1); MI_SET_PFN_DELETED(Pfn1); MdlPage += 1; } Process->VadPhysicalPages -= AllocatedPages; LOCK_PFN (OldIrqlPfn); MmVadPhysicalPages -= AllocatedPages; UNLOCK_PFN (OldIrqlPfn); MemoryDescriptorList = MemoryDescriptorList->Next; } // // Flush the TB entries for any pages which a malicious user may // have mapped. Note ZeroPte is only used when the FlushPte[0] // field is nonzero or if only a single PTE is being flushed. // MiFlushUserPhysicalPteList (&PteFlushList); // // Carefully check to see if the bitmap can be freed. // Remember the working set mutex was dropped, so other threads may // have created or added to the bitmap. If there is no one else using // the bitmap and it's empty, free the bitmap and return its quota. // Keep in mind that this thread may not have even been the creator. // if (Process->VadPhysicalPages == 0) { BitMap = Process->VadPhysicalPagesBitMap; ASSERT (BitMap != NULL); ASSERT (RtlFindSetBits (BitMap, 1, 0) == 0xFFFFFFFF); BitMapSize = sizeof(RTL_BITMAP) + (ULONG)((((MmHighestPossiblePhysicalPage + 1) + 31) / 32) * 4); Process->VadPhysicalPagesBitMap = NULL; ExFreePool (BitMap); PsReturnPoolQuota (Process, NonPagedPool, BitMapSize); } else { ASSERT (Process->VadPhysicalPagesBitMap != NULL); } UNLOCK_AWE (Process, OldIrql); UNLOCK_WS (Process); WsHeld = FALSE; // // Now that we're back at APC level or below, free the pages. // MemoryDescriptorList = MemoryDescriptorHead; while (MemoryDescriptorList != NULL) { MmFreePagesFromMdl (MemoryDescriptorList); MemoryDescriptorList = MemoryDescriptorList->Next; } // // Fall through... // } // // Free the space consumed by the MDLs now that the page frame numbers // have been saved in the bitmap and copied to the user. // MemoryDescriptorList = MemoryDescriptorHead; while (MemoryDescriptorList != NULL) { MemoryDescriptorList2 = MemoryDescriptorList->Next; ExFreePool (MemoryDescriptorList); MemoryDescriptorList = MemoryDescriptorList2; } ErrorReturn: if (WsHeld == TRUE) { UNLOCK_WS (Process); } if (Attached == TRUE) { KeDetachProcess(); } if (ProcessHandle != NtCurrentProcess()) { ObDereferenceObject (Process); } return Status; }

NTSTATUS NtFreeUserPhysicalPages (  IN HANDLE  ProcessHandle, IN OUT PULONG_PTR  NumberOfPages, IN PULONG_PTR  UserPfnArray )

Definition at line 1624 of file physical.c. References _EPROCESS::AddressSpaceDeleted, ASSERT, BitMap, _MDL::ByteCount, COPY_STACK_SIZE, _MMPTE_FLUSH_LIST::Count, EXCEPTION_EXECUTE_HANDLER, ExFreePool(), ExSystemExceptionFilter(), FALSE, _MMPTE_FLUSH_LIST::FlushPte, _MMPTE_FLUSH_LIST::FlushVa, KeAttachProcess(), KeDetachProcess(), KernelMode, KPROCESSOR_MODE, L, LOCK_AWE, LOCK_PFN2, LOCK_WS, LOWEST_USABLE_PHYSICAL_PAGE, MI_FRAME_TO_BITMAP_INDEX, MI_PFN_ELEMENT, MI_PFN_IS_AWE, MI_SET_PFN_DELETED, MI_WRITE_INVALID_PTE, MiFlushUserPhysicalPteList(), MiGetVirtualAddressMappedByPte, MM_MAXIMUM_FLUSH_COUNT, MmCreateMdl(), MmFreePagesFromMdl(), MmInitializeMdl, MmVadPhysicalPages, NT_SUCCESS, NTSTATUS(), NULL, ObDereferenceObject, ObReferenceObjectByHandle(), PAGE_SHIFT, PASSIVE_LEVEL, _EPROCESS::Pcb, ProbeForRead, ProbeForWritePointer, PsGetCurrentProcess, PsProcessType, RtlClearBits(), Status, TRUE, _MMPTE::u, UNLOCK_AWE, UNLOCK_PFN2, UNLOCK_WS, _EPROCESS::VadPhysicalPages, _EPROCESS::VadPhysicalPagesBitMap, and ZeroPte. : This function frees the nonpaged physical pages for the specified subject process. Any PTEs referencing these pages are also invalidated. Note there is no need to walk the entire VAD tree to clear the PTEs that match each page as each physical page can only be mapped at a single virtual address (alias addresses within the VAD are not allowed). Arguments: ProcessHandle - Supplies an open handle to a process object. NumberOfPages - Supplies the size in pages of the allocation to delete. Returns the actual number of pages deleted. UserPfnArray - Supplies a pointer to memory to retrieve the page frame numbers from. Return Value: Various NTSTATUS codes. --*/ { ULONG i; KIRQL OldIrql; KIRQL OldIrqlPfn; ULONG_PTR CapturedNumberOfPages; PMDL MemoryDescriptorList; PPFN_NUMBER MdlPage; PFN_NUMBER PagesInMdl; PFN_NUMBER PageFrameIndex; PRTL_BITMAP BitMap; ULONG BitMapIndex; ULONG_PTR PagesProcessed; PFN_NUMBER MdlHack[(sizeof(MDL) / sizeof(PFN_NUMBER)) + COPY_STACK_SIZE]; ULONG_PTR MdlPages; ULONG_PTR NumberOfBytes; PEPROCESS Process; KPROCESSOR_MODE PreviousMode; NTSTATUS Status; LOGICAL Attached; PMMPFN Pfn1; LOGICAL WsHeld; LOGICAL AweLockHeld; LOGICAL OnePassComplete; LOGICAL ProcessReferenced; MMPTE_FLUSH_LIST PteFlushList; PMMPTE PointerPte; MMPTE OldPteContents; ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL); // // Establish an exception handler, probe the specified addresses // for read access and capture the page frame numbers. // PreviousMode = KeGetPreviousMode(); if (PreviousMode != KernelMode) { try { ProbeForWritePointer (NumberOfPages); CapturedNumberOfPages = *NumberOfPages; // // Initialize the NumberOfPages freed to zero so the user can be // reasonably informed about errors that occur midway through // the transaction. // *NumberOfPages = 0; } except (ExSystemExceptionFilter()) { // // If an exception occurs during the probe or capture // of the initial values, then handle the exception and // return the exception code as the status value. // return GetExceptionCode(); } } else { CapturedNumberOfPages = *NumberOfPages; } if (CapturedNumberOfPages == 0) { return STATUS_INVALID_PARAMETER_2; } OnePassComplete = FALSE; PagesProcessed = 0; MemoryDescriptorList = (PMDL)0; if (CapturedNumberOfPages > COPY_STACK_SIZE) { // // Ensure the number of pages can fit into an MDL's ByteCount. // if (CapturedNumberOfPages > ((ULONG)MAXULONG >> PAGE_SHIFT)) { MdlPages = (ULONG_PTR)((ULONG)MAXULONG >> PAGE_SHIFT); } else { MdlPages = CapturedNumberOfPages; } while (MdlPages > COPY_STACK_SIZE) { MemoryDescriptorList = MmCreateMdl (NULL, 0, MdlPages << PAGE_SHIFT); if (MemoryDescriptorList != NULL) { break; } MdlPages >>= 1; } } if (MemoryDescriptorList == NULL) { MdlPages = COPY_STACK_SIZE; MemoryDescriptorList = (PMDL)&MdlHack[0]; } WsHeld = FALSE; AweLockHeld = FALSE; ProcessReferenced = FALSE; repeat: if (CapturedNumberOfPages < MdlPages) { MdlPages = CapturedNumberOfPages; } MmInitializeMdl (MemoryDescriptorList, 0, MdlPages << PAGE_SHIFT); MdlPage = (PPFN_NUMBER)(MemoryDescriptorList + 1); NumberOfBytes = MdlPages * sizeof(ULONG_PTR); Attached = FALSE; // // Establish an exception handler, probe the specified addresses // for read access and capture the page frame numbers. // if (PreviousMode != KernelMode) { try { // // Update the user's count so if anything goes wrong, the user can // be reasonably informed about how far into the transaction it // occurred. // *NumberOfPages = PagesProcessed; ProbeForRead (UserPfnArray, NumberOfBytes, sizeof(PULONG_PTR)); RtlCopyMemory ((PVOID)MdlPage, UserPfnArray, NumberOfBytes); } except (ExSystemExceptionFilter()) { // // If an exception occurs during the probe or capture // of the initial values, then handle the exception and // return the exception code as the status value. // Status = GetExceptionCode(); goto ErrorReturn; } } else { RtlCopyMemory ((PVOID)MdlPage, UserPfnArray, NumberOfBytes); } if (OnePassComplete == FALSE) { // // Reference the specified process handle for VM_OPERATION access. // if (ProcessHandle == NtCurrentProcess()) { Process = PsGetCurrentProcess(); } else { Status = ObReferenceObjectByHandle ( ProcessHandle, PROCESS_VM_OPERATION, PsProcessType, PreviousMode, (PVOID *)&Process, NULL ); if (!NT_SUCCESS(Status)) { goto ErrorReturn; } ProcessReferenced = TRUE; } } // // If the specified process is not the current process, attach // to the specified process. // if (PsGetCurrentProcess() != Process) { KeAttachProcess (&Process->Pcb); Attached = TRUE; } // // Get the address creation mutex to block multiple threads from // creating or deleting address space at the same time and // get the working set mutex so virtual address descriptors can // be inserted and walked. Block APCs so an APC which takes a page // fault does not corrupt various structures. // LOCK_WS (Process); WsHeld = TRUE; // // Make sure the address space was not deleted, if so, return an error. // if (Process->AddressSpaceDeleted != 0) { Status = STATUS_PROCESS_IS_TERMINATING; goto ErrorReturn; } LOCK_AWE (Process, OldIrql); AweLockHeld = TRUE; // // The physical pages bitmap must exist. // BitMap = Process->VadPhysicalPagesBitMap; if (BitMap == NULL) { Status = STATUS_INVALID_PARAMETER_2; goto ErrorReturn; } PteFlushList.Count = 0; Status = STATUS_SUCCESS; for (i = 0; i < MdlPages; i += 1, MdlPage += 1) { PageFrameIndex = *MdlPage; BitMapIndex = MI_FRAME_TO_BITMAP_INDEX(PageFrameIndex); #if defined (_WIN64) // // Ensure the frame is a 32-bit number. // if (BitMapIndex != PageFrameIndex) { Status = STATUS_CONFLICTING_ADDRESSES; break; } #endif // // Frames past the end of the bitmap are not allowed. // if (BitMapIndex >= BitMap->SizeOfBitMap) { Status = STATUS_CONFLICTING_ADDRESSES; break; } // // Frames not in the bitmap are not allowed. // if (RtlCheckBit (BitMap, BitMapIndex) == 0) { Status = STATUS_CONFLICTING_ADDRESSES; break; } ASSERT (PageFrameIndex >= LOWEST_USABLE_PHYSICAL_PAGE); PagesProcessed += 1; #if defined (_WIN64) // // This may become a problem for 64-bit systems with > 32tb // of physical memory as the 2nd parameter to RtlClearBits is // a ULONG. // ASSERT (PageFrameIndex < 0x100000000); #endif RtlClearBits (BitMap, BitMapIndex, 1L); Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); ASSERT (MI_PFN_IS_AWE (Pfn1)); #if DBG if (Pfn1->u2.ShareCount == 1) { ASSERT (Pfn1->PteAddress == (PMMPTE)0); } else if (Pfn1->u2.ShareCount == 2) { ASSERT (Pfn1->PteAddress != (PMMPTE)0); } else { ASSERT (FALSE); } #endif // // If the frame is currently mapped in the Vad then the PTE must // be cleared and the TB entry flushed. // if (Pfn1->u2.ShareCount != 1) { Pfn1->u2.ShareCount -= 1; PointerPte = Pfn1->PteAddress; Pfn1->PteAddress = (PMMPTE)0; OldPteContents = *PointerPte; ASSERT (OldPteContents.u.Hard.Valid == 1); if (PteFlushList.Count != MM_MAXIMUM_FLUSH_COUNT) { PteFlushList.FlushVa[PteFlushList.Count] = MiGetVirtualAddressMappedByPte (PointerPte); PteFlushList.FlushPte[PteFlushList.Count] = PointerPte; PteFlushList.Count += 1; } MI_WRITE_INVALID_PTE (PointerPte, ZeroPte); } MI_SET_PFN_DELETED(Pfn1); } // // Flush the TB entries for these pages. Note ZeroPte is only used // when the FlushPte[0] field is nonzero or if only a single PTE is // being flushed. // MiFlushUserPhysicalPteList (&PteFlushList); // // Free the actual pages (this may be a partially filled MDL). // PagesInMdl = MdlPage - (PPFN_NUMBER)(MemoryDescriptorList + 1); // // Set the ByteCount to the actual number of validated pages - the caller // may have lied and we have to sync up here to account for any bogus // frames. // MemoryDescriptorList->ByteCount = (ULONG)(PagesInMdl << PAGE_SHIFT); if (PagesInMdl != 0) { Process->VadPhysicalPages -= PagesInMdl; UNLOCK_AWE (Process, OldIrql); AweLockHeld = FALSE; LOCK_PFN2 (OldIrqlPfn); MmVadPhysicalPages -= PagesInMdl; UNLOCK_PFN2 (OldIrqlPfn); MmFreePagesFromMdl (MemoryDescriptorList); } else { if (AweLockHeld == TRUE) { UNLOCK_AWE (Process, OldIrql); AweLockHeld = FALSE; } } CapturedNumberOfPages -= PagesInMdl; if ((Status == STATUS_SUCCESS) && (CapturedNumberOfPages != 0)) { UNLOCK_WS (Process); WsHeld = FALSE; if (Attached == TRUE) { KeDetachProcess(); Attached = FALSE; } OnePassComplete = TRUE; ASSERT (MdlPages == PagesInMdl); UserPfnArray += MdlPages; // // Do it all again until all the pages are freed or an error occurs. // goto repeat; } // // Fall through. // ErrorReturn: if (AweLockHeld == TRUE) { UNLOCK_AWE (Process, OldIrql); } if (WsHeld == TRUE) { UNLOCK_WS (Process); } // // Free any pool acquired for holding MDLs. // if (MemoryDescriptorList != (PMDL)&MdlHack[0]) { ExFreePool (MemoryDescriptorList); } if (Attached == TRUE) { KeDetachProcess(); } // // Establish an exception handler and carefully write out the // number of pages actually processed. // try { *NumberOfPages = PagesProcessed; } except (EXCEPTION_EXECUTE_HANDLER) { // // Return success at this point even if the results // cannot be written. // NOTHING; } if (ProcessReferenced == TRUE) { ObDereferenceObject (Process); } return Status; }

NTSTATUS NtMapUserPhysicalPages (  IN PVOID  VirtualAddress, IN ULONG_PTR  NumberOfPages, IN PULONG_PTR UserPfnArray  OPTIONAL )

Definition at line 85 of file physical.c. References ASSERT, BitMap, COPY_STACK_SIZE, _MMPTE_FLUSH_LIST::Count, _MI_PHYSICAL_VIEW::EndVa, ExAllocatePoolWithTag, EXCEPTION_EXECUTE_HANDLER, ExFreePool(), _MMPTE_FLUSH_LIST::FlushPte, _MMPTE_FLUSH_LIST::FlushVa, LOCK_AWE, LOWEST_USABLE_PHYSICAL_PAGE, MI_FRAME_TO_BITMAP_INDEX, MI_MAKE_VALID_PTE, MI_PFN_ELEMENT, MI_PFN_IS_AWE, MI_SET_PTE_DIRTY, MI_WRITE_INVALID_PTE, MI_WRITE_VALID_PTE, MiFlushUserPhysicalPteList(), MiGetPteAddress, MM_MAXIMUM_FLUSH_COUNT, MM_READWRITE, NonPagedPool, NTSTATUS(), NULL, PAGE_ALIGN, PAGE_SHIFT, PAGE_SIZE, PASSIVE_LEVEL, _EPROCESS::PhysicalVadList, ProbeForRead, PsGetCurrentProcess, _MMPFN::PteAddress, _MI_PHYSICAL_VIEW::StartVa, Status, _MMPTE::u, _MMVAD::u, _MMPFN::u2, UNLOCK_AWE, _MI_PHYSICAL_VIEW::Vad, _EPROCESS::VadPhysicalPagesBitMap, VOID(), and ZeroPte. : This function maps the specified nonpaged physical pages into the specified user address range. Note no WSLEs are maintained for this range as it is all nonpaged. Arguments: VirtualAddress - Supplies a user virtual address within a UserPhysicalPages Vad. NumberOfPages - Supplies the number of pages to map. UserPfnArray - Supplies a pointer to the page frame numbers to map in. If this is zero, then the virtual addresses are set to NO_ACCESS. Return Value: Various NTSTATUS codes. --*/ { PMMVAD FoundVad; KIRQL OldIrql; ULONG_PTR i; PEPROCESS Process; PMMPTE PointerPte; PVOID EndAddress; PFN_NUMBER PageFrameIndex; PMMPFN Pfn1; NTSTATUS Status; MMPTE_FLUSH_LIST PteFlushList; PVOID PoolArea; PPFN_NUMBER FrameList; ULONG BitMapIndex; ULONG_PTR StackArray[COPY_STACK_SIZE]; MMPTE OldPteContents; MMPTE NewPteContents; ULONG_PTR NumberOfBytes; PRTL_BITMAP BitMap; PLIST_ENTRY NextEntry; PMI_PHYSICAL_VIEW PhysicalView; ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL); if (NumberOfPages > (MAXULONG_PTR / PAGE_SIZE)) { return STATUS_INVALID_PARAMETER_2; } VirtualAddress = PAGE_ALIGN(VirtualAddress); EndAddress = (PVOID)((PCHAR)VirtualAddress + (NumberOfPages << PAGE_SHIFT) -1); if (EndAddress <= VirtualAddress) { return STATUS_INVALID_PARAMETER_2; } // // Carefully probe and capture all user parameters. // PoolArea = (PVOID)&StackArray[0]; if (ARGUMENT_PRESENT(UserPfnArray)) { NumberOfBytes = NumberOfPages * sizeof(ULONG_PTR); if (NumberOfPages > COPY_STACK_SIZE) { PoolArea = ExAllocatePoolWithTag (NonPagedPool, NumberOfBytes, 'wRmM'); if (PoolArea == NULL) { return STATUS_INSUFFICIENT_RESOURCES; } } // // Capture the specified page frame numbers. // try { ProbeForRead (UserPfnArray, NumberOfBytes, sizeof(ULONG_PTR)); RtlCopyMemory (PoolArea, UserPfnArray, NumberOfBytes); } except(EXCEPTION_EXECUTE_HANDLER) { if (PoolArea != (PVOID)&StackArray[0]) { ExFreePool (PoolArea); } return GetExceptionCode(); } } PointerPte = MiGetPteAddress (VirtualAddress); Process = PsGetCurrentProcess(); // // The AWE lock protects insertion/removal of Vads into each process' // PhysicalVadList. It also protects creation/deletion and adds/removes // of the VadPhysicalPagesBitMap. Finally, it protects the PFN // modifications for pages in the bitmap. // LOCK_AWE (Process, OldIrql); // // The physical pages bitmap must exist. // BitMap = Process->VadPhysicalPagesBitMap; if (BitMap == NULL) { Status = STATUS_INVALID_PARAMETER_1; goto ErrorReturn; } // // Note that the PFN lock is not needed to traverse this list (even though // MmProbeAndLockPages uses it), because all modifications are made while // also holding the AWE lock. // // The PhysicalVadList should typically have just one entry - the view // we're looking for, so this traverse should be quick. // FoundVad = NULL; NextEntry = Process->PhysicalVadList.Flink; while (NextEntry != &Process->PhysicalVadList) { PhysicalView = CONTAINING_RECORD(NextEntry, MI_PHYSICAL_VIEW, ListEntry); if (PhysicalView->Vad->u.VadFlags.UserPhysicalPages == 1) { if ((VirtualAddress >= (PVOID)PhysicalView->StartVa) && (EndAddress <= (PVOID)PhysicalView->EndVa)) { FoundVad = PhysicalView->Vad; break; } } NextEntry = NextEntry->Flink; continue; } if (FoundVad == (PMMVAD)NULL) { // // No virtual address is reserved at the specified base address, // return an error. // Status = STATUS_INVALID_PARAMETER_1; goto ErrorReturn; } // // Ensure the PFN element corresponding to each specified page is owned // by the specified VAD. // // Since this ownership can only be changed while holding this process' // working set lock, the PFN can be scanned here without holding the PFN // lock. // // Note the PFN lock is not needed because any race with MmProbeAndLockPages // can only result in the I/O going to the old page or the new page. // If the user breaks the rules, the PFN database (and any pages being // windowed here) are still protected because of the reference counts // on the pages with inprogress I/O. This is possible because NO pages // are actually freed here - they are just windowed. // PteFlushList.Count = 0; if (ARGUMENT_PRESENT(UserPfnArray)) { // // By keeping the PFN bitmap in the VAD (instead of in the PFN // database itself), a few benefits are realized: // // 1. No need to acquire the PFN lock here. // 2. Faster handling of PFN databases with holes. // 3. Transparent support for dynamic PFN database growth. // 4. Less nonpaged memory is used (for the bitmap vs adding a // field to the PFN) on systems with no unused pack space in // the PFN database, presuming not many of these VADs get // allocated. // // // The first pass here ensures all the frames are secure. // // // N.B. This implies that PFN_NUMBER is always ULONG_PTR in width // as PFN_NUMBER is not exposed to application code today. // FrameList = (PPFN_NUMBER)PoolArea; for (i = 0; i < NumberOfPages; i += 1, FrameList += 1) { PageFrameIndex = *FrameList; // // Frames past the end of the bitmap are not allowed. // BitMapIndex = MI_FRAME_TO_BITMAP_INDEX(PageFrameIndex); #if defined (_WIN64) // // Ensure the frame is a 32-bit number. // if (BitMapIndex != PageFrameIndex) { Status = STATUS_CONFLICTING_ADDRESSES; goto ErrorReturn0; } #endif if (BitMapIndex >= BitMap->SizeOfBitMap) { Status = STATUS_CONFLICTING_ADDRESSES; goto ErrorReturn0; } // // Frames not in the bitmap are not allowed. // if (RtlCheckBit (BitMap, BitMapIndex) == 0) { Status = STATUS_CONFLICTING_ADDRESSES; goto ErrorReturn0; } // // The frame must not be already mapped anywhere. // Or be passed in twice in different spots in the array. // Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); if (Pfn1->u2.ShareCount != 1) { Status = STATUS_INVALID_PARAMETER_3; goto ErrorReturn0; } ASSERT (MI_PFN_IS_AWE (Pfn1)); // // Mark the frame as "about to be mapped". // Pfn1->u2.ShareCount = 3; ASSERT (PageFrameIndex >= LOWEST_USABLE_PHYSICAL_PAGE); } // // This pass actually inserts them all into the page table pages and // the TBs now that we know the frames are good. // FrameList = (PPFN_NUMBER)PoolArea; MI_MAKE_VALID_PTE (NewPteContents, PageFrameIndex, MM_READWRITE, PointerPte); MI_SET_PTE_DIRTY (NewPteContents); for (i = 0; i < NumberOfPages; i += 1) { PageFrameIndex = *FrameList; NewPteContents.u.Hard.PageFrameNumber = PageFrameIndex; OldPteContents = *PointerPte; // // Flush the TB entry for this page if it's valid. // if (OldPteContents.u.Hard.Valid == 1) { Pfn1 = MI_PFN_ELEMENT (OldPteContents.u.Hard.PageFrameNumber); ASSERT (Pfn1->PteAddress != (PMMPTE)0); ASSERT (Pfn1->u2.ShareCount == 2); Pfn1->u2.ShareCount -= 1; Pfn1->PteAddress = (PMMPTE)0; #if defined (_X86PAE_) (VOID)KeInterlockedSwapPte ((PHARDWARE_PTE)PointerPte, (PHARDWARE_PTE)&NewPteContents); #else *PointerPte = NewPteContents; #endif if (PteFlushList.Count != MM_MAXIMUM_FLUSH_COUNT) { PteFlushList.FlushVa[PteFlushList.Count] = VirtualAddress; PteFlushList.FlushPte[PteFlushList.Count] = PointerPte; PteFlushList.Count += 1; } } else { MI_WRITE_VALID_PTE (PointerPte, NewPteContents); } Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); ASSERT (Pfn1->PteAddress == (PMMPTE)0); ASSERT (Pfn1->u2.ShareCount == 3); Pfn1->u2.ShareCount = 2; Pfn1->PteAddress = PointerPte; VirtualAddress = (PVOID)((PCHAR)VirtualAddress + PAGE_SIZE); PointerPte += 1; FrameList += 1; } } else { // // Set the specified virtual address range to no access. // for (i = 0; i < NumberOfPages; i += 1) { OldPteContents = *PointerPte; // // Flush the TB entry for this page if it's valid. // if (OldPteContents.u.Hard.Valid == 1) { Pfn1 = MI_PFN_ELEMENT (OldPteContents.u.Hard.PageFrameNumber); ASSERT (Pfn1->PteAddress != (PMMPTE)0); ASSERT (Pfn1->u2.ShareCount == 2); ASSERT (MI_PFN_IS_AWE (Pfn1)); Pfn1->u2.ShareCount -= 1; Pfn1->PteAddress = (PMMPTE)0; if (PteFlushList.Count != MM_MAXIMUM_FLUSH_COUNT) { PteFlushList.FlushVa[PteFlushList.Count] = VirtualAddress; PteFlushList.FlushPte[PteFlushList.Count] = PointerPte; PteFlushList.Count += 1; } } MI_WRITE_INVALID_PTE (PointerPte, ZeroPte); VirtualAddress = (PVOID)((PCHAR)VirtualAddress + PAGE_SIZE); PointerPte += 1; } } // // Flush the TB entries for these pages. Note ZeroPte is only used // when the FlushPte[0] field is nonzero or if only a single PTE is // being flushed. // if (PteFlushList.Count != 0) { MiFlushUserPhysicalPteList (&PteFlushList); } UNLOCK_AWE (Process, OldIrql); if (PoolArea != (PVOID)&StackArray[0]) { ExFreePool (PoolArea); } return STATUS_SUCCESS; ErrorReturn0: while (i != 0) { FrameList -= 1; PageFrameIndex = *FrameList; Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); ASSERT (Pfn1->u2.ShareCount == 3); Pfn1->u2.ShareCount = 1; i -= 1; } ErrorReturn: UNLOCK_AWE (Process, OldIrql); if (PoolArea != (PVOID)&StackArray[0]) { ExFreePool (PoolArea); } return Status; }

NTSTATUS NtMapUserPhysicalPagesScatter (  IN PVOID *  VirtualAddresses, IN ULONG_PTR  NumberOfPages, IN PULONG_PTR UserPfnArray  OPTIONAL )

Definition at line 498 of file physical.c. References ASSERT, BitMap, COPY_STACK_SIZE, _MMPTE_FLUSH_LIST::Count, _MI_PHYSICAL_VIEW::EndVa, ExAllocatePoolWithTag, EXCEPTION_EXECUTE_HANDLER, ExFreePool(), _MMPTE_FLUSH_LIST::FlushPte, _MMPTE_FLUSH_LIST::FlushVa, LOCK_AWE, LOWEST_USABLE_PHYSICAL_PAGE, MI_FRAME_TO_BITMAP_INDEX, MI_MAKE_VALID_PTE, MI_PFN_ELEMENT, MI_PFN_IS_AWE, MI_SET_PTE_DIRTY, MI_WRITE_INVALID_PTE, MI_WRITE_VALID_PTE, MiFlushUserPhysicalPteList(), MiGetPteAddress, MM_MAXIMUM_FLUSH_COUNT, MM_READWRITE, NonPagedPool, NTSTATUS(), NULL, PAGE_SIZE, PASSIVE_LEVEL, _EPROCESS::PhysicalVadList, ProbeForRead, PsGetCurrentProcess, _MMPFN::PteAddress, _MI_PHYSICAL_VIEW::StartVa, Status, _MMPTE::u, _MMVAD::u, _MMPFN::u2, UNLOCK_AWE, _MI_PHYSICAL_VIEW::Vad, _EPROCESS::VadPhysicalPagesBitMap, VOID(), and ZeroPte. : This function maps the specified nonpaged physical pages into the specified user address range. Note no WSLEs are maintained for this range as it is all nonpaged. Arguments: VirtualAddresses - Supplies a pointer to an array of user virtual addresses within UserPhysicalPages Vads. Each array entry is presumed to map a single page. NumberOfPages - Supplies the number of pages to map. UserPfnArray - Supplies a pointer to the page frame numbers to map in. If this is zero, then the virtual addresses are set to NO_ACCESS. If the array entry is zero then just the corresponding virtual address is set to NO_ACCESS. Return Value: Various NTSTATUS codes. --*/ { PMMVAD FoundVad; KIRQL OldIrql; ULONG_PTR i; PEPROCESS Process; PMMPTE PointerPte; PFN_NUMBER PageFrameIndex; PMMPFN Pfn1; NTSTATUS Status; MMPTE_FLUSH_LIST PteFlushList; PVOID PoolArea; PVOID *PoolVirtualArea; PPFN_NUMBER FrameList; ULONG BitMapIndex; PVOID StackVirtualArray[COPY_STACK_SIZE]; ULONG_PTR StackArray[COPY_STACK_SIZE]; MMPTE OldPteContents; MMPTE NewPteContents; ULONG_PTR NumberOfBytes; PRTL_BITMAP BitMap; PLIST_ENTRY NextEntry; PMI_PHYSICAL_VIEW PhysicalView; PVOID VirtualAddress; ASSERT (KeGetCurrentIrql() == PASSIVE_LEVEL); if (NumberOfPages > (MAXULONG_PTR / PAGE_SIZE)) { return STATUS_INVALID_PARAMETER_2; } // // Carefully probe and capture the user virtual address array. // PoolArea = (PVOID)&StackArray[0]; PoolVirtualArea = (PVOID)&StackVirtualArray[0]; NumberOfBytes = NumberOfPages * sizeof(PVOID); if (NumberOfPages > COPY_STACK_SIZE) { PoolVirtualArea = ExAllocatePoolWithTag (NonPagedPool, NumberOfBytes, 'wRmM'); if (PoolVirtualArea == NULL) { return STATUS_INSUFFICIENT_RESOURCES; } } try { ProbeForRead (VirtualAddresses, NumberOfBytes, sizeof(PVOID)); RtlCopyMemory (PoolVirtualArea, VirtualAddresses, NumberOfBytes); } except(EXCEPTION_EXECUTE_HANDLER) { Status = GetExceptionCode(); goto ErrorReturn2; } // // Carefully probe and capture the user PFN array. // if (ARGUMENT_PRESENT(UserPfnArray)) { NumberOfBytes = NumberOfPages * sizeof(ULONG_PTR); if (NumberOfPages > COPY_STACK_SIZE) { PoolArea = ExAllocatePoolWithTag (NonPagedPool, NumberOfBytes, 'wRmM'); if (PoolArea == NULL) { Status = STATUS_INSUFFICIENT_RESOURCES; goto ErrorReturn2; } } // // Capture the specified page frame numbers. // try { ProbeForRead (UserPfnArray, NumberOfBytes, sizeof(ULONG_PTR)); RtlCopyMemory (PoolArea, UserPfnArray, NumberOfBytes); } except(EXCEPTION_EXECUTE_HANDLER) { Status = GetExceptionCode(); goto ErrorReturn2; } } Process = PsGetCurrentProcess(); // // The AWE lock protects insertion/removal of Vads into each process' // PhysicalVadList. It also protects creation/deletion and adds/removes // of the VadPhysicalPagesBitMap. Finally, it protects the PFN // modifications for pages in the bitmap. // PhysicalView = NULL; LOCK_AWE (Process, OldIrql); // // The physical pages bitmap must exist. // BitMap = Process->VadPhysicalPagesBitMap; if (BitMap == NULL) { Status = STATUS_INVALID_PARAMETER_1; goto ErrorReturn; } // // Note that the PFN lock is not needed to traverse this list (even though // MmProbeAndLockPages uses it), because all modifications are made while // also holding the AWE lock. // // The PhysicalVadList should typically have just one entry - the view // we're looking for, so this traverse should be quick. // for (i = 0; i < NumberOfPages; i += 1) { VirtualAddress = PoolVirtualArea[i]; if (PhysicalView != NULL) { ASSERT (PhysicalView->Vad->u.VadFlags.UserPhysicalPages == 1); if ((VirtualAddress >= (PVOID)PhysicalView->StartVa) && (VirtualAddress <= (PVOID)PhysicalView->EndVa)) { continue; } } FoundVad = NULL; NextEntry = Process->PhysicalVadList.Flink; while (NextEntry != &Process->PhysicalVadList) { PhysicalView = CONTAINING_RECORD(NextEntry, MI_PHYSICAL_VIEW, ListEntry); if (PhysicalView->Vad->u.VadFlags.UserPhysicalPages == 1) { if ((VirtualAddress >= (PVOID)PhysicalView->StartVa) && (VirtualAddress <= (PVOID)PhysicalView->EndVa)) { FoundVad = PhysicalView->Vad; break; } } NextEntry = NextEntry->Flink; continue; } if (FoundVad == (PMMVAD)NULL) { // // No virtual address is reserved at the specified base address, // return an error. // Status = STATUS_INVALID_PARAMETER_1; goto ErrorReturn; } } // // Ensure the PFN element corresponding to each specified page is owned // by the specified VAD. // // Since this ownership can only be changed while holding this process' // working set lock, the PFN can be scanned here without holding the PFN // lock. // // Note the PFN lock is not needed because any race with MmProbeAndLockPages // can only result in the I/O going to the old page or the new page. // If the user breaks the rules, the PFN database (and any pages being // windowed here) are still protected because of the reference counts // on the pages with inprogress I/O. This is possible because NO pages // are actually freed here - they are just windowed. // PteFlushList.Count = 0; if (ARGUMENT_PRESENT(UserPfnArray)) { // // By keeping the PFN bitmap in the VAD (instead of in the PFN // database itself), a few benefits are realized: // // 1. No need to acquire the PFN lock here. // 2. Faster handling of PFN databases with holes. // 3. Transparent support for dynamic PFN database growth. // 4. Less nonpaged memory is used (for the bitmap vs adding a // field to the PFN) on systems with no unused pack space in // the PFN database, presuming not many of these VADs get // allocated. // // // The first pass here ensures all the frames are secure. // // // N.B. This implies that PFN_NUMBER is always ULONG_PTR in width // as PFN_NUMBER is not exposed to application code today. // FrameList = (PPFN_NUMBER)PoolArea; for (i = 0; i < NumberOfPages; i += 1, FrameList += 1) { PageFrameIndex = *FrameList; // // Zero entries are treated as a command to unmap. // if (PageFrameIndex == 0) { continue; } // // Frames past the end of the bitmap are not allowed. // BitMapIndex = MI_FRAME_TO_BITMAP_INDEX(PageFrameIndex); #if defined (_WIN64) // // Ensure the frame is a 32-bit number. // if (BitMapIndex != PageFrameIndex) { Status = STATUS_CONFLICTING_ADDRESSES; goto ErrorReturn0; } #endif if (BitMapIndex >= BitMap->SizeOfBitMap) { Status = STATUS_CONFLICTING_ADDRESSES; goto ErrorReturn0; } // // Frames not in the bitmap are not allowed. // if (RtlCheckBit (BitMap, BitMapIndex) == 0) { Status = STATUS_CONFLICTING_ADDRESSES; goto ErrorReturn0; } // // The frame must not be already mapped anywhere. // Or be passed in twice in different spots in the array. // Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); if (Pfn1->u2.ShareCount != 1) { Status = STATUS_INVALID_PARAMETER_3; goto ErrorReturn0; } ASSERT (MI_PFN_IS_AWE (Pfn1)); // // Mark the frame as "about to be mapped". // Pfn1->u2.ShareCount = 3; ASSERT (PageFrameIndex >= LOWEST_USABLE_PHYSICAL_PAGE); } // // This pass actually inserts them all into the page table pages and // the TBs now that we know the frames are good. // FrameList = (PPFN_NUMBER)PoolArea; MI_MAKE_VALID_PTE (NewPteContents, PageFrameIndex, MM_READWRITE, 0); MI_SET_PTE_DIRTY (NewPteContents); for (i = 0; i < NumberOfPages; i += 1, FrameList += 1) { PageFrameIndex = *FrameList; VirtualAddress = PoolVirtualArea[i]; PointerPte = MiGetPteAddress (VirtualAddress); OldPteContents = *PointerPte; // // Flush the TB entry for this page if it's valid. // if (OldPteContents.u.Hard.Valid == 1) { Pfn1 = MI_PFN_ELEMENT (OldPteContents.u.Hard.PageFrameNumber); ASSERT (Pfn1->PteAddress != (PMMPTE)0); ASSERT (Pfn1->u2.ShareCount == 2); ASSERT (MI_PFN_IS_AWE (Pfn1)); Pfn1->u2.ShareCount -= 1; Pfn1->PteAddress = (PMMPTE)0; if (PageFrameIndex != 0) { NewPteContents.u.Hard.PageFrameNumber = PageFrameIndex; #if defined (_X86PAE_) (VOID)KeInterlockedSwapPte ((PHARDWARE_PTE)PointerPte, (PHARDWARE_PTE)&NewPteContents); #else *PointerPte = NewPteContents; #endif } else { MI_WRITE_INVALID_PTE (PointerPte, ZeroPte); } if (PteFlushList.Count != MM_MAXIMUM_FLUSH_COUNT) { PteFlushList.FlushVa[PteFlushList.Count] = VirtualAddress; PteFlushList.FlushPte[PteFlushList.Count] = PointerPte; PteFlushList.Count += 1; } } else { if (PageFrameIndex != 0) { NewPteContents.u.Hard.PageFrameNumber = PageFrameIndex; MI_WRITE_VALID_PTE (PointerPte, NewPteContents); } } if (PageFrameIndex != 0) { Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); ASSERT (Pfn1->PteAddress == (PMMPTE)0); ASSERT (Pfn1->u2.ShareCount == 3); Pfn1->u2.ShareCount = 2; Pfn1->PteAddress = PointerPte; } } } else { // // Set the specified virtual address range to no access. // for (i = 0; i < NumberOfPages; i += 1) { VirtualAddress = PoolVirtualArea[i]; PointerPte = MiGetPteAddress (VirtualAddress); OldPteContents = *PointerPte; // // Flush the TB entry for this page if it's valid. // if (OldPteContents.u.Hard.Valid == 1) { Pfn1 = MI_PFN_ELEMENT (OldPteContents.u.Hard.PageFrameNumber); ASSERT (Pfn1->PteAddress != (PMMPTE)0); ASSERT (Pfn1->u2.ShareCount == 2); ASSERT (MI_PFN_IS_AWE (Pfn1)); Pfn1->u2.ShareCount -= 1; Pfn1->PteAddress = (PMMPTE)0; if (PteFlushList.Count != MM_MAXIMUM_FLUSH_COUNT) { PteFlushList.FlushVa[PteFlushList.Count] = VirtualAddress; PteFlushList.FlushPte[PteFlushList.Count] = PointerPte; PteFlushList.Count += 1; } } MI_WRITE_INVALID_PTE (PointerPte, ZeroPte); } } // // Flush the TB entries for these pages. Note ZeroPte is only used // when the FlushPte[0] field is nonzero or if only a single PTE is // being flushed. // if (PteFlushList.Count != 0) { MiFlushUserPhysicalPteList (&PteFlushList); } Status = STATUS_SUCCESS; ErrorReturn: UNLOCK_AWE (Process, OldIrql); ErrorReturn2: if (PoolArea != (PVOID)&StackArray[0]) { ExFreePool (PoolArea); } if (PoolVirtualArea != (PVOID)&StackVirtualArray[0]) { ExFreePool (PoolVirtualArea); } return Status; ErrorReturn0: while (i != 0) { FrameList -= 1; PageFrameIndex = *FrameList; if (PageFrameIndex != 0) { Pfn1 = MI_PFN_ELEMENT (PageFrameIndex); ASSERT (Pfn1->u2.ShareCount == 3); ASSERT (MI_PFN_IS_AWE (Pfn1)); Pfn1->u2.ShareCount = 1; } i -= 1; } goto ErrorReturn; }

---

Variable Documentation

ULONG_PTR MmVadPhysicalPages

Definition at line 76 of file physical.c. Referenced by MiUpdateVadPhysicalPages(), NtAllocateUserPhysicalPages(), and NtFreeUserPhysicalPages().

---