wslist.c

Go to the documentation of this file.

/*++

Copyright (c) 1989  Microsoft Corporation

Module Name:

   wslist.c

Abstract:

    This module contains routines which operate on the working
    set list structure.

Author:

    Lou Perazzoli (loup) 10-Apr-1989
    Landy Wang (landyw) 02-Jun-1997

Revision History:

--*/

#include "mi.h"

#pragma alloc_text(INIT, MiInitializeSessionWsSupport)
#pragma alloc_text(PAGE, MmAssignProcessToJob)
#pragma alloc_text(PAGEHYDRA, MiSessionInitializeWorkingSetList)

#define MM_SYSTEM_CACHE_THRESHOLD ((1024*1024) / PAGE_SIZE)

extern ULONG MmMaximumWorkingSetSize;

ULONG MmFaultsTakenToGoAboveMaxWs = 100;
ULONG MmFaultsTakenToGoAboveMinWs = 16;

ULONG MmSystemCodePage;
ULONG MmSystemCachePage;
ULONG MmPagedPoolPage;
ULONG MmSystemDriverPage;

#ifdef _MI_USE_CLAIMS_
extern BOOLEAN MiReplacing;
#endif

#define MM_RETRY_COUNT 2

extern ULONG MmTransitionSharedPages;
ULONG MmTransitionSharedPagesPeak;

extern LOGICAL MiTrimRemovalPagesOnly;

ULONG
MiDoReplacement(
    IN PMMSUPPORT WsInfo,
    IN BOOLEAN MustReplace
    );

#ifdef _MI_USE_CLAIMS_
VOID
MiReplaceWorkingSetEntryUsingClaim(
    IN PMMSUPPORT WsInfo,
    IN BOOLEAN MustReplace
    );
#else
VOID
MiReplaceWorkingSetEntryUsingFaultInfo(
    IN PMMSUPPORT WsInfo,
    IN BOOLEAN MustReplace
    );
#endif

VOID
MiCheckWsleHash (
    IN PMMWSL WorkingSetList
    );

VOID
MiEliminateWorkingSetEntry (
    IN ULONG WorkingSetIndex,
    IN PMMPTE PointerPte,
    IN PMMPFN Pfn,
    IN PMMWSLE Wsle
    );

ULONG
MiAddWorkingSetPage (
    IN PMMSUPPORT WsInfo
    );

VOID
MiRemoveWorkingSetPages (
    IN PMMWSL WorkingSetList,
    IN PMMSUPPORT WsInfo
    );

VOID
MiCheckNullIndex (
    IN PMMWSL WorkingSetList
    );

VOID
MiDumpWsleInCacheBlock (
    IN PMMPTE CachePte
    );

ULONG
MiDumpPteInCacheBlock (
    IN PMMPTE PointerPte
    );

#ifdef ALLOC_PRAGMA
#pragma alloc_text(PAGELK, MmAdjustWorkingSetSize)
#endif // ALLOC_PRAGMA


WSLE_NUMBER
MiLocateAndReserveWsle (
    IN PMMSUPPORT WsInfo
    )

/*++

Routine Description:

    This function examines the Working Set List for the current
    process and locates an entry to contain a new page.  If the
    working set is not currently at its quota, the new page is
    added without removing a page, if the working set is at its
    quota a page is removed from the working set and the new
    page added in its place.

Arguments:

    None.

Return Value:

    Returns the working set index which is now reserved for the
    next page to be added.

Environment:

    Kernel mode, APCs disabled, working set lock.  PFN lock NOT held.

--*/

{
    WSLE_NUMBER WorkingSetIndex;
    PMMWSL WorkingSetList;
    PMMWSLE Wsle;
    ULONG QuotaIncrement;
    BOOLEAN MustReplace;

    MustReplace = FALSE;
    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

    //
    // Update page fault counts.
    //

    WsInfo->PageFaultCount += 1;
    MmInfoCounters.PageFaultCount += 1;

    //
    // Determine if a page should be removed from the working set to make
    // room for the new page.  If so, remove it.  In addition, determine
    // the size of the QuotaIncrement if the size needs to be boosted.
    //

retry_replacement:

    QuotaIncrement = MiDoReplacement(WsInfo, MustReplace);

    ASSERT (WsInfo->WorkingSetSize <= WorkingSetList->Quota);
    WsInfo->WorkingSetSize += 1;

    if (WsInfo->WorkingSetSize > WorkingSetList->Quota) {

        //
        // Increment the quota and check boundary conditions.
        //

        WorkingSetList->Quota += QuotaIncrement;

        WsInfo->LastTrimFaultCount = WsInfo->PageFaultCount;

        if (WorkingSetList->Quota > WorkingSetList->LastInitializedWsle) {

            //
            // Add more pages to the working set list structure.
            //

            if (MiAddWorkingSetPage (WsInfo) == TRUE) {
                ASSERT (WsInfo->WorkingSetSize <= WorkingSetList->Quota);
            }
            else {

                //
                // No page was added to the working set list structure.
                // We must replace a page within this working set.
                //

                WsInfo->WorkingSetSize -= 1;
                MustReplace = TRUE;
                goto retry_replacement;
            }
        }
    }

    //
    // Get the working set entry from the free list.
    //

    ASSERT (WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle);

    ASSERT (WorkingSetList->FirstFree >= WorkingSetList->FirstDynamic);

    WorkingSetIndex = WorkingSetList->FirstFree;
    WorkingSetList->FirstFree = (WSLE_NUMBER)(Wsle[WorkingSetIndex].u1.Long >> MM_FREE_WSLE_SHIFT);

    ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
            (WorkingSetList->FirstFree == WSLE_NULL_INDEX));

    if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) {
        MmPagesAboveWsMinimum += 1;
    }

    if (WsInfo->WorkingSetSize > WsInfo->PeakWorkingSetSize) {
        WsInfo->PeakWorkingSetSize = WsInfo->WorkingSetSize;
    }

    if (WsInfo == &MmSystemCacheWs) {
        if (WsInfo->WorkingSetSize + MmTransitionSharedPages > MmTransitionSharedPagesPeak) {
            MmTransitionSharedPagesPeak = WsInfo->WorkingSetSize + MmTransitionSharedPages;
        }
    }

    if (WorkingSetIndex > WorkingSetList->LastEntry) {
        WorkingSetList->LastEntry = WorkingSetIndex;
    }

    //
    // The returned entry is guaranteed to be available at this point.
    //

    ASSERT (Wsle[WorkingSetIndex].u1.e1.Valid == 0);

    return WorkingSetIndex;
}


ULONG
MiDoReplacement(
    IN PMMSUPPORT WsInfo,
    IN BOOLEAN MustReplace
    )

/*++

Routine Description:

    This function determines whether the working set should be
    grown or if a page should be replaced.  Replacement is
    done here if deemed necessary.

Arguments:

    WsInfo - Supplies the working set information structure to replace within.

    MustReplace - Supplies TRUE if replacement must succeed.

Return Value:

    Quota increment if growth is necessary.

Environment:

    Kernel mode, APCs disabled, working set lock.  PFN lock NOT held.

--*/

{
    PMMWSL WorkingSetList;
    ULONG CurrentSize;
    LARGE_INTEGER CurrentTime;
#if defined(_ALPHA_) && !defined(_AXP64_)
    KIRQL OldIrql;
#endif
    ULONG QuotaIncrement;
    PFN_NUMBER AvailablePageThreshold;
#ifdef _MI_USE_CLAIMS_
    PEPROCESS ProcessToTrim;
    ULONG Dummy1;
    ULONG Dummy2;
    ULONG Trim;
    ULONG TrimAge;
#endif

    WorkingSetList = WsInfo->VmWorkingSetList;

    if (WsInfo == &MmSystemCacheWs) {
        MM_SYSTEM_WS_LOCK_ASSERT();
        AvailablePageThreshold = MM_SYSTEM_CACHE_THRESHOLD;
    }

    //
    // Determine the number of pages that need to be available to
    // grow the working set and how much the quota should be
    // boosted if the working set grows over it.
    //
    // If below the Minimum use the defaults.
    //

recheck:

    AvailablePageThreshold = 0;
    QuotaIncrement = 1;

    if (WsInfo->WorkingSetSize >= WsInfo->MinimumWorkingSetSize) {

        if (WsInfo->AllowWorkingSetAdjustment == MM_FORCE_TRIM) {

            //
            // The working set manager cannot attach to this process
            // to trim it.  Force a trim now and update the working
            // set manager's fields properly to indicate a trim occurred.
            //

#ifdef _MI_USE_CLAIMS_

            Trim = WsInfo->Claim >>
                            ((WsInfo->MemoryPriority == MEMORY_PRIORITY_FOREGROUND)
                                ? MI_FOREGROUND_CLAIM_AVAILABLE_SHIFT
                                : MI_BACKGROUND_CLAIM_AVAILABLE_SHIFT);

            if (WsInfo == &MmSystemCacheWs) {
                ProcessToTrim = NULL;
            }
            else if (WsInfo->u.Flags.SessionSpace == 0) {
                ProcessToTrim = CONTAINING_RECORD(WsInfo, EPROCESS, Vm);
            }
            else {
                ProcessToTrim = NULL;       // Hydra session.
            }

            if (MmAvailablePages < 100) {
                if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) {
                    Trim = (WsInfo->WorkingSetSize - WsInfo->MinimumWorkingSetSize) >> 2;
                }
                TrimAge = MI_PASS4_TRIM_AGE;
            }
            else {
                TrimAge = MI_PASS0_TRIM_AGE;
            }

            MiTrimWorkingSet (Trim, WsInfo, TrimAge);

            MiAgeAndEstimateAvailableInWorkingSet (WsInfo,
                                                   TRUE,
                                                   &Dummy1,
                                                   &Dummy2
                                                   );
#else
            MiTrimWorkingSet(20, WsInfo, FALSE);
#endif

            KeQuerySystemTime (&CurrentTime);
            WsInfo->LastTrimTime = CurrentTime;
            WsInfo->LastTrimFaultCount = WsInfo->PageFaultCount;
#if defined(_ALPHA_) && !defined(_AXP64_)
            LOCK_EXPANSION_IF_ALPHA (OldIrql);
#endif
            WsInfo->AllowWorkingSetAdjustment = TRUE;
#if defined(_ALPHA_) && !defined(_AXP64_)
            UNLOCK_EXPANSION_IF_ALPHA (OldIrql);
#endif

            //
            // Set the quota to the current size.
            //

            WorkingSetList->Quota = WsInfo->WorkingSetSize;
            if (WorkingSetList->Quota < WsInfo->MinimumWorkingSetSize) {
                WorkingSetList->Quota = WsInfo->MinimumWorkingSetSize;
            }

            goto recheck;
        }

        CurrentSize = WsInfo->WorkingSetSize;
        ASSERT (CurrentSize <= (WorkingSetList->LastInitializedWsle + 1));

        if ((WsInfo->u.Flags.WorkingSetHard) && (CurrentSize >= WsInfo->MaximumWorkingSetSize)) {

            //
            // This is an enforced working set maximum triggering a replace.
            //

#ifdef _MI_USE_CLAIMS_
            MiReplaceWorkingSetEntryUsingClaim (WsInfo, MustReplace);
#else
            MiReplaceWorkingSetEntryUsingFaultInfo (WsInfo, MustReplace);
#endif
            return 0;
        }

#ifdef _MI_USE_CLAIMS_

        //
        // If using claims, don't grow if
        //      - we're over the max
        //      - there aren't any pages to take
        //      - or if we are growing too much in this
        //        time interval and there isn't much
        //        memory available
        //

        if (CurrentSize > MM_MAXIMUM_WORKING_SET || MmAvailablePages == 0 || MustReplace == TRUE) {

            //
            // Can't grow this one.
            //

            AvailablePageThreshold = 0xffffffff;
            MiReplacing = TRUE;
        }
#if defined (_X86PAE_)
        else if ((WsInfo->u.Flags.SessionSpace == 1) && (CurrentSize > MI_SESSION_MAXIMUM_WORKING_SET)) {

            //
            // Can't grow this one.
            //

            AvailablePageThreshold = 0xffffffff;
            MiReplacing = TRUE;
        }
#endif
        else if (MmAvailablePages < 10000 && MI_WS_GROWING_TOO_FAST(WsInfo)) {

            //
            // Can't grow this one either.
            //

            AvailablePageThreshold = 0xffffffff;
            MiReplacing = TRUE;
        }
#else
        //
        // Not using claims, base the growth on how much faulting
        // the process is doing.
        //

        if (MustReplace == TRUE) {
            AvailablePageThreshold = 0xffffffff;
        }
        if (CurrentSize < WorkingSetList->Quota) {

            //
            // Working set is below quota, allow it to grow with few pages
            // available.
            //

            AvailablePageThreshold = 10;
            QuotaIncrement = 1;
        } else if (CurrentSize < WsInfo->MaximumWorkingSetSize) {

            //
            // Working set is between min and max.  Allow it to grow if enough
            // faults have been taken since last adjustment.
            //

            if ((WsInfo->PageFaultCount - WsInfo->LastTrimFaultCount) <
                    MmFaultsTakenToGoAboveMinWs) {
                AvailablePageThreshold = MmMoreThanEnoughFreePages + 200;
                if (WsInfo->MemoryPriority == MEMORY_PRIORITY_FOREGROUND) {
                    AvailablePageThreshold -= 250;
                }
            } else {
                AvailablePageThreshold = MmWsAdjustThreshold;
            }
            QuotaIncrement = (ULONG)MmWorkingSetSizeIncrement;
        } else {

            //
            // Working set is above max.
            //

            if ((WsInfo->PageFaultCount - WsInfo->LastTrimFaultCount) <
                    (CurrentSize >> 3))
            {
                AvailablePageThreshold = MmMoreThanEnoughFreePages + 200;
                if (WsInfo->MemoryPriority == MEMORY_PRIORITY_FOREGROUND) {
                    AvailablePageThreshold -= 250;
                }
            } else {
                AvailablePageThreshold += MmWsExpandThreshold;
            }
            QuotaIncrement = (ULONG)MmWorkingSetSizeExpansion;

            if (CurrentSize > MM_MAXIMUM_WORKING_SET) {
                AvailablePageThreshold = 0xffffffff;
                QuotaIncrement = 1;
            }
#if defined (_X86PAE_)
            else if ((WsInfo->u.Flags.SessionSpace == 1) && (CurrentSize > MI_SESSION_MAXIMUM_WORKING_SET)) {
                AvailablePageThreshold = 0xffffffff;
                QuotaIncrement = 1;
            }
#endif
        }
#endif
    }

    //
    // If there isn't enough memory to allow growth, find a good page
    // to remove and remove it.
    //

    if (WsInfo->AddressSpaceBeingDeleted == 0 && AvailablePageThreshold != 0) {
        if ((MmAvailablePages <= AvailablePageThreshold) ||
             (WsInfo->WorkingSetExpansionLinks.Flink == MM_NO_WS_EXPANSION)) {

#ifdef _MI_USE_CLAIMS_
            MiReplaceWorkingSetEntryUsingClaim (WsInfo, MustReplace);
#else
            MiReplaceWorkingSetEntryUsingFaultInfo (WsInfo, MustReplace);
#endif
        }
        else {
            WsInfo->GrowthSinceLastEstimate += 1;
        }
    }
    else {
        WsInfo->GrowthSinceLastEstimate += 1;
    }

    return QuotaIncrement;
}


LOGICAL
MmEnforceWorkingSetLimit(
    IN PMMSUPPORT WsInfo,
    IN LOGICAL Enable
    )

/*++

Routine Description:

    This function enables hard enforcement of the working set maximum for
    the specified WsInfo.

Arguments:

    WsInfo - Supplies the working set info pointer.

    Enable - Supplies TRUE if enabling hard enforcement, FALSE if not.

Return Value:

    The previous state of the working set enforcement.

Environment:

    Kernel mode, APCs disabled.  The working set lock must NOT be held.
    The caller guarantees that the target WsInfo cannot go away.

--*/

{
    KIRQL OldIrql;

    LOGICAL PreviousWorkingSetEnforcement;

    LOCK_EXPANSION (OldIrql);

    PreviousWorkingSetEnforcement = WsInfo->u.Flags.WorkingSetHard;

    WsInfo->u.Flags.WorkingSetHard = Enable;

    UNLOCK_EXPANSION (OldIrql);

#if 0

    PEPROCESS CurrentProcess;

    //
    // Get the working set lock and disable APCs.
    // The working set could be trimmed at this point if it is excessive.
    //
    // The working set lock cannot be acquired at this point without updating
    // ps in order to avoid deadlock.
    //

    if (WsInfo == &MmSystemCacheWs) {
        LOCK_SYSTEM_WS (OldIrql2);
        UNLOCK_SYSTEM_WS (OldIrql2);
    }
    else if (WsInfo->u.Flags.SessionSpace == 0) {
        CurrentProcess = PsGetCurrentProcess ();
        LOCK_WS (CurrentProcess);

        UNLOCK_WS (CurrentProcess);
    }
#endif

    return PreviousWorkingSetEnforcement;
}

#ifdef _MI_USE_CLAIMS_

VOID
MiReplaceWorkingSetEntryUsingClaim(
    IN PMMSUPPORT WsInfo,
    IN BOOLEAN MustReplace
    )

/*++

Routine Description:

    This function tries to find a good working set entry to replace.

Arguments:

    WsInfo - Supplies the working set info pointer.

    MustReplace - Supplies TRUE if replacement must succeed.

Return Value:

    None

Environment:

    Kernel mode, APCs disabled, working set lock.  PFN lock NOT held.

--*/

{
    ULONG WorkingSetIndex;
    ULONG FirstDynamic;
    ULONG LastEntry;
    PMMWSL WorkingSetList;
    PMMWSLE Wsle;
    ULONG NumberOfCandidates;
    PMMPTE PointerPte;
    ULONG TheNextSlot;
    ULONG OldestWorkingSetIndex;
    LONG OldestAge;

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

    //
    // Toss a page out of the working set.
    //

    LastEntry = WorkingSetList->LastEntry;
    FirstDynamic = WorkingSetList->FirstDynamic;
    WorkingSetIndex = WorkingSetList->NextSlot;
    if (WorkingSetIndex > LastEntry || WorkingSetIndex < FirstDynamic) {
        WorkingSetIndex = FirstDynamic;
    }
    TheNextSlot = WorkingSetIndex;
    NumberOfCandidates = 0;

    OldestWorkingSetIndex = WSLE_NULL_INDEX;

    while (TRUE) {

        //
        // Keep track of the oldest page along the way in case we
        // don't find one that's >= MI_IMMEDIATE_REPLACEMENT_AGE
        // before we've looked at MM_WORKING_SET_LIST_SEARCH
        // entries.
        //

        while (Wsle[WorkingSetIndex].u1.e1.Valid == 0) {
            WorkingSetIndex += 1;
            if (WorkingSetIndex > LastEntry) {
                WorkingSetIndex = FirstDynamic;
            }
            if (WorkingSetIndex == TheNextSlot && MustReplace == FALSE) {
    
                //
                // Entire working set list has been searched, increase
                // the working set size.
                //
    
                WsInfo->GrowthSinceLastEstimate += 1;
                return;
            }
        }

        if (OldestWorkingSetIndex == WSLE_NULL_INDEX) {

            //
            // First time through, so initialize the OldestWorkingSetIndex
            // to the first valid WSLE.  As we go along, this will be repointed
            // at the oldest candidate we come across.
            //

            OldestWorkingSetIndex = WorkingSetIndex;
            OldestAge = -1;
        }

        PointerPte = MiGetPteAddress(Wsle[WorkingSetIndex].u1.VirtualAddress);

        if (MustReplace == TRUE ||
            ((MI_GET_ACCESSED_IN_PTE(PointerPte) == 0) &&
            (OldestAge < (LONG) MI_GET_WSLE_AGE(PointerPte, &Wsle[WorkingSetIndex])))) {

            //
            // This one is not used and it's older.
            //

            OldestAge = MI_GET_WSLE_AGE(PointerPte, &Wsle[WorkingSetIndex]);
            OldestWorkingSetIndex = WorkingSetIndex;
        }

        //
        // If it's old enough or we've searched too much then use this entry.
        //

        if (MustReplace == TRUE ||
            OldestAge >= MI_IMMEDIATE_REPLACEMENT_AGE ||
            NumberOfCandidates > MM_WORKING_SET_LIST_SEARCH) {

            PERFINFO_PAGE_INFO_REPLACEMENT_DECL();

            if (OldestWorkingSetIndex != WorkingSetIndex) {
                WorkingSetIndex = OldestWorkingSetIndex;
                PointerPte = MiGetPteAddress(Wsle[WorkingSetIndex].u1.VirtualAddress);
            }

            PERFINFO_GET_PAGE_INFO_REPLACEMENT(PointerPte);

            if (MiFreeWsle(WorkingSetIndex, WsInfo, PointerPte)) {

                PERFINFO_LOG_WS_REPLACEMENT(WsInfo);

                //
                // This entry was removed.
                //

                WorkingSetList->NextSlot = WorkingSetIndex + 1;
                break;
            }

            //
            // We failed to remove a page, try the next one.
            //
            // Clear the OldestWorkingSetIndex so that
            // it gets set to the next valid entry above like the
            // first time around.
            //

            WorkingSetIndex = OldestWorkingSetIndex + 1;

            OldestWorkingSetIndex = WSLE_NULL_INDEX;
        }
        else {
            WorkingSetIndex += 1;
        }

        if (WorkingSetIndex > LastEntry) {
            WorkingSetIndex = FirstDynamic;
        }

        NumberOfCandidates += 1;

        if (WorkingSetIndex == TheNextSlot && MustReplace == FALSE) {

            //
            // Entire working set list has been searched, increase
            // the working set size.
            //

            WsInfo->GrowthSinceLastEstimate += 1;
            break;
        }
    }
}
#else

VOID
MiReplaceWorkingSetEntryUsingFaultInfo(
    IN PMMSUPPORT WsInfo,
    IN BOOLEAN MustReplace
    )

/*++

Routine Description:

    This function tries to find a reasonable working set entry to replace.

Arguments:

    WsInfo - Supplies the working set info pointer.

    MustReplace - Supplies TRUE if replacement must succeed.

Return Value:

    None

Environment:

    Kernel mode, APCs disabled, working set lock.  PFN lock NOT held.

--*/

{
    ULONG WorkingSetIndex;
    ULONG FirstDynamic;
    ULONG LastEntry;
    PMMWSL WorkingSetList;
    PMMWSLE Wsle;
    ULONG NumberOfCandidates;
    PMMPTE PointerPte;
    ULONG TheNextSlot;

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

    //
    // Toss a page out of the working set.
    //

    LastEntry = WorkingSetList->LastEntry;
    FirstDynamic = WorkingSetList->FirstDynamic;
    WorkingSetIndex = WorkingSetList->NextSlot;

    if ((WorkingSetIndex > LastEntry) || (WorkingSetIndex < FirstDynamic)) {
        WorkingSetIndex = FirstDynamic;
    }

    TheNextSlot = WorkingSetIndex;
    NumberOfCandidates = 0;

    do {

        //
        // Find a valid entry within the set.
        //

        if (Wsle[WorkingSetIndex].u1.e1.Valid != 0) {

            PointerPte = MiGetPteAddress (
                              Wsle[WorkingSetIndex].u1.VirtualAddress);

            if ((MI_GET_ACCESSED_IN_PTE(PointerPte) == 0) ||
                (NumberOfCandidates > MM_WORKING_SET_LIST_SEARCH)) {

                PERFINFO_PAGE_INFO_REPLACEMENT_DECL();
                PERFINFO_GET_PAGE_INFO_REPLACEMENT(PointerPte);

                //
                // Don't pick the same entry we replaced the last time
                // unless we're under extreme pressure.
                //

                if ((WorkingSetIndex != TheNextSlot || MustReplace == TRUE) &&
                    MiFreeWsle (WorkingSetIndex, WsInfo, PointerPte)) {

                    PERFINFO_LOG_WS_REPLACEMENT(WsInfo);

                    //
                    // This entry was removed.
                    //

                    WorkingSetList->NextSlot = WorkingSetIndex + 1;
                    break;
                }
            }
            MI_SET_ACCESSED_IN_PTE (PointerPte, 0);
            NumberOfCandidates += 1;
        }

        WorkingSetIndex += 1;
        if (WorkingSetIndex > LastEntry) {
            WorkingSetIndex = FirstDynamic;
        }

    } while (WorkingSetIndex != TheNextSlot || MustReplace == TRUE);

    //
    // Entire working set list has been searched.  If an entry wasn't
    // removed, our caller can increase the working set size.
    //
}
#endif

ULONG
MiRemovePageFromWorkingSet (
    IN PMMPTE PointerPte,
    IN PMMPFN Pfn1,
    IN PMMSUPPORT WsInfo
    )

/*++

Routine Description:

    This function removes the page mapped by the specified PTE from
    the process's working set list.

Arguments:

    PointerPte - Supplies a pointer to the PTE mapping the page to
                 be removed from the working set list.

    Pfn1 - Supplies a pointer to the PFN database element referred to
           by the PointerPte.

Return Value:

    Returns TRUE if the specified page was locked in the working set,
    FALSE otherwise.

Environment:

    Kernel mode, APCs disabled, working set mutex held.

--*/

{
    WSLE_NUMBER WorkingSetIndex;
    PVOID VirtualAddress;
    WSLE_NUMBER Entry;
    PVOID SwapVa;
    MMWSLENTRY Locked;
    PMMWSL WorkingSetList;
    PMMWSLE Wsle;
    KIRQL OldIrql;

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

    VirtualAddress = MiGetVirtualAddressMappedByPte (PointerPte);
    WorkingSetIndex = MiLocateWsle (VirtualAddress,
                                    WorkingSetList,
                                    Pfn1->u1.WsIndex);

    ASSERT (WorkingSetIndex != WSLE_NULL_INDEX);
    LOCK_PFN (OldIrql);
    MiEliminateWorkingSetEntry (WorkingSetIndex,
                                PointerPte,
                                Pfn1,
                                Wsle);

    UNLOCK_PFN (OldIrql);

    //
    // Check to see if this entry is locked in the working set
    // or locked in memory.
    //

    Locked = Wsle[WorkingSetIndex].u1.e1;
    MiRemoveWsle (WorkingSetIndex, WorkingSetList);

    //
    // Add this entry to the list of free working set entries
    // and adjust the working set count.
    //

    MiReleaseWsle ((WSLE_NUMBER)WorkingSetIndex, WsInfo);

    if ((Locked.LockedInWs == 1) || (Locked.LockedInMemory == 1)) {

        //
        // This entry is locked.
        //

        WorkingSetList->FirstDynamic -= 1;

        if (WorkingSetIndex != WorkingSetList->FirstDynamic) {

            SwapVa = Wsle[WorkingSetList->FirstDynamic].u1.VirtualAddress;
            SwapVa = PAGE_ALIGN (SwapVa);

            PointerPte = MiGetPteAddress (SwapVa);
            Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
#if 0
            Entry = MiLocateWsleAndParent (SwapVa,
                                           &Parent,
                                           WorkingSetList,
                                           Pfn1->u1.WsIndex);

            //
            // Swap the removed entry with the last locked entry
            // which is located at first dynamic.
            //

            MiSwapWslEntries (Entry, Parent, WorkingSetIndex, WorkingSetList);
#endif //0

            Entry = MiLocateWsle (SwapVa, WorkingSetList, Pfn1->u1.WsIndex);

            MiSwapWslEntries (Entry, WorkingSetIndex, WsInfo);

        }
        return TRUE;
    } else {
        ASSERT (WorkingSetIndex >= WorkingSetList->FirstDynamic);
    }
    return FALSE;
}


VOID
MiReleaseWsle (
    IN WSLE_NUMBER WorkingSetIndex,
    IN PMMSUPPORT WsInfo
    )

/*++

Routine Description:

    This function releases a previously reserved working set entry to
    be reused.  A release occurs when a page fault is retried due to
    changes in PTEs and working sets during an I/O operation.

Arguments:

    WorkingSetIndex - Supplies the index of the working set entry to
                      release.

Return Value:

    None.

Environment:

    Kernel mode, APCs disabled, working set lock held and PFN lock held.

--*/

{
    PMMWSL WorkingSetList;
    PMMWSLE Wsle;

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;
#if DBG
    if (WsInfo == &MmSystemCacheWs) {
        MM_SYSTEM_WS_LOCK_ASSERT();
    }
#endif //DBG

    ASSERT (WorkingSetIndex <= WorkingSetList->LastInitializedWsle);

    //
    // Put the entry on the free list and decrement the current
    // size.
    //

    ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
            (WorkingSetList->FirstFree == WSLE_NULL_INDEX));
    Wsle[WorkingSetIndex].u1.Long = WorkingSetList->FirstFree << MM_FREE_WSLE_SHIFT;
    WorkingSetList->FirstFree = WorkingSetIndex;
    ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
            (WorkingSetList->FirstFree == WSLE_NULL_INDEX));
    if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) {
        MmPagesAboveWsMinimum -= 1;
    }
    WsInfo->WorkingSetSize -= 1;
    return;

}

VOID
MiUpdateWsle (
    IN OUT PWSLE_NUMBER DesiredIndex,
    IN PVOID VirtualAddress,
    PMMWSL WorkingSetList,
    IN PMMPFN Pfn
    )

/*++

Routine Description:

    This routine updates a reserved working set entry to place it into
    the valid state.

Arguments:

    DesiredIndex - Supplies the index of the working set entry to update.

    VirtualAddress - Supplies the virtual address which the working set
                     entry maps.

    WsInfo - Supplies a pointer to the working set info block for the
             process (or system cache).

    Pfn - Supplies a pointer to the PFN element for the page.

Return Value:

    None.

Environment:

    Kernel mode, APCs disabled, working set lock held and PFN lock held.

--*/

{
    PMMWSLE Wsle;
    WSLE_NUMBER Index;
    WSLE_NUMBER WorkingSetIndex;
#if PFN_CONSISTENCY
    KIRQL OldIrql;
#endif

    WorkingSetIndex = *DesiredIndex;
    Wsle = WorkingSetList->Wsle;

    if (WorkingSetList == MmSystemCacheWorkingSetList) {

        //
        // This assert doesn't hold for NT64 as we can be adding page
        // directories and page tables for the system cache WSLE hash tables.
        //

        ASSERT32 ((VirtualAddress < (PVOID)PTE_BASE) ||
                  (VirtualAddress >= (PVOID)MM_SYSTEM_SPACE_START));
    } else {
        ASSERT ((VirtualAddress < (PVOID)MM_SYSTEM_SPACE_START) ||
               (MI_IS_SESSION_ADDRESS (VirtualAddress)));
    }

    ASSERT (WorkingSetIndex >= WorkingSetList->FirstDynamic);

    if (WorkingSetList == MmSystemCacheWorkingSetList) {

        MM_SYSTEM_WS_LOCK_ASSERT();

        //
        // count system space inserts and removals.
        //

#if defined(_X86_)
        if (MI_IS_SYSTEM_CACHE_ADDRESS(VirtualAddress)) {
            MmSystemCachePage += 1;
        } else
#endif
        if (VirtualAddress < MmSystemCacheStart) {
            MmSystemCodePage += 1;
        } else if (VirtualAddress < MM_PAGED_POOL_START) {
            MmSystemCachePage += 1;
        } else if (VirtualAddress < MmNonPagedSystemStart) {
            MmPagedPoolPage += 1;
        } else {
            MmSystemDriverPage += 1;
        }
    }

    //
    // Make the wsle valid, referring to the corresponding virtual
    // page number.
    //

    //
    // The value 0 is invalid.  This is due to the fact that the working
    // set lock is a process wide lock and two threads in different
    // processes could be adding the same physical page to their working
    // sets.  Each one could see the WsIndex field in the PFN as 0, and
    // set the direct bit.  To solve this, the WsIndex field is set to
    // the current thread pointer.
    //

    ASSERT (Pfn->u1.WsIndex != 0);

#if DBG
    if (Pfn->u1.WsIndex <= WorkingSetList->LastInitializedWsle) {
        ASSERT ((PAGE_ALIGN(VirtualAddress) !=
                PAGE_ALIGN(Wsle[Pfn->u1.WsIndex].u1.VirtualAddress)) ||
                (Wsle[Pfn->u1.WsIndex].u1.e1.Valid == 0));
    }
#endif //DBG

    Wsle[WorkingSetIndex].u1.VirtualAddress = VirtualAddress;
    Wsle[WorkingSetIndex].u1.Long &= ~(PAGE_SIZE - 1);
    Wsle[WorkingSetIndex].u1.e1.Valid = 1;

    if ((ULONG_PTR)Pfn->u1.Event == (ULONG_PTR)PsGetCurrentThread()) {

        //
        // Directly index into the WSL for this entry via the PFN database
        // element.
        //

        CONSISTENCY_LOCK_PFN (OldIrql);

#if defined(_WIN64)

        //
        // The entire working set index union must be zeroed on Win64 systems.
        //

        MI_ZERO_WSINDEX (Pfn);

#endif

        Pfn->u1.WsIndex = WorkingSetIndex;

        CONSISTENCY_UNLOCK_PFN (OldIrql);

        Wsle[WorkingSetIndex].u1.e1.Direct = 1;

        return;

    } else if (WorkingSetList->HashTable == NULL) {

        //
        // Try to insert at WsIndex.
        //

        Index = Pfn->u1.WsIndex;

        if ((Index < WorkingSetList->LastInitializedWsle) &&
            (Index > WorkingSetList->FirstDynamic) &&
            (Index != WorkingSetIndex)) {

            if (Wsle[Index].u1.e1.Valid) {

                if (Wsle[Index].u1.e1.Direct) {

                    //
                    // Only move direct indexed entries.
                    //

                    PMMSUPPORT WsInfo;

                    if (Wsle == MmWsle) {
                        WsInfo = &PsGetCurrentProcess()->Vm;
                    }
                    else if (Wsle == MmSystemCacheWsle) {
                        WsInfo = &MmSystemCacheWs;
                    }
                    else {
                        WsInfo = &MmSessionSpace->Vm;
                    }

                    MiSwapWslEntries (Index, WorkingSetIndex, WsInfo);
                    WorkingSetIndex = Index;
                }
            } else {

                //
                // On free list, try to remove quickly without walking
                // all the free pages.
                //

                ULONG FreeIndex;
                MMWSLE Temp;

                FreeIndex = 0;

                ASSERT (WorkingSetList->FirstFree >= WorkingSetList->FirstDynamic);
                ASSERT (WorkingSetIndex >= WorkingSetList->FirstDynamic);

                if (WorkingSetList->FirstFree == Index) {
                    WorkingSetList->FirstFree = WorkingSetIndex;
                    Temp = Wsle[WorkingSetIndex];
                    Wsle[WorkingSetIndex] = Wsle[Index];
                    Wsle[Index] = Temp;
                    WorkingSetIndex = Index;
                    ASSERT (((Wsle[WorkingSetList->FirstFree].u1.Long >> MM_FREE_WSLE_SHIFT)
                                     <= WorkingSetList->LastInitializedWsle) ||
                            ((Wsle[WorkingSetList->FirstFree].u1.Long >> MM_FREE_WSLE_SHIFT)
                                    == WSLE_NULL_INDEX));
                } else if (Wsle[Index - 1].u1.e1.Valid == 0) {
                    if ((Wsle[Index - 1].u1.Long >> MM_FREE_WSLE_SHIFT) == Index) {
                        FreeIndex = Index - 1;
                    }
                } else if (Wsle[Index + 1].u1.e1.Valid == 0) {
                    if ((Wsle[Index + 1].u1.Long >> MM_FREE_WSLE_SHIFT) == Index) {
                        FreeIndex = Index + 1;
                    }
                }
                if (FreeIndex != 0) {

                    //
                    // Link the Wsle into the free list.
                    //

                    Temp = Wsle[WorkingSetIndex];
                    Wsle[FreeIndex].u1.Long = WorkingSetIndex << MM_FREE_WSLE_SHIFT;
                    Wsle[WorkingSetIndex] = Wsle[Index];
                    Wsle[Index] = Temp;
                    WorkingSetIndex = Index;

                    ASSERT (((Wsle[FreeIndex].u1.Long >> MM_FREE_WSLE_SHIFT)
                                     <= WorkingSetList->LastInitializedWsle) ||
                            ((Wsle[FreeIndex].u1.Long >> MM_FREE_WSLE_SHIFT)
                                    == WSLE_NULL_INDEX));
                }

            }
            *DesiredIndex = WorkingSetIndex;

            if (WorkingSetIndex > WorkingSetList->LastEntry) {
                WorkingSetList->LastEntry = WorkingSetIndex;
            }
        }
    }

    //
    // Insert the valid WSLE into the working set list.
    //

    MiInsertWsle (WorkingSetIndex, WorkingSetList);
    return;
}


ULONG
MiFreeWsle (
    IN WSLE_NUMBER WorkingSetIndex,
    IN PMMSUPPORT WsInfo,
    IN PMMPTE PointerPte
    )

/*++

Routine Description:

    This routine frees the specified WSLE and decrements the share
    count for the corresponding page, putting the PTE into a transition
    state if the share count goes to 0.

Arguments:

    WorkingSetIndex - Supplies the index of the working set entry to free.

    WsInfo - Supplies a pointer to the working set structure (process or
             system cache).

    PointerPte - Supplies a pointer to the PTE for the working set entry.

Return Value:

    Returns TRUE if the WSLE was removed, FALSE if it was not removed.
        Pages with valid PTEs are not removed (i.e. page table pages
        that contain valid or transition PTEs).

Environment:

    Kernel mode, APCs disabled, working set lock.  PFN lock NOT held.

--*/

{
    PMMPFN Pfn1;
    PMMWSL WorkingSetList;
    PMMWSLE Wsle;
    KIRQL OldIrql;

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

#if DBG
    if (WsInfo == &MmSystemCacheWs) {
        MM_SYSTEM_WS_LOCK_ASSERT();
    }
#endif //DBG

    ASSERT (Wsle[WorkingSetIndex].u1.e1.Valid == 1);

    //
    // Check to see if the located entry is eligible for removal.
    //

    ASSERT (PointerPte->u.Hard.Valid == 1);

    ASSERT (WorkingSetIndex >= WorkingSetList->FirstDynamic);

    //
    // Check to see if this is a page table with valid PTEs.
    //
    // Note, don't clear the access bit for page table pages
    // with valid PTEs as this could cause an access trap fault which
    // would not be handled (it is only handled for PTEs not PDEs).
    //

    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    LOCK_PFN (OldIrql);

    //
    // If the PTE is a page table page with non-zero share count or
    // within the system cache with its reference count greater
    // than 0, don't remove it.
    //

    if (WsInfo == &MmSystemCacheWs) {
        if (Pfn1->u3.e2.ReferenceCount > 1) {
            UNLOCK_PFN (OldIrql);
            return FALSE;
        }
    } else {
        if ((Pfn1->u2.ShareCount > 1) &&
            (Pfn1->u3.e1.PrototypePte == 0)) {

#if DBG
            if (WsInfo->u.Flags.SessionSpace == 1) {
                ASSERT (MI_IS_SESSION_ADDRESS (Wsle[WorkingSetIndex].u1.VirtualAddress));
            }
            else {
                ASSERT ((Wsle[WorkingSetIndex].u1.VirtualAddress >= (PVOID)PTE_BASE) &&
                 (Wsle[WorkingSetIndex].u1.VirtualAddress<= (PVOID)PDE_TOP));
            }
#endif

            //
            // Don't remove page table pages from the working set until
            // all transition pages have exited.
            //

            UNLOCK_PFN (OldIrql);
            return FALSE;
        }
    }

    //
    // Found a candidate, remove the page from the working set.
    //

    MiEliminateWorkingSetEntry (WorkingSetIndex,
                                PointerPte,
                                Pfn1,
                                Wsle);

    UNLOCK_PFN (OldIrql);

    //
    // Remove the working set entry from the working set.
    //

    MiRemoveWsle (WorkingSetIndex, WorkingSetList);

    ASSERT (WorkingSetList->FirstFree >= WorkingSetList->FirstDynamic);

    ASSERT (WorkingSetIndex >= WorkingSetList->FirstDynamic);

    //
    // Put the entry on the free list and decrement the current
    // size.
    //

    ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
            (WorkingSetList->FirstFree == WSLE_NULL_INDEX));
    Wsle[WorkingSetIndex].u1.Long = WorkingSetList->FirstFree << MM_FREE_WSLE_SHIFT;
    WorkingSetList->FirstFree = WorkingSetIndex;
    ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
            (WorkingSetList->FirstFree == WSLE_NULL_INDEX));

    if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) {
        MmPagesAboveWsMinimum -= 1;
    }
    WsInfo->WorkingSetSize -= 1;

#if 0
    if ((WsInfo == &MmSystemCacheWs) &&
       (Pfn1->u3.e1.Modified == 1))  {
        MiDumpWsleInCacheBlock (PointerPte);
    }
#endif //0
    return TRUE;
}

VOID
MiInitializeWorkingSetList (
    IN PEPROCESS CurrentProcess
    )

/*++

Routine Description:

    This routine initializes a process's working set to the empty
    state.

Arguments:

    CurrentProcess - Supplies a pointer to the process to initialize.

Return Value:

    None.

Environment:

    Kernel mode, APCs disabled.

--*/

{
    ULONG i;
    PMMWSLE WslEntry;
    ULONG CurrentEntry;
    PMMPTE PointerPte;
    PMMPFN Pfn1;
    WSLE_NUMBER NumberOfEntriesMapped;
    ULONG_PTR CurrentVa;
    PFN_NUMBER WorkingSetPage;
    MMPTE TempPte;
    KIRQL OldIrql;

    WslEntry = MmWsle;

    //
    // Initialize the temporary double mapping portion of hyperspace, if
    // it has not already been done.
    //
    // Initialize the working set list control cells.
    //

    MmWorkingSetList->LastEntry = CurrentProcess->Vm.MinimumWorkingSetSize;
    MmWorkingSetList->Quota = MmWorkingSetList->LastEntry;
    MmWorkingSetList->WaitingForImageMapping = (PKEVENT)NULL;
    MmWorkingSetList->HashTable = NULL;
    MmWorkingSetList->HashTableSize = 0;
    MmWorkingSetList->Wsle = MmWsle;
    MmWorkingSetList->HashTableStart = 
       (PVOID)((PCHAR)PAGE_ALIGN (&MmWsle[MM_MAXIMUM_WORKING_SET]) + PAGE_SIZE);

    MmWorkingSetList->HighestPermittedHashAddress = (PVOID)(HYPER_SPACE_END + 1);

    //
    // Fill in the reserved slots.  Start with the page directory page.
    //

#if !defined (_X86PAE_)

    WslEntry->u1.Long = PDE_BASE;
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress);
    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    CONSISTENCY_LOCK_PFN (OldIrql);

#if !defined (_WIN64)
    Pfn1->u1.Event = (PVOID)CurrentProcess;
#endif

    CONSISTENCY_UNLOCK_PFN (OldIrql);

#else

    //
    // Fill in all the page directory entries.
    //

    for (i = 0; i < PD_PER_SYSTEM; i += 1) {

        WslEntry->u1.VirtualAddress = (PVOID)(PDE_BASE + i * PAGE_SIZE);
        WslEntry->u1.e1.Valid = 1;
        WslEntry->u1.e1.LockedInWs = 1;
        WslEntry->u1.e1.Direct = 1;

        PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress);
        Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);
    
        ASSERT (Pfn1->u1.WsIndex == 0);
    
        CONSISTENCY_LOCK_PFN (OldIrql);
    
        Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmWsle);
    
        CONSISTENCY_UNLOCK_PFN (OldIrql);

        WslEntry += 1;
    }
    WslEntry -= 1;

#endif


#if defined (_WIN64)

    //
    // Fill in the entry for the page directory parent page.
    //

    WslEntry += 1;

    WslEntry->u1.Long = PDE_TBASE;
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress);
    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    ASSERT (Pfn1->u1.WsIndex == 0);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmWsle);

    CONSISTENCY_UNLOCK_PFN (OldIrql);

    //
    // Fill in the entry for the hyper space page directory page.
    //

    WslEntry += 1;

    WslEntry->u1.VirtualAddress = (PVOID)MiGetPdeAddress (HYPER_SPACE);
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress);
    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    ASSERT (Pfn1->u1.WsIndex == 0);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmWsle);

    CONSISTENCY_UNLOCK_PFN (OldIrql);

#if defined (_IA64_)

    //
    // Fill in the entry for the session space page directory parent page.
    //

    WslEntry += 1;

    WslEntry->u1.VirtualAddress = (PVOID)MiGetPpeAddress (MM_SESSION_SPACE_DEFAULT);
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress);
    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    ASSERT (Pfn1->u1.WsIndex == 0);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmWsle);

    CONSISTENCY_UNLOCK_PFN (OldIrql);
    
#endif

#endif

    //
    // Fill in the entry for the page table page which maps hyper space.
    //

    WslEntry += 1;

    WslEntry->u1.VirtualAddress = (PVOID)MiGetPteAddress (HYPER_SPACE);
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress);
    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    ASSERT (Pfn1->u1.WsIndex == 0);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmWsle);

    CONSISTENCY_UNLOCK_PFN (OldIrql);

#if defined (_X86PAE_)

    //
    // Fill in the entry for the second page table page which maps hyper space.
    //

    WslEntry += 1;

    WslEntry->u1.VirtualAddress = (PVOID)MiGetPteAddress (HYPER_SPACE2);
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress);
    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    ASSERT (Pfn1->u1.WsIndex == 0);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmWsle);

    CONSISTENCY_UNLOCK_PFN (OldIrql);

#endif

    //
    // Fill in the entry for the page which contains the working set list.
    //

    WslEntry += 1;

    WslEntry->u1.VirtualAddress = (PVOID)MmWorkingSetList;
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    PointerPte = MiGetPteAddress (WslEntry->u1.VirtualAddress);
    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    ASSERT (Pfn1->u1.WsIndex == 0);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmWsle);

    CONSISTENCY_UNLOCK_PFN (OldIrql);

    CurrentEntry = (WSLE_NUMBER)((WslEntry - MmWsle) + 1);

    //
    // Check to see if more pages are required in the working set list
    // to map the current maximum working set size.
    //

    NumberOfEntriesMapped = (WSLE_NUMBER)(((PMMWSLE)((PCHAR)WORKING_SET_LIST + PAGE_SIZE)) -
                                MmWsle);

    if (CurrentProcess->Vm.MaximumWorkingSetSize >= NumberOfEntriesMapped) {

        PointerPte = MiGetPteAddress (&MmWsle[0]);

        CurrentVa = (ULONG_PTR)MmWorkingSetList + PAGE_SIZE;

        //
        // The working set requires more than a single page.
        //

        LOCK_PFN (OldIrql);

        do {

            MiEnsureAvailablePageOrWait (NULL, NULL);

            PointerPte += 1;
            WorkingSetPage = MiRemoveZeroPage (
                                    MI_PAGE_COLOR_PTE_PROCESS (PointerPte,
                                              &CurrentProcess->NextPageColor));
            PointerPte->u.Long = MM_DEMAND_ZERO_WRITE_PTE;

            MiInitializePfn (WorkingSetPage, PointerPte, 1);

            MI_MAKE_VALID_PTE (TempPte,
                               WorkingSetPage,
                               MM_READWRITE,
                               PointerPte );

            MI_SET_PTE_DIRTY (TempPte);

            MI_SET_PTE_IN_WORKING_SET (&TempPte, CurrentEntry);

            MI_WRITE_VALID_PTE (PointerPte, TempPte);

            WslEntry += 1;

            WslEntry->u1.Long = CurrentVa;
            WslEntry->u1.e1.Valid = 1;
            WslEntry->u1.e1.LockedInWs = 1;
            WslEntry->u1.e1.Direct = 1;

            Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

            ASSERT (Pfn1->u1.WsIndex == 0);
            Pfn1->u1.WsIndex = CurrentEntry;

            // MiInsertWsle(CurrentEntry, MmWorkingSetList);

            CurrentEntry += 1;
            CurrentVa += PAGE_SIZE;

            NumberOfEntriesMapped += PAGE_SIZE / sizeof(MMWSLE);

        } while (CurrentProcess->Vm.MaximumWorkingSetSize >= NumberOfEntriesMapped);

        UNLOCK_PFN (OldIrql);
    }

    CurrentProcess->Vm.WorkingSetSize = CurrentEntry;
    MmWorkingSetList->FirstFree = CurrentEntry;
    MmWorkingSetList->FirstDynamic = CurrentEntry;
    MmWorkingSetList->NextSlot = CurrentEntry;

    //
    // Initialize the following slots as free.
    //

    i = CurrentEntry + 1;
    do {

        //
        // Build the free list, note that the first working
        // set entries (CurrentEntry) are not on the free list.
        // These entries are reserved for the pages which
        // map the working set and the page which contains the PDE.
        //

        WslEntry += 1;
        WslEntry->u1.Long = i << MM_FREE_WSLE_SHIFT;
        i += 1;
    } while (i <= NumberOfEntriesMapped);

    WslEntry->u1.Long = WSLE_NULL_INDEX << MM_FREE_WSLE_SHIFT;  // End of list.

    MmWorkingSetList->LastInitializedWsle =
                                NumberOfEntriesMapped - 1;

    if (CurrentProcess->Vm.MaximumWorkingSetSize > ((1536*1024) >> PAGE_SHIFT)) {

        //
        // The working set list consists of more than a single page.
        //

        MiGrowWsleHash (&CurrentProcess->Vm);
    }

    return;
}


VOID
MiInitializeSessionWsSupport(
    VOID
    )

/*++

Routine Description:

    This routine initializes the session space working set support.

Arguments:

    None.

Return Value:

    None.

Environment:

    Kernel mode, APC_LEVEL or below, no mutexes held.

--*/

{
    //
    // This is the list of all session spaces ordered in a working set list.
    //

    InitializeListHead (&MiSessionWsList);
}


NTSTATUS
MiSessionInitializeWorkingSetList (
    VOID
    )

/*++

Routine Description:

    This function initializes the working set for the session space and adds
    it to the list of session space working sets.

Arguments:

    None.

Return Value:

    NT_SUCCESS if success or STATUS_NO_MEMORY on failure.

Environment:

    Kernel mode, APC_LEVEL or below, no mutexes held.

--*/

{
    ULONG i;
    KIRQL OldIrql;
    PMMPTE PointerPte;
    PMMPTE PointerPde;
    MMPTE  TempPte;
    PMMWSLE WslEntry;
    PMMPFN Pfn1;
    ULONG PageColor;
    PFN_NUMBER ResidentPages;
    ULONG Index;
    PFN_NUMBER PageFrameIndex;
    ULONG CurrentEntry;
    ULONG NumberOfEntriesMapped;
    ULONG_PTR AdditionalBytes;
    ULONG NumberOfEntriesMappedByFirstPage;
    ULONG WorkingSetMaximum;
    PMM_SESSION_SPACE SessionGlobal;
    LOGICAL AllocatedPageTable;

    //
    // Use the global address for pointer references by
    // MmWorkingSetManager before it attaches to the address space.
    //

    SessionGlobal = SESSION_GLOBAL (MmSessionSpace);

    //
    // Set up the working set variables.
    //

    WorkingSetMaximum = MI_SESSION_SPACE_WORKING_SET_MAXIMUM;

    MmSessionSpace->Vm.VmWorkingSetList = (PMMWSL)MI_SESSION_SPACE_WS;
#if defined (_WIN64)
    MmSessionSpace->Wsle = (PMMWSLE)(MmSessionSpace->Vm.VmWorkingSetList + 1);
#else
    MmSessionSpace->Wsle =
            (PMMWSLE)(&MmSessionSpace->Vm.VmWorkingSetList->UsedPageTableEntries[0]);
#endif

    ASSERT (MmSessionSpace->WorkingSetLockOwner == NULL);
    ASSERT (MmSessionSpace->WorkingSetLockOwnerCount == 0);

    //
    // Build the PDE entry for the working set - note that the global bit
    // must be turned off.
    //

    PointerPde = MiGetPdeAddress (MmSessionSpace->Vm.VmWorkingSetList);

    //
    // The page table page for the working set and its first data page
    // are charged against MmResidentAvailablePages and for commitment.
    //

    if (PointerPde->u.Hard.Valid == 1) {

        //
        // The page directory entry for the working set is the same
        // as for another range in the session space.  Share the PDE.
        //

#ifndef _IA64_
        ASSERT (PointerPde->u.Hard.Global == 0);
#endif
        AllocatedPageTable = FALSE;
        ResidentPages = 1;
    }
    else {
        AllocatedPageTable = TRUE;
        ResidentPages = 2;
    }


    PointerPte = MiGetPteAddress (MmSessionSpace->Vm.VmWorkingSetList);

    //
    // The data pages needed to map up to maximum working set size are also
    // charged against MmResidentAvailablePages and for commitment.
    //

    NumberOfEntriesMappedByFirstPage = (WSLE_NUMBER)(
        ((PMMWSLE)((ULONG_PTR)MmSessionSpace->Vm.VmWorkingSetList + PAGE_SIZE)) -
            MmSessionSpace->Wsle);

    if (WorkingSetMaximum > NumberOfEntriesMappedByFirstPage) {
        AdditionalBytes = (WorkingSetMaximum - NumberOfEntriesMappedByFirstPage) * sizeof (MMWSLE);
        ResidentPages += BYTES_TO_PAGES (AdditionalBytes);
    }

    if (MiChargeCommitment (ResidentPages, NULL) == FALSE) {
#if DBG
        DbgPrint("MiSessionInitializeWorkingSetList: No commit for %d pages\n",
            ResidentPages);

#endif
        MM_BUMP_SESSION_FAILURES (MM_SESSION_FAILURE_NO_COMMIT);
        return STATUS_NO_MEMORY;
    }

    MM_TRACK_COMMIT (MM_DBG_COMMIT_SESSION_WS_INIT, ResidentPages);

    //
    // Use the global address for resources since they are linked
    // into the global system wide resource list.
    //

    ExInitializeResource (&SessionGlobal->WsLock);

    LOCK_PFN (OldIrql);

    //
    // Check to make sure the physical pages are available.
    //

    if ((SPFN_NUMBER)ResidentPages > MI_NONPAGABLE_MEMORY_AVAILABLE() - 20) {
#if DBG
        DbgPrint("MiSessionInitializeWorkingSetList: No Resident Pages %d, Need %d\n",
            MmResidentAvailablePages,
            ResidentPages);
#endif
        UNLOCK_PFN (OldIrql);

        MiReturnCommitment (ResidentPages);
        MM_TRACK_COMMIT (MM_DBG_COMMIT_RETURN_SESSION_WSL_FAILURE, ResidentPages);
        ExDeleteResource (&SessionGlobal->WsLock);
        MM_BUMP_SESSION_FAILURES (MM_SESSION_FAILURE_NO_RESIDENT);
        return STATUS_NO_MEMORY;
    }

    MmResidentAvailablePages -= ResidentPages;

    MM_BUMP_COUNTER(50, ResidentPages);

    if (AllocatedPageTable == TRUE) {

        MM_BUMP_SESS_COUNTER (MM_DBG_SESSION_WS_PAGETABLE_ALLOC, 1);

        MiEnsureAvailablePageOrWait (NULL, NULL);

        PageColor = MI_GET_PAGE_COLOR_FROM_VA (NULL);

        PageFrameIndex = MiRemoveZeroPageIfAny (PageColor);
        if (PageFrameIndex == 0) {
            PageFrameIndex = MiRemoveAnyPage (PageColor);
            UNLOCK_PFN (OldIrql);
            MiZeroPhysicalPage (PageFrameIndex, PageColor);
            LOCK_PFN (OldIrql);
        }

        //
        // The global bit is masked off since we need to make sure the TB entry
        // is flushed when we switch to a process in a different session space.
        //

        TempPte.u.Long = ValidKernelPdeLocal.u.Long;
        TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
        MI_WRITE_VALID_PTE (PointerPde, TempPte);

#if !defined (_WIN64)

        //
        // Add this to the session structure so other processes can fault it in.
        //

        Index = MiGetPdeSessionIndex (MmSessionSpace->Vm.VmWorkingSetList);

        MmSessionSpace->PageTables[Index] = TempPte;

#endif

        //
        // This page frame references the session space page table page.
        //

        MiInitializePfnForOtherProcess (PageFrameIndex,
                                        PointerPde,
                                        MmSessionSpace->SessionPageDirectoryIndex);

        MiFillMemoryPte (PointerPte, PAGE_SIZE, ZeroKernelPte.u.Long);

        Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);

        //
        // This page is never paged, ensure that its WsIndex stays clear so the
        // release of the page will be handled correctly.
        //

        ASSERT (Pfn1->u1.WsIndex == 0);

        KeFillEntryTb ((PHARDWARE_PTE) PointerPde, PointerPte, FALSE);
    }

    MiEnsureAvailablePageOrWait (NULL, NULL);

    PageColor = MI_GET_PAGE_COLOR_FROM_VA (NULL);

    PageFrameIndex = MiRemoveZeroPageIfAny (PageColor);
    if (PageFrameIndex == 0) {
        PageFrameIndex = MiRemoveAnyPage (PageColor);
        UNLOCK_PFN (OldIrql);
        MiZeroPhysicalPage (PageFrameIndex, PageColor);
        LOCK_PFN (OldIrql);
    }

    MM_BUMP_SESS_COUNTER (MM_DBG_SESSION_WS_PAGE_ALLOC, ResidentPages - 1);

#if DBG
    Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
    ASSERT (Pfn1->u1.WsIndex == 0);
#endif

    //
    // The global bit is masked off since we need to make sure the TB entry
    // is flushed when we switch to a process in a different session space.
    //

    TempPte.u.Long = ValidKernelPteLocal.u.Long;
    MI_SET_PTE_DIRTY (TempPte);
    TempPte.u.Hard.PageFrameNumber = PageFrameIndex;

    MI_WRITE_VALID_PTE (PointerPte, TempPte);

    MiInitializePfn (PageFrameIndex, PointerPte, 1);

#if DBG
    Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
    ASSERT (Pfn1->u1.WsIndex == 0);
#endif

    UNLOCK_PFN (OldIrql);

    KeFillEntryTb ((PHARDWARE_PTE) PointerPte,
                   (PMMPTE)MmSessionSpace->Vm.VmWorkingSetList,
                   FALSE);

    //
    // Fill in the reserved slots starting with the session data page.
    //

    WslEntry = MmSessionSpace->Wsle;

    WslEntry->u1.VirtualAddress = (PVOID)MmSessionSpace;
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    Pfn1 = MI_PFN_ELEMENT (MiGetPteAddress (WslEntry->u1.VirtualAddress)->u.Hard.PageFrameNumber);

    ASSERT (Pfn1->u1.WsIndex == 0);

    //
    // The next reserved slot is for the page table page mapping
    // the session data page.
    //

    WslEntry += 1;

    WslEntry->u1.VirtualAddress = MiGetPteAddress (MmSessionSpace);
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    Pfn1 = MI_PFN_ELEMENT (MiGetPteAddress (WslEntry->u1.VirtualAddress)->u.Hard.PageFrameNumber);

    ASSERT (Pfn1->u1.WsIndex == 0);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmSessionSpace->Wsle);

    CONSISTENCY_UNLOCK_PFN (OldIrql);

    //
    // The next reserved slot is for the working set page.
    //

    WslEntry += 1;

    WslEntry->u1.VirtualAddress = MmSessionSpace->Vm.VmWorkingSetList;
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);

    ASSERT (Pfn1->u1.WsIndex == 0);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmSessionSpace->Wsle);

    CONSISTENCY_UNLOCK_PFN (OldIrql);

    if (AllocatedPageTable == TRUE) {

        //
        // The next reserved slot is for the page table page
        // mapping the working set page.
        //

        WslEntry += 1;

        WslEntry->u1.VirtualAddress = (PVOID)PointerPte;
        WslEntry->u1.e1.Valid = 1;
        WslEntry->u1.e1.LockedInWs = 1;
        WslEntry->u1.e1.Direct = 1;

        Pfn1 = MI_PFN_ELEMENT (MiGetPteAddress (WslEntry->u1.VirtualAddress)->u.Hard.PageFrameNumber);

        ASSERT (Pfn1->u1.WsIndex == 0);

        CONSISTENCY_LOCK_PFN (OldIrql);

        Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmSessionSpace->Wsle);

        CONSISTENCY_UNLOCK_PFN (OldIrql);
    }

    //
    // The next reserved slot is for the page table page
    // mapping the first session paged pool page.
    //

    WslEntry += 1;

    WslEntry->u1.VirtualAddress = (PVOID)MiGetPteAddress (MmSessionSpace->PagedPoolStart);
    WslEntry->u1.e1.Valid = 1;
    WslEntry->u1.e1.LockedInWs = 1;
    WslEntry->u1.e1.Direct = 1;

    Pfn1 = MI_PFN_ELEMENT (MiGetPteAddress (WslEntry->u1.VirtualAddress)->u.Hard.PageFrameNumber);

    ASSERT (Pfn1->u1.WsIndex == 0);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.WsIndex = (WSLE_NUMBER)(WslEntry - MmSessionSpace->Wsle);

    CONSISTENCY_UNLOCK_PFN (OldIrql);

    CurrentEntry = (WSLE_NUMBER)(WslEntry + 1 - MmSessionSpace->Wsle);

    MmSessionSpace->Vm.u.Flags.SessionSpace = 1;
    MmSessionSpace->Vm.MinimumWorkingSetSize = MI_SESSION_SPACE_WORKING_SET_MINIMUM;
    MmSessionSpace->Vm.MaximumWorkingSetSize = WorkingSetMaximum;

    //
    // Don't trim from this session till we're finished setting up and
    // it's got some pages in it...
    //

    MmSessionSpace->Vm.AllowWorkingSetAdjustment = FALSE;

    MmSessionSpace->Vm.VmWorkingSetList->LastEntry = MI_SESSION_SPACE_WORKING_SET_MINIMUM;
    MmSessionSpace->Vm.VmWorkingSetList->Quota = MmSessionSpace->Vm.VmWorkingSetList->LastEntry;
    MmSessionSpace->Vm.VmWorkingSetList->HashTable = NULL;
    MmSessionSpace->Vm.VmWorkingSetList->HashTableSize = 0;
    MmSessionSpace->Vm.VmWorkingSetList->Wsle = MmSessionSpace->Wsle;

    MmSessionSpace->Vm.VmWorkingSetList->HashTableStart =
       (PVOID)((PCHAR)PAGE_ALIGN (&MmSessionSpace->Wsle[MI_SESSION_MAXIMUM_WORKING_SET]) + PAGE_SIZE);

#if defined (_X86PAE_)

    //
    // One less page table page is needed on PAE systems.
    //

    MmSessionSpace->Vm.VmWorkingSetList->HighestPermittedHashAddress =
        (PVOID)(MI_SESSION_VIEW_START - MM_VA_MAPPED_BY_PDE);
#else
    MmSessionSpace->Vm.VmWorkingSetList->HighestPermittedHashAddress =
        (PVOID)MI_SESSION_VIEW_START;
#endif

    NumberOfEntriesMapped = (WSLE_NUMBER)(((PMMWSLE)((ULONG_PTR)MmSessionSpace->Vm.VmWorkingSetList +
                                PAGE_SIZE)) - MmSessionSpace->Wsle);

    LOCK_PFN (OldIrql);

    while (NumberOfEntriesMapped < WorkingSetMaximum) {

        PointerPte += 1;

        MiEnsureAvailablePageOrWait (NULL, NULL);

        PageFrameIndex = MiRemoveZeroPage(MI_GET_PAGE_COLOR_FROM_VA (NULL));

        PointerPte->u.Long = MM_DEMAND_ZERO_WRITE_PTE;

        MiInitializePfn (PageFrameIndex, PointerPte, 1);

        TempPte.u.Hard.PageFrameNumber = PageFrameIndex;

        MI_SET_PTE_IN_WORKING_SET (&TempPte, CurrentEntry);

        MI_WRITE_VALID_PTE (PointerPte, TempPte);

        WslEntry += 1;

        WslEntry->u1.VirtualAddress = MiGetVirtualAddressMappedByPte (PointerPte);
        WslEntry->u1.e1.Valid = 1;
        WslEntry->u1.e1.LockedInWs = 1;
        WslEntry->u1.e1.Direct = 1;

        Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

        ASSERT (Pfn1->u1.WsIndex == 0);
        Pfn1->u1.WsIndex = CurrentEntry;

        // MiInsertWsle(CurrentEntry, MmWorkingSetList);

        CurrentEntry += 1;

        NumberOfEntriesMapped += PAGE_SIZE / sizeof(MMWSLE);
    }

    UNLOCK_PFN (OldIrql);

    MmSessionSpace->Vm.WorkingSetSize = CurrentEntry;
    MmSessionSpace->Vm.VmWorkingSetList->FirstFree = CurrentEntry;
    MmSessionSpace->Vm.VmWorkingSetList->FirstDynamic = CurrentEntry;
    MmSessionSpace->Vm.VmWorkingSetList->NextSlot = CurrentEntry;

    MmSessionSpace->NonPagablePages += ResidentPages;
    MmSessionSpace->CommittedPages += ResidentPages;

    //
    // Initialize the following slots as free.
    //

    WslEntry = MmSessionSpace->Wsle + CurrentEntry;

    for (i = CurrentEntry + 1; i < NumberOfEntriesMapped; i += 1) {

        //
        // Build the free list, note that the first working
        // set entries (CurrentEntry) are not on the free list.
        // These entries are reserved for the pages which
        // map the working set and the page which contains the PDE.
        //

        WslEntry->u1.Long = i << MM_FREE_WSLE_SHIFT;
        WslEntry += 1;
    }

    WslEntry->u1.Long = WSLE_NULL_INDEX << MM_FREE_WSLE_SHIFT;  // End of list.

    MmSessionSpace->Vm.VmWorkingSetList->LastInitializedWsle = NumberOfEntriesMapped - 1;

    if (WorkingSetMaximum > ((1536*1024) >> PAGE_SHIFT)) {

        //
        // The working set list consists of more than a single page.
        //

        MiGrowWsleHash (&MmSessionSpace->Vm);
    }

    //
    // Put this session's working set in lists using its global address.
    //

    LOCK_EXPANSION (OldIrql);

    InsertTailList (&MiSessionWsList, &SessionGlobal->WsListEntry);

    MmSessionSpace->u.Flags.HasWsLock = 1;

    MmSessionSpace->u.Flags.SessionListInserted = 1;

    UNLOCK_EXPANSION (OldIrql);

    return STATUS_SUCCESS;
}


LOGICAL
MmAssignProcessToJob (
    IN PEPROCESS Process
    )

/*++

Routine Description:

    This routine acquires the working set lock so a consistent snapshot of
    the argument process' commit charges and working set size can be used
    when adding this process to a job.

Arguments:

    Process - Supplies a pointer to the process to operate upon.

Return Value:

    TRUE if the process is allowed to join the job, FALSE otherwise.

    Note that FALSE cannot be returned without changing the code in ps.

Environment:

    Kernel mode, IRQL APC_LEVEL or below.  The caller provides protection
    from the target process going away.

--*/

{
    LOGICAL Attached;
    LOGICAL Status;

    PAGED_CODE ();

    Attached = FALSE;

    if (PsGetCurrentProcess() != Process) {
        KeAttachProcess (&Process->Pcb);
        Attached = TRUE;
    }

    LOCK_WS_AND_ADDRESS_SPACE (Process);

    Status = PsChangeJobMemoryUsage (Process->CommitCharge);

    //
    // Join the job unconditionally.  If the process is over any limits, it
    // will be caught on its next request.
    //

    Process->JobStatus |= PS_JOB_STATUS_REPORT_COMMIT_CHANGES;

    UNLOCK_WS_AND_ADDRESS_SPACE (Process);

    if (Attached) {
        KeDetachProcess();
    }

    return TRUE;
}


NTSTATUS
MmAdjustWorkingSetSize (
    IN SIZE_T WorkingSetMinimumInBytes,
    IN SIZE_T WorkingSetMaximumInBytes,
    IN ULONG SystemCache
    )

/*++

Routine Description:

    This routine adjusts the current size of a process's working set
    list.  If the maximum value is above the current maximum, pages
    are removed from the working set list.

    An exception is raised if the limit cannot be granted.  This
    could occur if too many pages were locked in the process's
    working set.

    Note: if the minimum and maximum are both (SIZE_T)-1, the working set
          is purged, but the default sizes are not changed.

Arguments:

    WorkingSetMinimumInBytes - Supplies the new minimum working set size in
                               bytes.

    WorkingSetMaximumInBytes - Supplies the new maximum working set size in
                               bytes.

    SystemCache - Supplies TRUE if the system cache working set is being
                  adjusted, FALSE for all other working sets.

Return Value:

    NTSTATUS.

Environment:

    Kernel mode, IRQL APC_LEVEL or below.

--*/


{
    PEPROCESS CurrentProcess;
    ULONG Entry;
    ULONG LastFreed;
    PMMWSLE Wsle;
    KIRQL OldIrql;
    KIRQL OldIrql2;
    SPFN_NUMBER i;
    PMMPTE PointerPte;
    NTSTATUS ReturnStatus;
    LONG PagesAbove;
    LONG NewPagesAbove;
    ULONG FreeTryCount;
    PMMSUPPORT WsInfo;
    PMMWSL WorkingSetList;
    WSLE_NUMBER WorkingSetMinimum;
    WSLE_NUMBER WorkingSetMaximum;

    PERFINFO_PAGE_INFO_DECL();

    FreeTryCount = 0;

    if (SystemCache) {
        WsInfo = &MmSystemCacheWs;
    } else {
        CurrentProcess = PsGetCurrentProcess ();
        WsInfo = &CurrentProcess->Vm;
    }

    if ((WorkingSetMinimumInBytes == (SIZE_T)-1) &&
        (WorkingSetMaximumInBytes == (SIZE_T)-1)) {
        return MiEmptyWorkingSet (WsInfo, TRUE);
    }

    if (WorkingSetMinimumInBytes == 0) {
        WorkingSetMinimum = WsInfo->MinimumWorkingSetSize;
    }
    else {
        WorkingSetMinimum = (WSLE_NUMBER)(WorkingSetMinimumInBytes >> PAGE_SHIFT);
    }

    if (WorkingSetMaximumInBytes == 0) {
        WorkingSetMaximum = WsInfo->MaximumWorkingSetSize;
    }
    else {
        WorkingSetMaximum = (WSLE_NUMBER)(WorkingSetMaximumInBytes >> PAGE_SHIFT);
    }

    if (WorkingSetMinimum > WorkingSetMaximum) {
        return STATUS_BAD_WORKING_SET_LIMIT;
    }

    MmLockPagableSectionByHandle(ExPageLockHandle);

    ReturnStatus = STATUS_SUCCESS;

    //
    // Get the working set lock and disable APCs.
    //

    if (SystemCache) {
        LOCK_SYSTEM_WS (OldIrql2);
    } else {
        LOCK_WS (CurrentProcess);

        if (CurrentProcess->AddressSpaceDeleted != 0) {
            ReturnStatus = STATUS_PROCESS_IS_TERMINATING;
            goto Returns;
        }
    }

    if (WorkingSetMaximum > MmMaximumWorkingSetSize) {
        WorkingSetMaximum = MmMaximumWorkingSetSize;
        ReturnStatus = STATUS_WORKING_SET_LIMIT_RANGE;
    }

    if (WorkingSetMinimum > MmMaximumWorkingSetSize) {
        WorkingSetMinimum = MmMaximumWorkingSetSize;
        ReturnStatus = STATUS_WORKING_SET_LIMIT_RANGE;
    }

    if (WorkingSetMinimum < MmMinimumWorkingSetSize) {
        WorkingSetMinimum = (ULONG)MmMinimumWorkingSetSize;
        ReturnStatus = STATUS_WORKING_SET_LIMIT_RANGE;
    }

    //
    // Make sure that the number of locked pages will not
    // make the working set not fluid.
    //

    if ((WsInfo->VmWorkingSetList->FirstDynamic + MM_FLUID_WORKING_SET) >=
         WorkingSetMaximum) {
        ReturnStatus = STATUS_BAD_WORKING_SET_LIMIT;
        goto Returns;
    }

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

    //
    // Check to make sure ample resident physical pages exist for
    // this operation.
    //

    LOCK_PFN (OldIrql);

    i = WorkingSetMinimum - WsInfo->MinimumWorkingSetSize;

    if (i > 0) {

        //
        // New minimum working set is greater than the old one.  Ensure that
        // we don't allow this process' working set minimum to increase to
        // a point where subsequent nonpaged pool allocations could cause
        // us to run out of pages.  Additionally, leave 100 extra pages around
        // so the user can later bring up tlist and kill processes if necessary.
        //

        if (MmAvailablePages < (20 + (i / (PAGE_SIZE / sizeof (MMWSLE))))) {
            UNLOCK_PFN (OldIrql);
            ReturnStatus = STATUS_INSUFFICIENT_RESOURCES;
            goto Returns;
        }

        if (MI_NONPAGABLE_MEMORY_AVAILABLE() - 100 < i) {
            UNLOCK_PFN (OldIrql);
            ReturnStatus = STATUS_INSUFFICIENT_RESOURCES;
            goto Returns;
        }
    }

    //
    // Adjust the number of resident pages up or down dependent on
    // the size of the new minimum working set size versus the previous
    // minimum size.
    //

    MmResidentAvailablePages -= i;
    MM_BUMP_COUNTER(27, i);

    UNLOCK_PFN (OldIrql);

    if (WsInfo->AllowWorkingSetAdjustment == FALSE) {
        MmAllowWorkingSetExpansion ();
    }

    if (WorkingSetMaximum > WorkingSetList->LastInitializedWsle) {

         do {

            //
            // The maximum size of the working set is being increased, check
            // to ensure the proper number of pages are mapped to cover
            // the complete working set list.
            //

            if (!MiAddWorkingSetPage (WsInfo)) {
                WorkingSetMaximum = WorkingSetList->LastInitializedWsle - 1;
                break;
            }
        } while (WorkingSetMaximum > WorkingSetList->LastInitializedWsle);

    } else {

        //
        // The new working set maximum is less than the current working set
        // maximum.
        //

        if (WsInfo->WorkingSetSize > WorkingSetMaximum) {

            //
            // Remove some pages from the working set.
            //

            //
            // Make sure that the number of locked pages will not
            // make the working set not fluid.
            //

            if ((WorkingSetList->FirstDynamic + MM_FLUID_WORKING_SET) >=
                 WorkingSetMaximum) {

                ReturnStatus = STATUS_BAD_WORKING_SET_LIMIT;

                LOCK_PFN (OldIrql);

                MmResidentAvailablePages += i;
                MM_BUMP_COUNTER(54, i);

                UNLOCK_PFN (OldIrql);

                goto Returns;
            }

            //
            // Attempt to remove the pages from the Maximum downward.
            //

            LastFreed = WorkingSetList->LastEntry;
            if (WorkingSetList->LastEntry > WorkingSetMaximum) {

                while (LastFreed >= WorkingSetMaximum) {

                    PointerPte = MiGetPteAddress(
                                        Wsle[LastFreed].u1.VirtualAddress);

                    PERFINFO_GET_PAGE_INFO(PointerPte);

                    if ((Wsle[LastFreed].u1.e1.Valid != 0) &&
                        (!MiFreeWsle (LastFreed,
                                      WsInfo,
                                      PointerPte))) {

                        //
                        // This LastFreed could not be removed.
                        //

                        break;
                    }
                    PERFINFO_LOG_WS_REMOVAL(PERFINFO_LOG_TYPE_OUTWS_ADJUSTWS, WsInfo);
                    LastFreed -= 1;
                }
                WorkingSetList->LastEntry = LastFreed;
            }

            //
            // Remove pages.
            //

            Entry = WorkingSetList->FirstDynamic;

            while (WsInfo->WorkingSetSize > WorkingSetMaximum) {
                if (Wsle[Entry].u1.e1.Valid != 0) {
                    PointerPte = MiGetPteAddress (
                                            Wsle[Entry].u1.VirtualAddress);
                    PERFINFO_GET_PAGE_INFO(PointerPte);

                    if (MiFreeWsle(Entry, WsInfo, PointerPte)) {
                        PERFINFO_LOG_WS_REMOVAL(PERFINFO_LOG_TYPE_OUTWS_ADJUSTWS,
                                              WsInfo);
                    }
                }
                Entry += 1;
                if (Entry > LastFreed) {
                    FreeTryCount += 1;
                    if (FreeTryCount > MM_RETRY_COUNT) {

                        //
                        // Page table pages are not becoming free, give up
                        // and return an error.
                        //

                        ReturnStatus = STATUS_BAD_WORKING_SET_LIMIT;

                        break;
                    }
                    Entry = WorkingSetList->FirstDynamic;
                }
            }

            if (FreeTryCount <= MM_RETRY_COUNT) {
                WorkingSetList->Quota = WorkingSetMaximum;
            }
        }
    }

    //
    // Adjust the number of pages above the working set minimum.
    //

    PagesAbove = (LONG)WsInfo->WorkingSetSize -
                               (LONG)WsInfo->MinimumWorkingSetSize;
    NewPagesAbove = (LONG)WsInfo->WorkingSetSize -
                               (LONG)WorkingSetMinimum;

    LOCK_PFN (OldIrql);
    if (PagesAbove > 0) {
        MmPagesAboveWsMinimum -= (ULONG)PagesAbove;
    }
    if (NewPagesAbove > 0) {
        MmPagesAboveWsMinimum += (ULONG)NewPagesAbove;
    }
    UNLOCK_PFN (OldIrql);

    if (FreeTryCount <= MM_RETRY_COUNT) {
        WsInfo->MaximumWorkingSetSize = WorkingSetMaximum;
        WsInfo->MinimumWorkingSetSize = WorkingSetMinimum;

        if (WorkingSetMinimum >= WorkingSetList->Quota) {
            WorkingSetList->Quota = WorkingSetMinimum;
        }
    }
    else {
        LOCK_PFN (OldIrql);

        MmResidentAvailablePages += i;
        MM_BUMP_COUNTER(55, i);

        UNLOCK_PFN (OldIrql);
    }


    ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
            (WorkingSetList->FirstFree == WSLE_NULL_INDEX));

    if ((WorkingSetList->HashTable == NULL) &&
        (WsInfo->MaximumWorkingSetSize > ((1536*1024) >> PAGE_SHIFT))) {

        //
        // The working set list consists of more than a single page.
        //

        MiGrowWsleHash (WsInfo);
    }

Returns:

    if (SystemCache) {
        UNLOCK_SYSTEM_WS (OldIrql2);
    } else {
        UNLOCK_WS (CurrentProcess);
    }

    MmUnlockPagableImageSection(ExPageLockHandle);

    return ReturnStatus;
}

ULONG
MiAddWorkingSetPage (
    IN PMMSUPPORT WsInfo
    )

/*++

Routine Description:

    This function grows the working set list above working set
    maximum during working set adjustment.  At most one page
    can be added at a time.

Arguments:

    None.

Return Value:

    Returns FALSE if no working set page could be added.

Environment:

    Kernel mode, APCs disabled, working set mutexes held.

--*/

{
    ULONG SwapEntry;
    ULONG CurrentEntry;
    PMMWSLE WslEntry;
    ULONG i;
    PMMPTE PointerPte;
    PMMPTE PointerPde;
    PMMPTE Va;
    MMPTE TempPte;
    WSLE_NUMBER NumberOfEntriesMapped;
    PFN_NUMBER WorkingSetPage;
    WSLE_NUMBER WorkingSetIndex;
    PMMWSL WorkingSetList;
    PMMWSLE Wsle;
    PMMPFN Pfn1;
    KIRQL OldIrql;
    LOGICAL PageTablePageAllocated;

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

#if DBG
    if (WsInfo == &MmSystemCacheWs) {
        MM_SYSTEM_WS_LOCK_ASSERT();
    }
#endif //DBG

    //
    // The maximum size of the working set is being increased, check
    // to ensure the proper number of pages are mapped to cover
    // the complete working set list.
    //

    PointerPte = MiGetPteAddress (&Wsle[WorkingSetList->LastInitializedWsle]);

    ASSERT (PointerPte->u.Hard.Valid == 1);

    PointerPte += 1;

    Va = (PMMPTE)MiGetVirtualAddressMappedByPte (PointerPte);

    if ((PVOID)Va >= WorkingSetList->HashTableStart) {

        //
        // Adding this entry would overrun the hash table.  The caller
        // must replace instead.
        //

        WorkingSetList->Quota = WorkingSetList->LastInitializedWsle;
        return FALSE;
    }

    PageTablePageAllocated = FALSE;

#if defined (_WIN64)
    PointerPde = MiGetPteAddress (PointerPte);
    if (PointerPde->u.Hard.Valid == 0) {

        ASSERT (WsInfo->u.Flags.SessionSpace == 0);

        //
        // Map in a new working set page.
        //
    
        LOCK_PFN (OldIrql);
        if (MmAvailablePages < 21) {
    
            //
            // No pages are available, set the quota to the last
            // initialized WSLE and return.
            //
    
            WorkingSetList->Quota = WorkingSetList->LastInitializedWsle;
            UNLOCK_PFN (OldIrql);
            return FALSE;
        }
    
        PageTablePageAllocated = TRUE;
        WorkingSetPage = MiRemoveZeroPage (MI_GET_PAGE_COLOR_FROM_PTE (PointerPde));
        PointerPde->u.Long = MM_DEMAND_ZERO_WRITE_PTE;
        MiInitializePfn (WorkingSetPage, PointerPde, 1);
        UNLOCK_PFN (OldIrql);
    
        MI_MAKE_VALID_PTE (TempPte,
                           WorkingSetPage,
                           MM_READWRITE,
                           PointerPde);
    
        MI_SET_PTE_DIRTY (TempPte);
        MI_WRITE_VALID_PTE (PointerPde, TempPte);
    
        //
        // Further down in this routine (once an actual working set page
        // has been allocated) the quota will be increased by 1 to reflect
        // the working set size entry for the new page table page.
        // The page table page will be put in a working set entry which will
        // be locked into the working set.
        //
    }
#endif

    ASSERT (PointerPte->u.Hard.Valid == 0);

    NumberOfEntriesMapped = (WSLE_NUMBER)(((PMMWSLE)((PCHAR)Va + PAGE_SIZE)) - Wsle);

    //
    // Map in a new working set page.
    //

    LOCK_PFN (OldIrql);
    if ((PageTablePageAllocated == FALSE) && (MmAvailablePages < 20)) {

        //
        // No pages are available, set the quota to the last
        // initialized WSLE and return.
        //

        WorkingSetList->Quota = WorkingSetList->LastInitializedWsle;
        UNLOCK_PFN (OldIrql);
        return FALSE;
    }

    WorkingSetPage = MiRemoveZeroPage (MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));
    PointerPte->u.Long = MM_DEMAND_ZERO_WRITE_PTE;
    MiInitializePfn (WorkingSetPage, PointerPte, 1);
    UNLOCK_PFN (OldIrql);

    MI_MAKE_VALID_PTE (TempPte,
                       WorkingSetPage,
                       MM_READWRITE,
                       PointerPte);

    MI_SET_PTE_DIRTY (TempPte);
    MI_WRITE_VALID_PTE (PointerPte, TempPte);

    if (WsInfo->u.Flags.SessionSpace == 1) {
        MM_BUMP_SESS_COUNTER (MM_DBG_SESSION_WS_PAGE_ALLOC_GROWTH, 1);
        MmSessionSpace->NonPagablePages += 1;
        MmSessionSpace->CommittedPages += 1;
        MiChargeCommitmentCantExpand (1, TRUE);
        LOCK_PFN (OldIrql);
        MM_BUMP_COUNTER (48, 1);
        MmResidentAvailablePages -= 1;
        UNLOCK_PFN (OldIrql);
        MM_TRACK_COMMIT (MM_DBG_COMMIT_SESSION_ADDITIONAL_WS_PAGES, 1);
    }

    CurrentEntry = WorkingSetList->LastInitializedWsle + 1;

    ASSERT (NumberOfEntriesMapped > CurrentEntry);

    WslEntry = &Wsle[CurrentEntry - 1];

    for (i = CurrentEntry; i < NumberOfEntriesMapped; i += 1) {

        //
        // Build the free list, note that the first working
        // set entries (CurrentEntry) are not on the free list.
        // These entries are reserved for the pages which
        // map the working set and the page which contains the PDE.
        //

        WslEntry += 1;
        WslEntry->u1.Long = (i + 1) << MM_FREE_WSLE_SHIFT;
    }

    WslEntry->u1.Long = WorkingSetList->FirstFree << MM_FREE_WSLE_SHIFT;

    ASSERT (CurrentEntry >= WorkingSetList->FirstDynamic);

    WorkingSetList->FirstFree = CurrentEntry;

    WorkingSetList->LastInitializedWsle = (NumberOfEntriesMapped - 1);

    ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
            (WorkingSetList->FirstFree == WSLE_NULL_INDEX));

    //
    // As we are growing the working set, make sure the quota is
    // above the working set size by adding 1 to the quota.
    //

    WorkingSetList->Quota += 1;

    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.Event = (PVOID)PsGetCurrentThread();

    CONSISTENCY_UNLOCK_PFN (OldIrql);

    //
    // Get a working set entry.
    //

    WsInfo->WorkingSetSize += 1;

    ASSERT (WorkingSetList->FirstFree != WSLE_NULL_INDEX);
    ASSERT (WorkingSetList->FirstFree >= WorkingSetList->FirstDynamic);

    WorkingSetIndex = WorkingSetList->FirstFree;
    WorkingSetList->FirstFree = (WSLE_NUMBER)(Wsle[WorkingSetIndex].u1.Long >> MM_FREE_WSLE_SHIFT);
    ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
            (WorkingSetList->FirstFree == WSLE_NULL_INDEX));

    if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) {
        MmPagesAboveWsMinimum += 1;
    }
    if (WorkingSetIndex > WorkingSetList->LastEntry) {
        WorkingSetList->LastEntry = WorkingSetIndex;
    }

    MiUpdateWsle (&WorkingSetIndex, Va, WorkingSetList, Pfn1);

    MI_SET_PTE_IN_WORKING_SET (PointerPte, WorkingSetIndex);

    //
    // Lock any created page table pages into the working set.
    //

    if (WorkingSetIndex >= WorkingSetList->FirstDynamic) {

        SwapEntry = WorkingSetList->FirstDynamic;

        if (WorkingSetIndex != WorkingSetList->FirstDynamic) {

            //
            // Swap this entry with the one at first dynamic.
            //

            MiSwapWslEntries (WorkingSetIndex, SwapEntry, WsInfo);
        }

        WorkingSetList->FirstDynamic += 1;

        Wsle[SwapEntry].u1.e1.LockedInWs = 1;
        ASSERT (Wsle[SwapEntry].u1.e1.Valid == 1);
    }

#if defined (_WIN64)
    if (PageTablePageAllocated == TRUE) {
    
        //
        // As we are growing the working set, make sure the quota is
        // above the working set size by adding 1 to the quota.
        //
    
        WorkingSetList->Quota += 1;
    
        Pfn1 = MI_PFN_ELEMENT (PointerPde->u.Hard.PageFrameNumber);
    
        CONSISTENCY_LOCK_PFN (OldIrql);
    
        Pfn1->u1.Event = (PVOID)PsGetCurrentThread();
    
        CONSISTENCY_UNLOCK_PFN (OldIrql);
    
        //
        // Get a working set entry.
        //
    
        WsInfo->WorkingSetSize += 1;
    
        ASSERT (WorkingSetList->FirstFree != WSLE_NULL_INDEX);
        ASSERT (WorkingSetList->FirstFree >= WorkingSetList->FirstDynamic);
    
        WorkingSetIndex = WorkingSetList->FirstFree;
        WorkingSetList->FirstFree = (WSLE_NUMBER)(Wsle[WorkingSetIndex].u1.Long >> MM_FREE_WSLE_SHIFT);
        ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
                (WorkingSetList->FirstFree == WSLE_NULL_INDEX));
    
        if (WsInfo->WorkingSetSize > WsInfo->MinimumWorkingSetSize) {
            MmPagesAboveWsMinimum += 1;
        }
        if (WorkingSetIndex > WorkingSetList->LastEntry) {
            WorkingSetList->LastEntry = WorkingSetIndex;
        }
    
        MiUpdateWsle (&WorkingSetIndex, PointerPte, WorkingSetList, Pfn1);
    
        MI_SET_PTE_IN_WORKING_SET (PointerPde, WorkingSetIndex);
    
        //
        // Lock the created page table page into the working set.
        //
    
        if (WorkingSetIndex >= WorkingSetList->FirstDynamic) {
    
            SwapEntry = WorkingSetList->FirstDynamic;
    
            if (WorkingSetIndex != WorkingSetList->FirstDynamic) {
    
                //
                // Swap this entry with the one at first dynamic.
                //
    
                MiSwapWslEntries (WorkingSetIndex, SwapEntry, WsInfo);
            }
    
            WorkingSetList->FirstDynamic += 1;
    
            Wsle[SwapEntry].u1.e1.LockedInWs = 1;
            ASSERT (Wsle[SwapEntry].u1.e1.Valid == 1);
        }
    }
#endif

    ASSERT ((MiGetPteAddress(&Wsle[WorkingSetList->LastInitializedWsle]))->u.Hard.Valid == 1);

    if ((WorkingSetList->HashTable == NULL) &&
        (MmAvailablePages > 20)) {

        //
        // Add a hash table to support shared pages in the working set to
        // eliminate costly lookups.
        //

        LOCK_EXPANSION_IF_ALPHA (OldIrql);
        ASSERT (WsInfo->AllowWorkingSetAdjustment != FALSE);
        WsInfo->AllowWorkingSetAdjustment = MM_GROW_WSLE_HASH;
        UNLOCK_EXPANSION_IF_ALPHA (OldIrql);
    }

    ASSERT (WsInfo->WorkingSetSize <= WorkingSetList->Quota);
    return TRUE;
}

LOGICAL
MiAddWsleHash (
    IN PMMSUPPORT WsInfo,
    IN PMMPTE PointerPte
    )

/*++

Routine Description:

    This function adds a page directory, page table or actual mapping page
    for hash table creation (or expansion) for the current process.

Arguments:

    WsInfo - Supplies a pointer to the working set info block for the
             process (or system cache).

    PointerPte - Supplies a pointer to the PTE to be filled.

Return Value:

    None.

Environment:

    Kernel mode, APCs disabled, working set lock held.

--*/
{
    KIRQL OldIrql;
    PMMPFN Pfn1;
    ULONG SwapEntry;
    MMPTE TempPte;
    PVOID Va;
    PMMWSLE Wsle;
    PFN_NUMBER WorkingSetPage;
    WSLE_NUMBER WorkingSetIndex;
    PMMWSL WorkingSetList;

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

    ASSERT (PointerPte->u.Hard.Valid == 0);

    LOCK_PFN (OldIrql);

    if (MmAvailablePages < 10) {
        UNLOCK_PFN (OldIrql);
        return FALSE;
    }

    WorkingSetPage = MiRemoveZeroPage (
                                MI_GET_PAGE_COLOR_FROM_PTE (PointerPte));

    PointerPte->u.Long = MM_DEMAND_ZERO_WRITE_PTE;
    MiInitializePfn (WorkingSetPage, PointerPte, 1);

    MI_MAKE_VALID_PTE (TempPte,
                       WorkingSetPage,
                       MM_READWRITE,
                       PointerPte );

    MI_SET_PTE_DIRTY (TempPte);
    MI_WRITE_VALID_PTE (PointerPte, TempPte);

    UNLOCK_PFN (OldIrql);

    //
    // As we are growing the working set, we know that quota
    // is above the current working set size.  Just take the
    // next free WSLE from the list and use it.
    //

    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

    CONSISTENCY_LOCK_PFN (OldIrql);

    Pfn1->u1.Event = (PVOID)PsGetCurrentThread();

    CONSISTENCY_UNLOCK_PFN (OldIrql);

    Va = (PMMPTE)MiGetVirtualAddressMappedByPte (PointerPte);

    WorkingSetIndex = MiLocateAndReserveWsle (WsInfo);
    MiUpdateWsle (&WorkingSetIndex, Va, WorkingSetList, Pfn1);
    MI_SET_PTE_IN_WORKING_SET (PointerPte, WorkingSetIndex);

    //
    // Lock any created page table pages into the working set.
    //

    if (WorkingSetIndex >= WorkingSetList->FirstDynamic) {

        SwapEntry = WorkingSetList->FirstDynamic;

        if (WorkingSetIndex != WorkingSetList->FirstDynamic) {

            //
            // Swap this entry with the one at first dynamic.
            //

            MiSwapWslEntries (WorkingSetIndex, SwapEntry, WsInfo);
        }

        WorkingSetList->FirstDynamic += 1;

        Wsle[SwapEntry].u1.e1.LockedInWs = 1;
        ASSERT (Wsle[SwapEntry].u1.e1.Valid == 1);
    }

    if (WsInfo->u.Flags.SessionSpace == 1) {
        MM_BUMP_SESS_COUNTER (MM_DBG_SESSION_WS_HASHPAGE_ALLOC, 1);
        MmSessionSpace->NonPagablePages += 1;
        MmSessionSpace->CommittedPages += 1;
        MiChargeCommitmentCantExpand (1, TRUE);
        MM_TRACK_COMMIT (MM_DBG_COMMIT_SESSION_ADDITIONAL_WS_HASHPAGES, 1);
    }
    return TRUE;
}

VOID
MiGrowWsleHash (
    IN PMMSUPPORT WsInfo
    )

/*++

Routine Description:

    This function grows (or adds) a hash table to the working set list
    to allow direct indexing for WSLEs than cannot be located via the
    PFN database WSINDEX field.

    The hash table is located AFTER the WSLE array and the pages are
    locked into the working set just like standard WSLEs.

    Note that the hash table is expanded by setting the hash table
    field in the working set to NULL, but leaving the size as non-zero.
    This indicates that the hash should be expanded and the initial
    portion of the table zeroed.

Arguments:

    WsInfo - Supplies a pointer to the working set info block for the
             process (or system cache).

Return Value:

    None.

Environment:

    Kernel mode, APCs disabled, working set lock held.

--*/
{
    LONG Size;
    PMMWSLE Wsle;
    PMMPTE StartPte;
    PMMPTE EndPte;
    PMMPTE PointerPte;
    PMMPTE PointerPde;
    PMMPTE PointerPpe;
    PMMPTE AllocatedPde;
    PMMPTE AllocatedPpe;
    ULONG First;
    ULONG Hash;
    ULONG NewSize;
    PMMWSLE_HASH Table;
    PMMWSLE_HASH OriginalTable;
    ULONG j;
    PMMWSL WorkingSetList;
    KIRQL OldIrql;
    ULONG Count;
    LOGICAL LoopStart;
    PVOID EntryHashTableEnd;
    PVOID TempVa;
    PEPROCESS CurrentProcess;

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

    Table = WorkingSetList->HashTable;
    OriginalTable = WorkingSetList->HashTable;

    First = WorkingSetList->HashTableSize;

    if (Table == NULL) {

        NewSize = PtrToUlong(PAGE_ALIGN (((1 + WorkingSetList->NonDirectCount) *
                            2 * sizeof(MMWSLE_HASH)) + PAGE_SIZE - 1));

        //
        // Note that the Table may be NULL and the HashTableSize/PTEs nonzero
        // in the case where the hash has been contracted.
        //

        j = First * sizeof(MMWSLE_HASH);

        //
        // Don't try for additional hash pages if we already have
        // the right amount (or too many).
        //

        if ((j + PAGE_SIZE > NewSize) && (j != 0)) {
            return;
        }

        Table = (PMMWSLE_HASH)(WorkingSetList->HashTableStart);
        EntryHashTableEnd = &Table[WorkingSetList->HashTableSize];

        WorkingSetList->HashTableSize = 0;

    } else {

        //
        // Attempt to add 4 pages, make sure the working set list has
        // 4 free entries.
        //

        if ((WorkingSetList->LastInitializedWsle + 5) > WsInfo->WorkingSetSize) {
            NewSize = PAGE_SIZE * 4;
        } else {
            NewSize = PAGE_SIZE;
        }
        EntryHashTableEnd = &Table[WorkingSetList->HashTableSize];
    }

    if ((PCHAR)EntryHashTableEnd + NewSize > (PCHAR)WorkingSetList->HighestPermittedHashAddress) {
        NewSize =
            (ULONG)((PCHAR)(WorkingSetList->HighestPermittedHashAddress) -
                ((PCHAR)EntryHashTableEnd));
        if (NewSize == 0) {
            if (OriginalTable == NULL) {
                WorkingSetList->HashTableSize = First;
            }
            return;
        }
    }

    ASSERT64 ((MiGetPpeAddress(EntryHashTableEnd)->u.Hard.Valid == 0) ||
              (MiGetPdeAddress(EntryHashTableEnd)->u.Hard.Valid == 0) ||
              (MiGetPteAddress(EntryHashTableEnd)->u.Hard.Valid == 0));

    ASSERT32 (MiGetPteAddress(EntryHashTableEnd)->u.Hard.Valid == 0);

    Size = NewSize;
    PointerPte = MiGetPteAddress (EntryHashTableEnd);
    StartPte = PointerPte;
    EndPte = PointerPte + (NewSize >> PAGE_SHIFT);

#if defined (_WIN64)
    LoopStart = TRUE;
    AllocatedPde = NULL;
    AllocatedPpe = NULL;
#endif

    do {

#if defined (_WIN64)
        if (LoopStart == TRUE || MiIsPteOnPdeBoundary(PointerPte)) {

            PointerPpe = MiGetPdeAddress(PointerPte);
            PointerPde = MiGetPteAddress(PointerPte);

            if (PointerPpe->u.Hard.Valid == 0) {
                if (MiAddWsleHash (WsInfo, PointerPpe) == FALSE) {
                    break;
                }
                AllocatedPpe = PointerPpe;
            }

            if (PointerPde->u.Hard.Valid == 0) {
                if (MiAddWsleHash (WsInfo, PointerPde) == FALSE) {
                    break;
                }
                AllocatedPde = PointerPde;
            }

            LoopStart = FALSE;
        }
        else {
            AllocatedPde = NULL;
            AllocatedPpe = NULL;
        }
#endif

        if (PointerPte->u.Hard.Valid == 0) {
            if (MiAddWsleHash (WsInfo, PointerPte) == FALSE) {
                break;
            }
        }

        PointerPte += 1;
        Size -= PAGE_SIZE;
    } while (Size > 0);

    //
    // If MiAddWsleHash was unable to allocate memory above, then roll back
    // any extra PPEs & PDEs that may have been created.  Note NewSize must
    // be recalculated to handle the fact that memory may have run out.
    //

#if !defined (_WIN64)
    if (PointerPte == StartPte) {
        if (OriginalTable == NULL) {
            WorkingSetList->HashTableSize = First;
        }
        return;
    }
#else
    if (PointerPte != EndPte) {

        //
        // Clean up the last allocated PPE/PDE as they are not needed.
        // Note that the system cache and the session space working sets
        // have no current process (which MiDeletePte requires) which is
        // needed for WSLE and PrivatePages adjustments.
        //

        if (WsInfo != &MmSystemCacheWs && WsInfo->u.Flags.SessionSpace == 0) {
            CurrentProcess = PsGetCurrentProcess();
    
            if (AllocatedPde != NULL) {
                ASSERT (AllocatedPde->u.Hard.Valid == 1);
                TempVa = MiGetVirtualAddressMappedByPte(AllocatedPde);
                LOCK_PFN (OldIrql);
                MiDeletePte (AllocatedPde,
                             TempVa,
                             FALSE,
                             CurrentProcess,
                             NULL,
                             NULL);
                //
                // Add back in the private page MiDeletePte subtracted.
                //

                CurrentProcess->NumberOfPrivatePages += 1;
                UNLOCK_PFN (OldIrql);
            }
    
            if (AllocatedPpe != NULL) {
                ASSERT (AllocatedPpe->u.Hard.Valid == 1);
                TempVa = MiGetVirtualAddressMappedByPte(AllocatedPpe);
                LOCK_PFN (OldIrql);
                MiDeletePte (AllocatedPpe,
                             TempVa,
                             FALSE,
                             CurrentProcess,
                             NULL,
                             NULL);
                //
                // Add back in the private page MiDeletePte subtracted.
                //

                CurrentProcess->NumberOfPrivatePages += 1;
                UNLOCK_PFN (OldIrql);
            }
        }

        if (PointerPte == StartPte) {
            if (OriginalTable == NULL) {
                WorkingSetList->HashTableSize = First;
            }
        }

        return;
    }
#endif

    NewSize = (ULONG)((PointerPte - StartPte) << PAGE_SHIFT);

    ASSERT ((MiGetVirtualAddressMappedByPte(PointerPte) == WorkingSetList->HighestPermittedHashAddress) ||
            (PointerPte->u.Hard.Valid == 0));

    WorkingSetList->HashTableSize = First + NewSize / sizeof (MMWSLE_HASH);
    WorkingSetList->HashTable = Table;

    ASSERT ((&Table[WorkingSetList->HashTableSize] == WorkingSetList->HighestPermittedHashAddress) ||
        (MiGetPteAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0));

    if (First != 0) {
        RtlZeroMemory (Table, First * sizeof(MMWSLE_HASH));
    }

    //
    // Fill hash table
    //

    j = 0;
    Count = WorkingSetList->NonDirectCount;

    Size = WorkingSetList->HashTableSize;

    do {
        if ((Wsle[j].u1.e1.Valid == 1) &&
            (Wsle[j].u1.e1.Direct == 0)) {

            //
            // Hash this.
            //

            Count -= 1;

            Hash = MI_WSLE_HASH(Wsle[j].u1.Long, WorkingSetList);

            while (Table[Hash].Key != 0) {
                Hash += 1;
                if (Hash >= (ULONG)Size) {
                    Hash = 0;
                }
            }

            Table[Hash].Key = Wsle[j].u1.Long & ~(PAGE_SIZE - 1);
            Table[Hash].Index = j;
#if DBG
            PointerPte = MiGetPteAddress(Wsle[j].u1.VirtualAddress);
            ASSERT (PointerPte->u.Hard.Valid);
#endif //DBG

        }
        ASSERT (j <= WorkingSetList->LastEntry);
        j += 1;
    } while (Count);

#if DBG
    MiCheckWsleHash (WorkingSetList);
#endif //DBG
    return;
}


ULONG
MiTrimWorkingSet (
    ULONG Reduction,
    IN PMMSUPPORT WsInfo,
    IN ULONG ForcedReductionOrTrimAge
    )

/*++

Routine Description:

    This function reduces the working set by the specified amount.

Arguments:

    Reduction - Supplies the number of pages to remove from the working
                set.

    WsInfo - Supplies a pointer to the working set information for the
             process (or system cache) to trim.

    ForcedReductionOrTrimAge - If using fault-based trimming, this is set to
                               TRUE if the reduction is being done to free up
                               pages in which case we should try to reduce
                               working set pages as well.  Set to FALSE when
                               the reduction is trying to increase the fault
                               rates in which case the policy should be more
                               like locate and reserve.

                               If using claim-based trimming, this is the age
                               value to use - ie: pages of this age or older
                               will be removed.

Return Value:

    Returns the actual number of pages removed.

Environment:

    Kernel mode, APCs disabled, working set lock.  PFN lock NOT held.

--*/

{
    ULONG TryToFree;
    ULONG StartEntry;
    ULONG LastEntry;
    PMMWSL WorkingSetList;
    PMMWSLE Wsle;
    PMMPTE PointerPte;
    ULONG NumberLeftToRemove;
    ULONG LoopCount;
    ULONG EndCount;
    BOOLEAN StartFromZero;

    NumberLeftToRemove = Reduction;
    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

#if DBG
    if (WsInfo == &MmSystemCacheWs) {
        MM_SYSTEM_WS_LOCK_ASSERT();
    }
#endif //DBG

    LastEntry = WorkingSetList->LastEntry;

    TryToFree = WorkingSetList->NextSlot;
    if (TryToFree > LastEntry || TryToFree < WorkingSetList->FirstDynamic) {
        TryToFree = WorkingSetList->FirstDynamic;
    }

    StartEntry = TryToFree;

#ifdef _MI_USE_CLAIMS_

    while (NumberLeftToRemove != 0) {
        if (Wsle[TryToFree].u1.e1.Valid == 1) {
            PointerPte = MiGetPteAddress (Wsle[TryToFree].u1.VirtualAddress);

            if ((ForcedReductionOrTrimAge == 0) ||
                ((MI_GET_ACCESSED_IN_PTE (PointerPte) == 0) &&
                (MI_GET_WSLE_AGE(PointerPte, &Wsle[TryToFree]) >= ForcedReductionOrTrimAge))) {

                PERFINFO_GET_PAGE_INFO_WITH_DECL(PointerPte);

                if (MiFreeWsle (TryToFree, WsInfo, PointerPte)) {
                    PERFINFO_LOG_WS_REMOVAL(PERFINFO_LOG_TYPE_OUTWS_VOLUNTRIM, WsInfo);
                    NumberLeftToRemove -= 1;
                }
            }
        }
        TryToFree += 1;

        if (TryToFree > LastEntry) {
            TryToFree = WorkingSetList->FirstDynamic;
        }

        if (TryToFree == StartEntry) {
            break;
        }
    }

#else

    LoopCount = 0;

    if (ForcedReductionOrTrimAge) {
        EndCount = 4;
    } else {
        EndCount = 1;
    }

    StartFromZero = FALSE;

    while (NumberLeftToRemove != 0 && LoopCount != EndCount) {
        while ((NumberLeftToRemove != 0) && (TryToFree <= LastEntry)) {

            if (Wsle[TryToFree].u1.e1.Valid == 1) {
                PointerPte = MiGetPteAddress (Wsle[TryToFree].u1.VirtualAddress);
                if (MI_GET_ACCESSED_IN_PTE (PointerPte)) {

                    //
                    // If accessed bit is set, clear it.  If accessed
                    // bit is clear, remove from working set.
                    //

                    MI_SET_ACCESSED_IN_PTE (PointerPte, 0);
                } else {
                    PERFINFO_GET_PAGE_INFO_WITH_DECL(PointerPte);
                    if (MiFreeWsle (TryToFree, WsInfo, PointerPte)) {
                        PERFINFO_LOG_WS_REMOVAL(PERFINFO_LOG_TYPE_OUTWS_VOLUNTRIM, WsInfo);
                        NumberLeftToRemove -= 1;
                    }
                }
            }
            TryToFree += 1;

            if (StartFromZero == TRUE && EndCount == 1 && TryToFree >= StartEntry) {
                LoopCount = EndCount;
                if (TryToFree > LastEntry) {
                    TryToFree = WorkingSetList->FirstDynamic;
                }
                break;
            }
        }

        if (TryToFree > LastEntry) {
            TryToFree = WorkingSetList->FirstDynamic;
            if (StartFromZero == TRUE) {

                //
                // We've already wrapped once but didn't get back to
                // the StartEntry.  Use the first dynamic as our base now.
                //

                StartEntry = TryToFree;
            }
            else {
                StartFromZero = TRUE;
            }

            if (TryToFree >= StartEntry) {

                //
                // We've wrapped. If this is not a forced trim, then bail
                // now so we don't cannibalize entries we just cleared the
                // access bit for because they haven't had a fair chance to
                // be re-accessed yet.
                //

                LoopCount += 1;
                StartFromZero = FALSE;
            }
        }
    }

#endif

    WorkingSetList->NextSlot = TryToFree;

    //
    // If this is not the system cache or a session working set, see if the
    // working set list can be contracted.
    //

    if (WsInfo != &MmSystemCacheWs && WsInfo->u.Flags.SessionSpace == 0) {

        //
        // Make sure we are at least a page above the working set maximum.
        //

        if (WorkingSetList->FirstDynamic == WsInfo->WorkingSetSize) {
                MiRemoveWorkingSetPages (WorkingSetList, WsInfo);
        } else {

            if ((WorkingSetList->Quota + 15 + (PAGE_SIZE / sizeof(MMWSLE))) <
                                                    WorkingSetList->LastEntry) {
                if ((WsInfo->MaximumWorkingSetSize + 15 + (PAGE_SIZE / sizeof(MMWSLE))) <
                     WorkingSetList->LastEntry ) {
                    MiRemoveWorkingSetPages (WorkingSetList, WsInfo);
                }
            }
        }
    }
    return Reduction - NumberLeftToRemove;
}

#if 0 //COMMENTED OUT.
VOID
MmPurgeWorkingSet (
     IN PEPROCESS Process,
     IN PVOID BaseAddress,
     IN ULONG RegionSize
     )

/*++

Routine Description:

    This function removes any valid pages with a reference count
    of 1 within the specified address range of the specified process.

    If the address range is within the system cache, the process
    parameter is ignored.

Arguments:

    Process - Supplies a pointer to the process to operate upon.

    BaseAddress - Supplies the base address of the range to operate upon.

    RegionSize - Supplies the size of the region to operate upon.

Return Value:

    None.

Environment:

    Kernel mode, APC_LEVEL or below.

--*/

{
    PMMSUPPORT WsInfo;
    PMMPTE PointerPte;
    PMMPTE PointerPde;
    PMMPTE LastPte;
    PMMPFN Pfn1;
    MMPTE PteContents;
    PEPROCESS CurrentProcess;
    PVOID EndingAddress;
    ULONG SystemCache;
    KIRQL OldIrql;

    //
    // Determine if the specified base address is within the system
    // cache and if so, don't attach, the working set lock is still
    // required to "lock" paged pool pages (proto PTEs) into the
    // working set.
    //

    CurrentProcess = PsGetCurrentProcess ();

    ASSERT (RegionSize != 0);

    EndingAddress = (PVOID)((PCHAR)BaseAddress + RegionSize - 1);

    if ((BaseAddress <= MM_HIGHEST_USER_ADDRESS) ||
        ((BaseAddress >= (PVOID)PTE_BASE) &&
         (BaseAddress < (PVOID)MM_SYSTEM_SPACE_START)) ||
        ((BaseAddress >= MM_PAGED_POOL_START) &&
         (BaseAddress <= MmPagedPoolEnd))) {

        SystemCache = FALSE;

        //
        // Attach to the specified process.
        //

        KeAttachProcess (&Process->Pcb);

        WsInfo = &Process->Vm,

        LOCK_WS (Process);
    } else {

        SystemCache = TRUE;
        Process = CurrentProcess;
        WsInfo = &MmSystemCacheWs;
    }

    PointerPde = MiGetPdeAddress (BaseAddress);
    PointerPte = MiGetPteAddress (BaseAddress);
    LastPte = MiGetPteAddress (EndingAddress);

    while (!MiDoesPdeExistAndMakeValid(PointerPde, Process, FALSE)) {

        //
        // No page table page exists for this address.
        //

        PointerPde += 1;

        PointerPte = MiGetVirtualAddressMappedByPte (PointerPde);

        if (PointerPte > LastPte) {
            break;
        }
    }

    LOCK_PFN (OldIrql);

    while (PointerPte <= LastPte) {

        PteContents = *PointerPte;

        if (PteContents.u.Hard.Valid == 1) {

            //
            // Remove this page from the working set.
            //

            Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);

            if (Pfn1->u3.e2.ReferenceCount == 1) {
                MiRemovePageFromWorkingSet (PointerPte, Pfn1, WsInfo);
            }
        }

        PointerPte += 1;

        if (((ULONG_PTR)PointerPte & (PAGE_SIZE - 1)) == 0) {

            PointerPde = MiGetPteAddress (PointerPte);

            while ((PointerPte <= LastPte) &&
                   (!MiDoesPdeExistAndMakeValid(PointerPde, Process, TRUE))) {

                //
                // No page table page exists for this address.
                //

                PointerPde += 1;

                PointerPte = MiGetVirtualAddressMappedByPte (PointerPde);
            }
        }
    }

    UNLOCK_PFN (OldIrql);

    if (!SystemCache) {

        UNLOCK_WS (Process);
        KeDetachProcess();
    }
    return;
}
#endif //0

VOID
MiEliminateWorkingSetEntry (
    IN ULONG WorkingSetIndex,
    IN PMMPTE PointerPte,
    IN PMMPFN Pfn,
    IN PMMWSLE Wsle
    )

/*++

Routine Description:

    This routine removes the specified working set list entry
    from the working set, flushes the TB for the page, decrements
    the share count for the physical page, and, if necessary turns
    the PTE into a transition PTE.

Arguments:

    WorkingSetIndex - Supplies the working set index to remove.

    PointerPte - Supplies a pointer to the PTE corresponding to the virtual
                 address in the working set.

    Pfn - Supplies a pointer to the PFN element corresponding to the PTE.

    Wsle - Supplies a pointer to the first working set list entry for this
           working set.

Return Value:

    None.

Environment:

    Kernel mode, Working set lock and PFN lock held, APCs disabled.

--*/

{
    PMMPTE ContainingPageTablePage;
    MMPTE TempPte;
    MMPTE PreviousPte;
    PFN_NUMBER PageFrameIndex;
    PEPROCESS Process;
    PVOID VirtualAddress;

    //
    // Remove the page from the working set.
    //

    MM_PFN_LOCK_ASSERT ();

    TempPte = *PointerPte;
    PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (&TempPte);

#ifdef _X86_
#if DBG
#if !defined(NT_UP)
    if (TempPte.u.Hard.Writable == 1) {
        ASSERT (TempPte.u.Hard.Dirty == 1);
    }
    ASSERT (TempPte.u.Hard.Accessed == 1);
#endif //NTUP
#endif //DBG
#endif //X86

    MI_MAKING_VALID_PTE_INVALID (FALSE);

    if (Pfn->u3.e1.PrototypePte) {

        //
        // This is a prototype PTE.  The PFN database does not contain
        // the contents of this PTE it contains the contents of the
        // prototype PTE.  This PTE must be reconstructed to contain
        // a pointer to the prototype PTE.
        //
        // The working set list entry contains information about
        // how to reconstruct the PTE.
        //

        if (Wsle[WorkingSetIndex].u1.e1.SameProtectAsProto == 0) {

            //
            // The protection for the prototype PTE is in the
            // WSLE.
            //

            ASSERT (Wsle[WorkingSetIndex].u1.e1.Protection != 0);

            if (!MI_IS_SESSION_IMAGE_ADDRESS (MiGetVirtualAddressMappedByPte(PointerPte))) {
                TempPte.u.Long = 0;
                TempPte.u.Soft.Protection =
                    MI_GET_PROTECTION_FROM_WSLE (&Wsle[WorkingSetIndex]);
                TempPte.u.Soft.PageFileHigh = MI_PTE_LOOKUP_NEEDED;
            }
            else {

                //
                // The session PTE protection must be carefully preserved.
                //
    
                TempPte.u.Long = MiProtoAddressForPte (Pfn->PteAddress);
            }
        } else {

            //
            // The protection is in the prototype PTE.
            //

            TempPte.u.Long = MiProtoAddressForPte (Pfn->PteAddress);

            if (MI_IS_SESSION_IMAGE_ADDRESS (MiGetVirtualAddressMappedByPte(PointerPte))) {
    
                //
                // The session PTE protection must be carefully preserved.
                //
    
                TempPte.u.Proto.ReadOnly = 1;
            }
        }
    
        TempPte.u.Proto.Prototype = 1;

        //
        // Decrement the share count of the containing page table
        // page as the PTE for the removed page is no longer valid
        // or in transition
        //

        ContainingPageTablePage = MiGetPteAddress (PointerPte);
#if defined (_WIN64)
        ASSERT (ContainingPageTablePage->u.Hard.Valid == 1);
#else
        if (ContainingPageTablePage->u.Hard.Valid == 0) {
            if (!NT_SUCCESS(MiCheckPdeForPagedPool (PointerPte))) {
                KeBugCheckEx (MEMORY_MANAGEMENT,
                              0x61940, 
                              (ULONG_PTR)PointerPte,
                              (ULONG_PTR)ContainingPageTablePage->u.Long,
                              (ULONG_PTR)MiGetVirtualAddressMappedByPte(PointerPte));
            }
        }
#endif
        MiDecrementShareAndValidCount (MI_GET_PAGE_FRAME_FROM_PTE (ContainingPageTablePage));

    } else {

        //
        // This is a private page, make it transition.
        //

        //
        // Assert that the share count is 1 for all user mode pages.
        //

        ASSERT ((Pfn->u2.ShareCount == 1) ||
                (Wsle[WorkingSetIndex].u1.VirtualAddress >
                        (PVOID)MM_HIGHEST_USER_ADDRESS));

        //
        // Set the working set index to zero.  This allows page table
        // pages to be brought back in with the proper WSINDEX.
        //

        ASSERT (Pfn->u1.WsIndex != 0);
        MI_ZERO_WSINDEX (Pfn);
        MI_MAKE_VALID_PTE_TRANSITION (TempPte,
                                      Pfn->OriginalPte.u.Soft.Protection);
    }

    if (Wsle == MmWsle) {
        PreviousPte.u.Flush = KeFlushSingleTb (
                                    Wsle[WorkingSetIndex].u1.VirtualAddress,
                                    TRUE,
                                    FALSE,
                                    (PHARDWARE_PTE)PointerPte,
                                    TempPte.u.Flush);
    }
    else if (Wsle == MmSystemCacheWsle) {

        //
        // Must be the system cache.
        //

        PreviousPte.u.Flush = KeFlushSingleTb (
                                    Wsle[WorkingSetIndex].u1.VirtualAddress,
                                    TRUE,
                                    TRUE,
                                    (PHARDWARE_PTE)PointerPte,
                                    TempPte.u.Flush);
    }
    else {

        //
        // Must be a session space.
        //

        MI_FLUSH_SINGLE_SESSION_TB (Wsle[WorkingSetIndex].u1.VirtualAddress,
                                    TRUE,
                                    FALSE,
                                    (PHARDWARE_PTE)PointerPte,
                                    TempPte.u.Flush,
                                    PreviousPte);
    }

    ASSERT (PreviousPte.u.Hard.Valid == 1);

    //
    // A page is being removed from the working set, on certain
    // hardware the dirty bit should be ORed into the modify bit in
    // the PFN element.
    //

    MI_CAPTURE_DIRTY_BIT_TO_PFN (&PreviousPte, Pfn);

    //
    // If the PTE indicates the page has been modified (this is different
    // from the PFN indicating this), then ripple it back to the write watch
    // bitmap now since we are still in the correct process context.
    //

    if (MiActiveWriteWatch != 0) {
        if ((Pfn->u3.e1.PrototypePte == 0) &&
            (MI_IS_PTE_DIRTY(PreviousPte))) {

            Process = PsGetCurrentProcess();

            if (Process->Vm.u.Flags.WriteWatch == 1) {

                //
                // This process has (or had) write watch VADs.  Search now
                // for a write watch region encapsulating the PTE being
                // invalidated.
                //

                VirtualAddress = MiGetVirtualAddressMappedByPte (PointerPte);
                MiCaptureWriteWatchDirtyBit (Process, VirtualAddress);
            }
        }
    }

    //
    // Flush the translation buffer and decrement the number of valid
    // PTEs within the containing page table page.  Note that for a
    // private page, the page table page is still needed because the
    // page is in transition.
    //

    MiDecrementShareCount (PageFrameIndex);

    return;
}

VOID
MiRemoveWorkingSetPages (
    IN PMMWSL WorkingSetList,
    IN PMMSUPPORT WsInfo
    )

/*++

Routine Description:

    This routine compresses the WSLEs into the front of the working set
    and frees the pages for unneeded working set entries.

Arguments:

    WorkingSetList - Supplies a pointer to the working set list to compress.

Return Value:

    None.

Environment:

    Kernel mode, Working set lock held, APCs disabled.

--*/

{
    PMMWSLE FreeEntry;
    PMMWSLE LastEntry;
    PMMWSLE Wsle;
    ULONG FreeIndex;
    ULONG LastIndex;
    ULONG LastInvalid;
    PMMPTE LastPte;
    PMMPTE PointerPte;
    PMMPTE PointerPde;
    PMMPTE PointerPpe;
    PMMPTE WsPte;
    PMMPFN Pfn1;
    PEPROCESS CurrentProcess;
    MMPTE_FLUSH_LIST PteFlushList;
    ULONG NewSize;
    PMMWSLE_HASH Table;
    KIRQL OldIrql;

    ASSERT (WsInfo != &MmSystemCacheWs && WsInfo->u.Flags.SessionSpace == 0);

    PteFlushList.Count = 0;
    CurrentProcess = PsGetCurrentProcess();

#if DBG
    MiCheckNullIndex (WorkingSetList);
#endif //DBG

    //
    // Check to see if the wsle hash table should be contracted.
    //

    if (WorkingSetList->HashTable) {

        Table = WorkingSetList->HashTable;

#if DBG
        if ((PVOID)(&Table[WorkingSetList->HashTableSize]) < WorkingSetList->HighestPermittedHashAddress) {
            ASSERT (MiGetPteAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0);
        }
#endif

        if (WsInfo->WorkingSetSize < 200) {
            NewSize = 0;
        }
        else {
            NewSize = PtrToUlong(PAGE_ALIGN ((WorkingSetList->NonDirectCount * 2 *
                                       sizeof(MMWSLE_HASH)) + PAGE_SIZE - 1));
    
            NewSize = NewSize / sizeof(MMWSLE_HASH);
        }

        if (NewSize < WorkingSetList->HashTableSize) {

#if defined(_ALPHA_) && !defined(_AXP64_)
            LOCK_EXPANSION_IF_ALPHA (OldIrql);
#endif
            if (NewSize && WsInfo->AllowWorkingSetAdjustment) {
                WsInfo->AllowWorkingSetAdjustment = MM_GROW_WSLE_HASH;
            }
#if defined(_ALPHA_) && !defined(_AXP64_)
            UNLOCK_EXPANSION_IF_ALPHA (OldIrql);
#endif

            //
            // Remove pages from hash table.
            //

            ASSERT (((ULONG_PTR)&WorkingSetList->HashTable[NewSize] &
                                                    (PAGE_SIZE - 1)) == 0);

            PointerPte = MiGetPteAddress (&WorkingSetList->HashTable[NewSize]);

            LastPte = MiGetPteAddress (WorkingSetList->HighestPermittedHashAddress);
            //
            // Set the hash table to null indicating that no hashing
            // is going on.
            //

            WorkingSetList->HashTable = NULL;
            WorkingSetList->HashTableSize = NewSize;

            LOCK_PFN (OldIrql);
            while ((PointerPte < LastPte) && (PointerPte->u.Hard.Valid == 1)) {

                MiDeletePte (PointerPte,
                             MiGetVirtualAddressMappedByPte (PointerPte),
                             FALSE,
                             CurrentProcess,
                             NULL,
                             &PteFlushList);

                //
                // Add back in the private page MiDeletePte subtracted.
                //

                CurrentProcess->NumberOfPrivatePages += 1;

                PointerPte += 1;

#if defined (_WIN64)
                //
                // If all the entries have been removed from the previous page
                // table page, delete the page table page itself.  Likewise with
                // the page directory page.
                //
    
                if ((MiIsPteOnPdeBoundary(PointerPte)) ||
                    ((MiGetPdeAddress(PointerPte))->u.Hard.Valid == 0) ||
                    ((MiGetPteAddress(PointerPte))->u.Hard.Valid == 0) ||
                    (PointerPte->u.Hard.Valid == 0)) {
    
                    MiFlushPteList (&PteFlushList, FALSE, ZeroPte);

                    PointerPde = MiGetPteAddress (PointerPte - 1);
    
                    ASSERT (PointerPde->u.Hard.Valid == 1);
    
                    Pfn1 = MI_PFN_ELEMENT (MI_GET_PAGE_FRAME_FROM_PTE (PointerPde));
    
                    if (Pfn1->u2.ShareCount == 1 && Pfn1->u3.e2.ReferenceCount == 1)
                    {
                        MiDeletePte (PointerPde,
                                     PointerPte - 1,
                                     FALSE,
                                     CurrentProcess,
                                     NULL,
                                     NULL);

                        //
                        // Add back in the private page MiDeletePte subtracted.
                        //
        
                        CurrentProcess->NumberOfPrivatePages += 1;
                    }
                
                    if (MiIsPteOnPpeBoundary(PointerPte)) {
        
                        PointerPpe = MiGetPteAddress (PointerPde);
        
                        ASSERT (PointerPpe->u.Hard.Valid == 1);
        
                        Pfn1 = MI_PFN_ELEMENT (MI_GET_PAGE_FRAME_FROM_PTE (PointerPpe));
        
                        if (Pfn1->u2.ShareCount == 1 && Pfn1->u3.e2.ReferenceCount == 1)
                        {
                            MiDeletePte (PointerPpe,
                                         PointerPde,
                                         FALSE,
                                         CurrentProcess,
                                         NULL,
                                         NULL);

                            //
                            // Add back in the private page MiDeletePte subtracted.
                            //
            
                            CurrentProcess->NumberOfPrivatePages += 1;
                        }
                    }
                    PointerPde = MiGetPteAddress (PointerPte);
                    PointerPpe = MiGetPdeAddress (PointerPte);
                    if ((PointerPpe->u.Hard.Valid == 0) ||
                        (PointerPde->u.Hard.Valid == 0)) {
                            break;
                    }
                }
#endif
            }
            MiFlushPteList (&PteFlushList, FALSE, ZeroPte);

            if (WsInfo->u.Flags.SessionSpace == 1) {
        
                //
                // Session space has no ASN - flush the entire TB.
                //
            
                MI_FLUSH_ENTIRE_SESSION_TB (TRUE, TRUE);
            }
        
            UNLOCK_PFN (OldIrql);
        }
#if defined (_WIN64)

        //
        // For 64-bit NT, the page tables and page directories are also
        // deleted during contraction.
        //

        ASSERT ((MiGetPpeAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0) ||
                (MiGetPdeAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0) ||
                (MiGetPteAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0));

#else

        ASSERT (MiGetPteAddress(&Table[WorkingSetList->HashTableSize])->u.Hard.Valid == 0);

#endif
    }

    //
    // If the only pages in the working set are locked pages (that
    // is all pages are BEFORE first dynamic, just reorganize the
    // free list).
    //

    Wsle = WorkingSetList->Wsle;
    if (WorkingSetList->FirstDynamic == WsInfo->WorkingSetSize) {

        LastIndex = WorkingSetList->FirstDynamic;
        LastEntry = &Wsle[LastIndex];

    } else {

        //
        // Start from the first dynamic and move towards the end looking
        // for free entries.  At the same time start from the end and
        // move towards first dynamic looking for valid entries.
        //

        LastInvalid = 0;
        FreeIndex = WorkingSetList->FirstDynamic;
        FreeEntry = &Wsle[FreeIndex];
        LastIndex = WorkingSetList->LastEntry;
        LastEntry = &Wsle[LastIndex];

        while (FreeEntry < LastEntry) {
            if (FreeEntry->u1.e1.Valid == 1) {
                FreeEntry += 1;
                FreeIndex += 1;
            } else if (LastEntry->u1.e1.Valid == 0) {
                LastEntry -= 1;
                LastIndex -= 1;
            } else {

                //
                // Move the WSLE at LastEntry to the free slot at FreeEntry.
                //

                LastInvalid = 1;
                *FreeEntry = *LastEntry;
                PointerPte = MiGetPteAddress (LastEntry->u1.VirtualAddress);

                if (LastEntry->u1.e1.Direct) {

                    Pfn1 = MI_PFN_ELEMENT (PointerPte->u.Hard.PageFrameNumber);

                    CONSISTENCY_LOCK_PFN (OldIrql);

                    Pfn1->u1.WsIndex = FreeIndex;

                    CONSISTENCY_UNLOCK_PFN (OldIrql);

                } else {

                    //
                    // This entry is in the working set.  Remove it
                    // and then add the entry add the free slot.
                    //

                    MiRemoveWsle (LastIndex, WorkingSetList);
                    MiInsertWsle (FreeIndex, WorkingSetList);
                }

                MI_SET_PTE_IN_WORKING_SET (PointerPte, FreeIndex);
                LastEntry->u1.Long = 0;
                LastEntry -= 1;
                LastIndex -= 1;
                FreeEntry += 1;
                FreeIndex += 1;
            }
        }

        //
        // If no entries were freed, just return.
        //

        if (LastInvalid == 0) {
#if DBG
            MiCheckNullIndex (WorkingSetList);
#endif //DBG
            return;
        }
    }

    //
    // Reorganize the free list.  Make last entry the first free.
    //

    ASSERT ((LastEntry - 1)->u1.e1.Valid == 1);

    if (LastEntry->u1.e1.Valid == 1) {
        LastEntry += 1;
        LastIndex += 1;
    }

    WorkingSetList->LastEntry = LastIndex - 1;
    WorkingSetList->FirstFree = LastIndex;

    ASSERT ((LastEntry - 1)->u1.e1.Valid == 1);
    ASSERT ((LastEntry)->u1.e1.Valid == 0);

    //
    // Point free entry to the first invalid page.
    //

    FreeEntry = LastEntry;

    while (LastIndex < WorkingSetList->LastInitializedWsle) {

        //
        // Put the remainder of the WSLEs on the free list.
        //

        ASSERT (LastEntry->u1.e1.Valid == 0);
        LastIndex += 1;
        LastEntry->u1.Long = LastIndex << MM_FREE_WSLE_SHIFT;
        LastEntry += 1;
    }

    //LastEntry->u1.Long = WSLE_NULL_INDEX << MM_FREE_WSLE_SHIFT;  // End of list.

    //
    // Delete the working set pages at the end.
    //

    PointerPte = MiGetPteAddress (&Wsle[WorkingSetList->LastInitializedWsle]);
    if (&Wsle[WsInfo->MinimumWorkingSetSize] > FreeEntry) {
        FreeEntry = &Wsle[WsInfo->MinimumWorkingSetSize];
    }

    WsPte = MiGetPteAddress (FreeEntry);

#if 0
    if (MiGetPteAddress (FreeEntry) != MiGetPteAddress (FreeEntry + 1)) {
        DbgPrint ("MiRemoveWorkingSetPages caught boundary case %x\n", FreeEntry);
        DbgBreakPoint();

        WsPte = MiGetPteAddress (FreeEntry + 1);
    }
#endif

    ASSERT (WorkingSetList->FirstFree >= WorkingSetList->FirstDynamic);

    LOCK_PFN (OldIrql);
    while (PointerPte > WsPte) {
        ASSERT (PointerPte->u.Hard.Valid == 1);

        MiDeletePte (PointerPte,
                     MiGetVirtualAddressMappedByPte (PointerPte),
                     FALSE,
                     CurrentProcess,
                     NULL,
                     &PteFlushList);

        //
        // Add back in the private page MiDeletePte subtracted.
        //

        CurrentProcess->NumberOfPrivatePages += 1;

#if defined (_WIN64)
        //
        // If all the entries have been removed from the previous page
        // table page, delete the page table page itself.  Likewise with
        // the page directory page.
        //

        if (MiIsPteOnPdeBoundary(PointerPte)) {

            PointerPde = MiGetPteAddress (PointerPte);

            ASSERT (PointerPde->u.Hard.Valid == 1);

            Pfn1 = MI_PFN_ELEMENT (MI_GET_PAGE_FRAME_FROM_PTE (PointerPde));

            if (Pfn1->u2.ShareCount == 1 && Pfn1->u3.e2.ReferenceCount == 1) {

                MiFlushPteList (&PteFlushList, FALSE, ZeroPte);

                MiDeletePte (PointerPde,
                             PointerPte,
                             FALSE,
                             CurrentProcess,
                             NULL,
                             NULL);

                //
                // Add back in the private page MiDeletePte subtracted.
                //

                CurrentProcess->NumberOfPrivatePages += 1;
            }
        
            if (MiIsPteOnPpeBoundary(PointerPte)) {

                PointerPpe = MiGetPteAddress (PointerPde);

                ASSERT (PointerPpe->u.Hard.Valid == 1);

                Pfn1 = MI_PFN_ELEMENT (MI_GET_PAGE_FRAME_FROM_PTE (PointerPpe));

                if (Pfn1->u2.ShareCount == 1 && Pfn1->u3.e2.ReferenceCount == 1)
                {

                    MiFlushPteList (&PteFlushList, FALSE, ZeroPte);

                    MiDeletePte (PointerPpe,
                                 PointerPde,
                                 FALSE,
                                 CurrentProcess,
                                 NULL,
                                 NULL);

                    //
                    // Add back in the private page MiDeletePte subtracted.
                    //
    
                    CurrentProcess->NumberOfPrivatePages += 1;
                }
            }
        }
#endif

        PointerPte -= 1;
    }

    MiFlushPteList (&PteFlushList, FALSE, ZeroPte);

    if (WsInfo->u.Flags.SessionSpace == 1) {

        //
        // Session space has no ASN - flush the entire TB.
        //
    
        MI_FLUSH_ENTIRE_SESSION_TB (TRUE, TRUE);
    }

    UNLOCK_PFN (OldIrql);

    ASSERT (WorkingSetList->FirstFree >= WorkingSetList->FirstDynamic);

    //
    // Mark the last PTE in the list as free.
    //

    LastEntry = (PMMWSLE)((PCHAR)(PAGE_ALIGN(FreeEntry)) + PAGE_SIZE);
    LastEntry -= 1;

    ASSERT (LastEntry->u1.e1.Valid == 0);
    LastEntry->u1.Long = WSLE_NULL_INDEX << MM_FREE_WSLE_SHIFT; //End of List.
    ASSERT (LastEntry > &Wsle[0]);
    WorkingSetList->LastInitializedWsle = (WSLE_NUMBER)(LastEntry - &Wsle[0]);
    WorkingSetList->NextSlot = WorkingSetList->FirstDynamic;

    ASSERT (WorkingSetList->LastEntry <= WorkingSetList->LastInitializedWsle);

    if (WorkingSetList->Quota < WorkingSetList->LastInitializedWsle) {
        WorkingSetList->Quota = WorkingSetList->LastInitializedWsle;
    }

    ASSERT ((MiGetPteAddress(&Wsle[WorkingSetList->LastInitializedWsle]))->u.Hard.Valid == 1);
    ASSERT ((WorkingSetList->FirstFree <= WorkingSetList->LastInitializedWsle) ||
            (WorkingSetList->FirstFree == WSLE_NULL_INDEX));
#if DBG
    MiCheckNullIndex (WorkingSetList);
#endif //DBG
    return;
}


NTSTATUS
MiEmptyWorkingSet (
    IN PMMSUPPORT WsInfo,
    IN LOGICAL WaitOk
    )

/*++

Routine Description:

    This routine frees all pages from the working set.

Arguments:

    WsInfo - Supplies the working set information entry to trim.

    WaitOk - Supplies TRUE if the caller can wait, FALSE if not.

Return Value:

    Status of operation.

Environment:

    Kernel mode. No locks.  For session operations, the caller is responsible
    for attaching into the proper session.

--*/

{
    PEPROCESS Process;
    KIRQL OldIrql;
    PMMPTE PointerPte;
    ULONG Entry;
    ULONG Count;
    ULONG LastFreed;
    PMMWSL WorkingSetList;
    PMMWSLE Wsle;
    PMMPFN Pfn1;
    PFN_NUMBER PageFrameIndex;
    ULONG Last;
    NTSTATUS Status;
    PMM_SESSION_SPACE SessionSpace;

    if (WsInfo == &MmSystemCacheWs) {
        if (WaitOk == TRUE) {
            LOCK_SYSTEM_WS (OldIrql);
        }
        else {
            KeRaiseIrql (APC_LEVEL, &OldIrql);
            if (!ExTryToAcquireResourceExclusiveLite (&MmSystemWsLock)) {

                //
                // System working set lock was not granted, don't trim
                // the system cache.
                //

                KeLowerIrql (OldIrql);
                return STATUS_SUCCESS;
            }

            MmSystemLockOwner = PsGetCurrentThread();
        }
    }
    else if (WsInfo->u.Flags.SessionSpace == 0) {
        Process = PsGetCurrentProcess ();
        if (WaitOk == TRUE) {
            LOCK_WS (Process);
        }
        else {
            Count = 0;
            do {
                if (ExTryToAcquireFastMutex(&Process->WorkingSetLock) != FALSE) {
                    break;
                }
                KeDelayExecutionThread (KernelMode, FALSE, &MmShortTime);
                Count += 1;
                if (Count == 5) {

                    //
                    // Could not get the lock, don't trim this process.
                    //

                    return STATUS_SUCCESS;
                }
            } while (TRUE);
        }
        if (Process->AddressSpaceDeleted != 0) {
            Status = STATUS_PROCESS_IS_TERMINATING;
            goto Deleted;
        }
    }
    else {
        if (WaitOk == TRUE) {
            LOCK_SESSION_SPACE_WS (OldIrql);
        }
        else {
            ASSERT (MiHydra == TRUE);
            SessionSpace = CONTAINING_RECORD(WsInfo,
                                             MM_SESSION_SPACE,
                                             Vm);

            KeRaiseIrql (APC_LEVEL, &OldIrql);

            if (!ExTryToAcquireResourceExclusiveLite (&SessionSpace->WsLock)) {

                //
                // This session space's working set lock was not
                // granted, don't trim it.
                //

                KeLowerIrql (OldIrql);
                return STATUS_SUCCESS;
            }

            MM_SET_SESSION_RESOURCE_OWNER();
        }
    }

    WorkingSetList = WsInfo->VmWorkingSetList;
    Wsle = WorkingSetList->Wsle;

    //
    // Attempt to remove the pages starting at the bottom.
    //

    LastFreed = WorkingSetList->LastEntry;
    for (Entry = WorkingSetList->FirstDynamic; Entry <= LastFreed; Entry += 1) {

        if (Wsle[Entry].u1.e1.Valid != 0) {
            PERFINFO_PAGE_INFO_DECL();

            PointerPte = MiGetPteAddress (Wsle[Entry].u1.VirtualAddress);

            PERFINFO_GET_PAGE_INFO(PointerPte);

            if (MiTrimRemovalPagesOnly == TRUE) {
                PageFrameIndex = MI_GET_PAGE_FRAME_FROM_PTE (PointerPte);
                Pfn1 = MI_PFN_ELEMENT (PageFrameIndex);
                if (Pfn1->u3.e1.RemovalRequested == 0) {
                    Pfn1 = MI_PFN_ELEMENT (Pfn1->PteFrame);
                    if (Pfn1->u3.e1.RemovalRequested == 0) {
#if defined (_WIN64)
                        Pfn1 = MI_PFN_ELEMENT (Pfn1->PteFrame);
                        if (Pfn1->u3.e1.RemovalRequested == 0) {
                            continue;
                        }
#else
                        continue;
#endif
                    }
                }
            }

            if (MiFreeWsle (Entry, WsInfo, PointerPte)) {
                PERFINFO_LOG_WS_REMOVAL(PERFINFO_LOG_TYPE_OUTWS_EMPTYQ, WsInfo);
            }
        }
    }

    if (WsInfo != &MmSystemCacheWs && WsInfo->u.Flags.SessionSpace == 0) {
        MiRemoveWorkingSetPages (WorkingSetList,WsInfo);
    }
    WorkingSetList->Quota = WsInfo->WorkingSetSize;
    WorkingSetList->NextSlot = WorkingSetList->FirstDynamic;

    //
    // Attempt to remove the pages from the front to the end.
    //

    //
    // Reorder the free list.
    //

    Last = 0;
    Entry = WorkingSetList->FirstDynamic;
    LastFreed = WorkingSetList->LastInitializedWsle;
    while (Entry <= LastFreed) {
        if (Wsle[Entry].u1.e1.Valid == 0) {
            if (Last == 0) {
                WorkingSetList->FirstFree = Entry;
            } else {
                Wsle[Last].u1.Long = Entry << MM_FREE_WSLE_SHIFT;
            }
            Last = Entry;
        }
        Entry += 1;
    }
    if (Last != 0) {
        Wsle[Last].u1.Long = WSLE_NULL_INDEX << MM_FREE_WSLE_SHIFT;  // End of list.
    }
    Status = STATUS_SUCCESS;
Deleted:

    if (WsInfo == &MmSystemCacheWs) {
        UNLOCK_SYSTEM_WS (OldIrql);
    }
    else if (WsInfo->u.Flags.SessionSpace == 0) {
        UNLOCK_WS (Process);
    }
    else {
        UNLOCK_SESSION_SPACE_WS (OldIrql);
    }

    return Status;
}

#if 0

#define x256k_pte_mask (((256*1024) >> (PAGE_SHIFT - PTE_SHIFT)) - (sizeof(MMPTE)))

VOID
MiDumpWsleInCacheBlock (
    IN PMMPTE CachePte
    )

/*++

Routine Description:

    The routine checks the prototype PTEs adjacent to the supplied
    PTE and if they are modified, in the system cache working set,
    and have a reference count of 1, removes it from the system
    cache working set.

Arguments:

    CachePte - Supplies a pointer to the cache PTE.

Return Value:

    None.

Environment:

    Kernel mode, Working set lock and PFN lock held, APCs disabled.

--*/

{
    PMMPTE LoopPte;
    PMMPTE PointerPte;

    LoopPte = (PMMPTE)((ULONG_PTR)CachePte & ~x256k_pte_mask);
    PointerPte = CachePte - 1;

    while (PointerPte >= LoopPte ) {

        if (MiDumpPteInCacheBlock (PointerPte) == FALSE) {
            break;
        }
        PointerPte -= 1;
    }

    PointerPte = CachePte + 1;
    LoopPte = (PMMPTE)((ULONG_PTR)CachePte | x256k_pte_mask);

    while (PointerPte <= LoopPte ) {

        if (MiDumpPteInCacheBlock (PointerPte) == FALSE) {
            break;
        }
        PointerPte += 1;
    }
    return;
}

ULONG
MiDumpPteInCacheBlock (
    IN PMMPTE PointerPte
    )

{
    PMMPFN Pfn1;
    MMPTE PteContents;
    ULONG WorkingSetIndex;

    PteContents = *PointerPte;

    if (PteContents.u.Hard.Valid == 1) {

        Pfn1 = MI_PFN_ELEMENT (PteContents.u.Hard.PageFrameNumber);

        //
        // If the PTE is valid and dirty (or pfn indicates dirty)
        // and the Wsle is direct index via the pfn wsindex element
        // and the reference count is one, then remove this page from
        // the cache manager's working set list.
        //

        if ((Pfn1->u3.e2.ReferenceCount == 1) &&
            ((Pfn1->u3.e1.Modified == 1) ||
                (MI_IS_PTE_DIRTY (PteContents))) &&
                (MiGetPteAddress (
                    MmSystemCacheWsle[Pfn1->u1.WsIndex].u1.VirtualAddress) ==
                    PointerPte)) {

            //
            // Found a candidate, remove the page from the working set.
            //

            WorkingSetIndex = Pfn1->u1.WsIndex;
            LOCK_PFN (OldIrql);
            MiEliminateWorkingSetEntry (WorkingSetIndex,
                                        PointerPte,
                                        Pfn1,
                                        MmSystemCacheWsle);
            UNLOCK_PFN (OldIrql);

            //
            // Remove the working set entry from the working set.
            //

            MiRemoveWsle (WorkingSetIndex, MmSystemCacheWorkingSetList);

            //
            // Put the entry on the free list and decrement the current
            // size.
            //

            MmSystemCacheWsle[WorkingSetIndex].u1.Long =
                  MmSystemCacheWorkingSetList->FirstFree << MM_FREE_WSLE_SHIFT;
            MmSystemCacheWorkingSetList->FirstFree = WorkingSetIndex;

            if (MmSystemCacheWs.WorkingSetSize > MmSystemCacheWs.MinimumWorkingSetSize) {
                MmPagesAboveWsMinimum -= 1;
            }
            MmSystemCacheWs.WorkingSetSize -= 1;
            return TRUE;
        }
    }
    return FALSE;
}
#endif //0

#if DBG
VOID
MiCheckNullIndex (
    IN PMMWSL WorkingSetList
    )

{
    PMMWSLE Wsle;
    ULONG j;
    ULONG Nulls = 0;

    Wsle = WorkingSetList->Wsle;
    for (j = 0;j <= WorkingSetList->LastInitializedWsle; j += 1) {
        if ((((Wsle[j].u1.Long)) >> MM_FREE_WSLE_SHIFT) == WSLE_NULL_INDEX) {
            Nulls += 1;
        }
    }
    ASSERT (Nulls == 1);
    return;
}

#endif //DBG

---