flush.c File Reference

#include "ki.h"
#include "kxia64.h"

Go to the source code of this file.

Functions
VOID	KiSweepDcacheTarget (IN PULONG SignalDone, IN PVOID Parameter1, IN PVOID Parameter2, IN PVOID Parameter3)
VOID	KiSweepIcacheTarget (IN PULONG SignalDone, IN PVOID Parameter1, IN PVOID Parameter2, IN PVOID Parameter3)
VOID	KiFlushIoBuffersTarget (IN PKIPI_CONTEXT SignalDone, IN PVOID Mdl, IN PVOID ReadOperation, IN PVOID DmaOperation)
VOID	KiSyncCacheTarget (IN PKIPI_CONTEXT SignalDone, IN PVOID Parameter1, IN PVOID Parameter2, IN PVOID Parameter3)
ULONG_PTR	KiSyncMC_DrainTarget ()
ULONG_PTR	KiSyncMC_Drain (IN BOOLEAN AllProcessors, IN PVOID BaseAddress, IN ULONG Length)
ULONG_PTR	KiSyncPrefetchVisibleTarget ()
ULONG_PTR	KiSyncPrefetchVisible (IN BOOLEAN AllProcessors, IN PVOID BaseAddress, IN ULONG Length)
VOID	KeSweepIcache (IN BOOLEAN AllProcessors)
VOID	KeSweepDcache (IN BOOLEAN AllProcessors)
VOID	KeSweepCacheRange (IN BOOLEAN AllProcessors, IN PVOID BaseAddress, IN ULONG Length)
VOID	KeSweepIcacheRange (IN BOOLEAN AllProcessors, IN PVOID BaseAddress, IN ULONG Length)
VOID	KeSweepDcacheRange (IN BOOLEAN AllProcessors, IN PVOID BaseAddress, IN ULONG Length)
VOID	KeSweepCacheRangeWithDrain (IN BOOLEAN AllProcessors, IN PVOID BaseAddress, IN ULONG Length)
Variables
ULONG	ProbePalVisibilitySupport = 1
ULONG	NeedPalVisibilitySupport = 1
KSPIN_LOCK	KiCacheFlushLock

---

Function Documentation

VOID KeSweepCacheRange (  IN BOOLEAN  AllProcessors, IN PVOID  BaseAddress, IN ULONG  Length )

Definition at line 453 of file ia64/flush.c. References ASSERT, KeActiveProcessors, KeLowerIrql(), KiIpiSendPacket(), KiIpiStallOnPacketTargets(), KiSyncCacheTarget(), and NULL. Referenced by KeSweepCacheRangeWithDrain(). : This function is used to flush a range of virtual addresses from both the instruction and data cache on all processors in the system. Irrespective of the length of the range, it should not call SweepIcache or SweepDcache. This is because SweepDcache will only sweep D cache and not the I cache and Vice versa. Since the caller of KeSweepCacheRange assumes both the caches are being swept, one cannot call SweepIcache or SweepDcache in trying to optimize. Arguments: AllProcessors - Not used BaseAddress - Supplies a pointer to the base of the range that is flushed. Length - Supplies the length of the range that is flushed if the base address is specified. Return Value: None. --*/ { KIRQL OldIrql; KAFFINITY TargetProcessors; // // We will not raise IRQL to synchronization level so that we can allow // a context switch in between Flush Cache. FC need not run in the same processor // throughout. It can be context switched. So no binding is done to any processor. // // HalSweepCacheRange(BaseAddress,Length); ASSERT(KeGetCurrentIrql() <= KiSynchIrql); // // Raise IRQL to synchronization level to prevent a context switch. // #if !defined(NT_UP) OldIrql = KeRaiseIrqlToSynchLevel(); // // Compute the set of target processors and send the sync parameters // to the target processors, if any, for execution. // TargetProcessors = KeActiveProcessors & PCR->NotMember; if (TargetProcessors != 0) { KiIpiSendPacket(TargetProcessors, KiSyncCacheTarget, NULL, NULL, NULL); } #endif // // Synchronize the Instruction Prefetch pipe in the local processor. // __synci(); __isrlz(); // // Wait until all target processors have finished sweeping the their // data cache. // #if !defined(NT_UP) if (TargetProcessors != 0) { KiIpiStallOnPacketTargets(TargetProcessors); } // // Lower IRQL to its previous level and return. // KeLowerIrql(OldIrql); #endif return; }

VOID KeSweepCacheRangeWithDrain (  IN BOOLEAN  AllProcessors, IN PVOID  BaseAddress, IN ULONG  Length )

Definition at line 979 of file ia64/flush.c. References ASSERT, KeSweepCacheRange(), KiSyncMC_Drain(), KiSyncPrefetchVisible(), and Status. Referenced by MiSweepCacheMachineDependent(). : This function is used to drain prefetches,demand references followed by flushing the cache followed by draining pending fc cache line evictions to a specified range address in all processors in the system. Arguments: AllProcessors - All processors in the system. BaseAddress - Supplies a pointer to the base of the range that is flushed and drained. Length - Supplies the length of the range that is flushed and drained for the base address is specified. Return Value: None. Note: This is used when changing attributes of WB pages to UC pages. --*/ { ULONG_PTR Status; Status = KiSyncPrefetchVisible( AllProcessors, BaseAddress, Length ); ASSERT(Status != PAL_STATUS_ERROR); KeSweepCacheRange ( AllProcessors, BaseAddress, Length ); Status = KiSyncMC_Drain ( AllProcessors, BaseAddress, Length ); ASSERT(Status == PAL_STATUS_SUCCESS); return; }

VOID KeSweepDcache (  IN BOOLEAN  AllProcessors  )

Definition at line 276 of file ia64/flush.c. References ASSERT, DISPATCH_LEVEL, KeAcquireSpinLock, KeActiveProcessors, KeReleaseSpinLock(), KiCacheFlushLock, KiIpiSendPacket(), KiIpiStallOnPacketTargets(), KiSweepDcacheTarget(), and NULL. 00282 : 00283 00284 This function flushes the data cache on all processors that are currently 00285 running threads which are children of the current process or flushes the 00286 data cache on all processors in the host configuration. 00287 00288 N.B. PowerPC maintains cache coherency across processors however 00289 in this routine, the range of addresses being flushed is unknown 00290 so we must still broadcast the request to the other processors. 00291 00292 Arguments: 00293 00294 AllProcessors - Supplies a boolean value that determines which data 00295 caches are flushed. 00296 00297 Return Value: 00298 00299 None. 00300 00301 --*/ 00302 00303 { 00304 00305 KIRQL OldIrql; 00306 KAFFINITY TargetProcessors; 00307 00308 ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL); 00309 00310 #if !defined(NT_UP) 00311 // 00312 // Acquire cache flush spinlock 00313 // Cache flush is not MP safe yet 00314 // 00315 KeAcquireSpinLock(&KiCacheFlushLock, &OldIrql); 00316 00317 #endif 00318 00319 HalSweepDcache(); 00320 00321 #if !defined(NT_UP) 00322 00323 // 00324 // Compute the set of target processors and send the sweep parameters 00325 // to the target processors, if any, for execution. 00326 // 00327 00328 TargetProcessors = KeActiveProcessors & PCR->NotMember; 00329 if (TargetProcessors != 0) { 00330 KiIpiSendPacket(TargetProcessors, 00331 KiSweepDcacheTarget, 00332 NULL, 00333 NULL, 00334 NULL); 00335 } 00336 00337 00338 // 00339 // Wait until all target processors have finished sweeping their 00340 // data caches. 00341 // 00342 00343 00344 if (TargetProcessors != 0) { 00345 KiIpiStallOnPacketTargets(TargetProcessors); 00346 } 00347 00348 // 00349 // Lower IRQL to its previous level and return. 00350 // 00351 00352 KeReleaseSpinLock(&KiCacheFlushLock, OldIrql); 00353 00354 #endif 00355 00356 return; 00357 }

VOID KeSweepDcacheRange (  IN BOOLEAN  AllProcessors, IN PVOID  BaseAddress, IN ULONG  Length )

Definition at line 674 of file ia64/flush.c. References ASSERT, KeActiveProcessors, KeLowerIrql(), KiIpiSendPacket(), KiIpiStallOnPacketTargets(), KiSyncCacheTarget(), and NULL. : This function is used to flush a range of virtual addresses from the primary data cache on all processors in the host configuration. If the length of the range is greater than the size of the data cache, then one can call HalSweepDcache which calls SAL to flush the entire cache. Since SAL does not take care of MP flushing, HalSweepDcache has to use IPI mechanism to execute SAL flush from each processor. We need to weight the overhead of all these versus using HalSweepDcacheRange and avoiding IPI mechanism since HalSweepDcacheRange uses fc instruction and fc instruction takes care of MP. Arguments: AllProcessors - Not used BaseAddress - Supplies a pointer to the base of the range that is flushed. Length - Supplies the length of the range that is flushed if the base address is specified. Return Value: None. Note: For performance reason, we may update KeSweepDcacheRange to do the following: if the range asked to sweep is very large, we may call KeSweepDcache to flush the full cache. --*/ { KIRQL OldIrql; KAFFINITY TargetProcessors; // // We will not raise IRQL to synchronization level so that we can allow // a context switch in between Flush Cache. FC need not run in the same processor // throughout. It can be context switched. So no binding is done to any processor. // // HalSweepDcacheRange(BaseAddress,Length); ASSERT(KeGetCurrentIrql() <= KiSynchIrql); // // Raise IRQL to synchronization level to prevent a context switch. // #if !defined(NT_UP) OldIrql = KeRaiseIrqlToSynchLevel(); // // Compute the set of target processors and send the sync parameters // to the target processors, if any, for execution. // TargetProcessors = KeActiveProcessors & PCR->NotMember; if (TargetProcessors != 0) { KiIpiSendPacket(TargetProcessors, KiSyncCacheTarget, NULL, NULL, NULL); } #endif // // Synchronize the Instruction Prefetch pipe in the local processor. // __synci(); __isrlz(); // // Wait until all target processors have finished sweeping the their // data cache. // #if !defined(NT_UP) if (TargetProcessors != 0) { KiIpiStallOnPacketTargets(TargetProcessors); } // // Lower IRQL to its previous level and return. // KeLowerIrql(OldIrql); #endif return; }

VOID KeSweepIcache (  IN BOOLEAN  AllProcessors  )

Definition at line 143 of file ia64/flush.c. References ASSERT, DISPATCH_LEVEL, KeAcquireSpinLock, KeActiveProcessors, KeReleaseSpinLock(), KiCacheFlushLock, KiIpiSendPacket(), KiIpiStallOnPacketTargets(), KiSweepIcacheTarget(), and NULL. 00149 : 00150 00151 This function flushes the instruction cache on all processors that are 00152 currently running threads which are children of the current process or 00153 flushes the instruction cache on all processors in the host configuration. 00154 00155 N.B. Although PowerPC maintains cache coherency across processors, we 00156 use the flash invalidate function (h/w) for I-Cache sweeps which doesn't 00157 maintain coherency so we still do the MP I-Cache flush in s/w. plj. 00158 00159 Arguments: 00160 00161 AllProcessors - Supplies a boolean value that determines which instruction 00162 caches are flushed. 00163 00164 Return Value: 00165 00166 None. 00167 00168 --*/ 00169 00170 { 00171 00172 KIRQL OldIrql; 00173 KAFFINITY TargetProcessors; 00174 00175 ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL); 00176 00177 #if !defined(NT_UP) 00178 // 00179 // Acquire cache flush spinlock 00180 // Cache flush is not MP safe yet 00181 // 00182 KeAcquireSpinLock(&KiCacheFlushLock, &OldIrql); 00183 00184 #endif 00185 00186 HalSweepDcache(); 00187 HalSweepIcache(); 00188 00189 #if !defined(NT_UP) 00190 00191 // 00192 // Compute the set of target processors and send the sweep parameters 00193 // to the target processors, if any, for execution. 00194 // 00195 00196 TargetProcessors = KeActiveProcessors & PCR->NotMember; 00197 if (TargetProcessors != 0) { 00198 KiIpiSendPacket(TargetProcessors, 00199 KiSweepIcacheTarget, 00200 NULL, 00201 NULL, 00202 NULL); 00203 } 00204 00205 00206 // 00207 // Wait until all target processors have finished sweeping their 00208 // instruction caches. 00209 // 00210 00211 00212 if (TargetProcessors != 0) { 00213 KiIpiStallOnPacketTargets(TargetProcessors); 00214 } 00215 00216 // 00217 // Lower IRQL to its previous level and return. 00218 // 00219 00220 KeReleaseSpinLock(&KiCacheFlushLock, OldIrql); 00221 00222 #endif 00223 00224 return; 00225 }

VOID KeSweepIcacheRange (  IN BOOLEAN  AllProcessors, IN PVOID  BaseAddress, IN ULONG  Length )

Definition at line 559 of file ia64/flush.c. References ASSERT, KeActiveProcessors, KeLowerIrql(), KiIpiSendPacket(), KiIpiStallOnPacketTargets(), KiSyncCacheTarget(), and NULL. : This function is used to flush a range of virtual addresses from the primary instruction cache on all processors in the host configuration. If the length of the range is greater than the size of the instruction cache, then one can call HalSweepIcache which calls SAL to flush the entire cache. Since SAL does not take care of MP flushing, HalSweepIcache has to use IPI mechanism to execute SAL flush from each processor. We need to weight the overhead of all these versus using HalSweepIcacheRange and avoiding IPI mechanism since HalSweepIcacheRange uses fc instruction and fc instruction takes care of MP. Arguments: AllProcessors - Not used BaseAddress - Supplies a pointer to the base of the range that is flushed. Length - Supplies the length of the range that is flushed if the base address is specified. Return Value: None. Note: For performance reason, we may update KeSweepIcacheRange to do the following: if the range asked to sweep is very large, we may call KeSweepIcache to flush the full cache. --*/ { KIRQL OldIrql; KAFFINITY TargetProcessors; // // We will not raise IRQL to synchronization level so that we can allow // a context switch in between Flush Cache. FC need not run in the same processor // throughout. It can be context switched. So no binding is done to any processor. // // HalSweepIcacheRange(BaseAddress,Length); ASSERT(KeGetCurrentIrql() <= KiSynchIrql); // // Raise IRQL to synchronization level to prevent a context switch. // #if !defined(NT_UP) OldIrql = KeRaiseIrqlToSynchLevel(); // // Compute the set of target processors and send the sync parameters // to the target processors, if any, for execution. // TargetProcessors = KeActiveProcessors & PCR->NotMember; if (TargetProcessors != 0) { KiIpiSendPacket(TargetProcessors, KiSyncCacheTarget, NULL, NULL, NULL); } #endif // // Synchronize the Instruction Prefetch pipe in the local processor. // __synci(); __isrlz(); // // Wait until all target processors have finished sweeping the their // data cache. // #if !defined(NT_UP) if (TargetProcessors != 0) { KiIpiStallOnPacketTargets(TargetProcessors); } // // Lower IRQL to its previous level and return. // KeLowerIrql(OldIrql); #endif return; }

VOID KiFlushIoBuffersTarget (  IN PKIPI_CONTEXT  SignalDone, IN PVOID  Mdl, IN PVOID  ReadOperation, IN PVOID  DmaOperation )

VOID KiSweepDcacheTarget (  IN PULONG  SignalDone, IN PVOID  Parameter1, IN PVOID  Parameter2, IN PVOID  Parameter3 )

Definition at line 360 of file ia64/flush.c. References KiIpiSignalPacketDone(). 00369 : 00370 00371 This is the target function for sweeping the data cache on target 00372 processors. 00373 00374 Arguments: 00375 00376 SignalDone Supplies a pointer to a variable that is cleared when the 00377 requested operation has been performed. 00378 00379 Parameter1 - Parameter3 - Not used. 00380 00381 Return Value: 00382 00383 None. 00384 00385 --*/ 00386 00387 { 00388 00389 // 00390 // Sweep the data cache on the current processor and clear the sweep 00391 // data cache packet address to signal the source to continue. 00392 // 00393 00394 #if !defined(NT_UP) 00395 00396 HalSweepDcache(); 00397 KiIpiSignalPacketDone(SignalDone); 00398 00399 #endif 00400 00401 return; 00402 }

VOID KiSweepIcacheTarget (  IN PULONG  SignalDone, IN PVOID  Parameter1, IN PVOID  Parameter2, IN PVOID  Parameter3 )

Definition at line 228 of file ia64/flush.c. References KiIpiSignalPacketDone(). 00237 : 00238 00239 This is the target function for sweeping the instruction cache on 00240 target processors. 00241 00242 Arguments: 00243 00244 SignalDone Supplies a pointer to a variable that is cleared when the 00245 requested operation has been performed. 00246 00247 Parameter1 - Parameter3 - Not used. 00248 00249 Return Value: 00250 00251 None. 00252 00253 --*/ 00254 00255 { 00256 00257 // 00258 // Sweep the instruction cache on the current processor and clear 00259 // the sweep instruction cache packet address to signal the source 00260 // to continue. 00261 // 00262 00263 #if !defined(NT_UP) 00264 00265 HalSweepDcache(); 00266 HalSweepIcache(); 00267 00268 KiIpiSignalPacketDone(SignalDone); 00269 00270 #endif 00271 00272 return; 00273 }

VOID KiSyncCacheTarget (  IN PKIPI_CONTEXT  SignalDone, IN PVOID  Parameter1, IN PVOID  Parameter2, IN PVOID  Parameter3 )

Definition at line 103 of file ia64/flush.c. References KiIpiSignalPacketDone(). Referenced by KeSweepCacheRange(), KeSweepDcacheRange(), and KeSweepIcacheRange(). : This function synchronizes the I-fetch pipeline. Typically this routine will be executed by every processor in the system in response to an IPI after the cache is flushed. Each processor executing RFI while leaving the IPI produces the serialization effect that is required after isync to make sure that further instruction prefetches wait till the ISYNC completes. Arguements: SignalDone Supplies a pointer to a variable that is cleared when the requested operation has been performed. Parameter1 - Parameter3 - Not used. Return Value: Nothing. --*/ { #if !defined(NT_UP) __synci(); KiIpiSignalPacketDone(SignalDone); #endif return; }

ULONG_PTR KiSyncMC_Drain (  IN BOOLEAN  AllProcessors, IN PVOID  BaseAddress, IN ULONG  Length )

Definition at line 787 of file ia64/flush.c. References ASSERT, KiIpiGenericCall(), KiSyncMC_DrainTarget(), NULL, PKIPI_BROADCAST_WORKER, and Status. Referenced by KeSweepCacheRangeWithDrain(). : KiSyncMC_Drain issues PAL_MC_DRAIN to drain either prefetches, demand references or pending fc cache line evictions to all the processors in the system. DrainTypePointer points to the variable, DrainType, which determines the type of drain to be performed. This is typically used when changing the memory attribute from WB to UC. Arguments: AllProcessors - All processors in the system. BaseAddress - Supplies a pointer to the base of the range that is to be drained. Length - Supplies the length of the range that is drained for the base address specified. Return Value: Note: This is used when changing attributes of WB pages to UC pages. --*/ { ULONG_PTR Status; // // KiIpiGenericCall returns ULONG_PTR as the function value of the specified function // Status = (KiIpiGenericCall ( (PKIPI_BROADCAST_WORKER)KiSyncMC_DrainTarget, (ULONG_PTR)NULL) ); ASSERT(Status == PAL_STATUS_SUCCESS); return Status; }

ULONG_PTR KiSyncMC_DrainTarget (   )

Definition at line 407 of file ia64/flush.c. References ASSERT, and Status. Referenced by KiSyncMC_Drain(). : This is the target function for issuing PAL_MC_DRAIN to drain prefetches, demand references and pending fc cache line evictions on the target CPU it executes. Argument: None Return Value: Returns the status from the function HalCallPal --*/ { ULONG_PTR Status; // // Call HalCallPal to drain. // Status = HalCallPal(PAL_MC_DRAIN, 0, 0, 0, 0, 0, 0, 0); ASSERT(Status == PAL_STATUS_SUCCESS); return Status; }

ULONG_PTR KiSyncPrefetchVisible (  IN BOOLEAN  AllProcessors, IN PVOID  BaseAddress, IN ULONG  Length )

Definition at line 885 of file ia64/flush.c. References ASSERT, KiIpiGenericCall(), KiSyncPrefetchVisibleTarget(), NeedPalVisibilitySupport, NULL, PKIPI_BROADCAST_WORKER, ProbePalVisibilitySupport, and Status. Referenced by KeSweepCacheRangeWithDrain(). : KiSyncPrefetchVisible issues PAL_PREFETCH_VISIBILITY to cause the processor to make all pending prefetches visible to subsequent fc instructions; or does nothing, on processor implementations which does not require PAL support for disabling prefetch in the architectural sequence. On processors that require PAL support for this sequence, the actions performed by this procedure may include any or all of the following (or none, as long as the processor guarantees that prefetches that were issued prior to this call are not resident in the processor's caches after the architected sequence is complete. This is typically used when changing the memory attribute from WB to UC. Arguments: AllProcessors - All processors in the system. BaseAddress - Supplies a pointer to the base of the range that is to be drained. Length - Supplies the length of the range that is drained for the base address specified. Return Value: Status of the PAL CALL 0 Success 1 Call not needed -3 Error returned Note: This is used when changing attributes of WB pages to UC pages. --*/ { ULONG_PTR Status; switch (ProbePalVisibilitySupport) { case 0: if (NeedPalVisibilitySupport == 0) return PAL_STATUS_SUPPORT_NOT_NEEDED; else { Status = (KiIpiGenericCall ( (PKIPI_BROADCAST_WORKER)KiSyncPrefetchVisibleTarget, (ULONG_PTR)NULL) ); ASSERT(Status != PAL_STATUS_ERROR); return Status; } break; case 1: Status = KiSyncPrefetchVisibleTarget(); ASSERT(Status != PAL_STATUS_ERROR); ProbePalVisibilitySupport = 0; if (Status == PAL_STATUS_SUPPORT_NOT_NEEDED) { NeedPalVisibilitySupport = 0; return PAL_STATUS_SUPPORT_NOT_NEEDED; } else { Status = (KiIpiGenericCall ( (PKIPI_BROADCAST_WORKER)KiSyncPrefetchVisibleTarget, (ULONG_PTR)NULL) ); ASSERT(Status != PAL_STATUS_ERROR); return Status; } break; } }

ULONG_PTR KiSyncPrefetchVisibleTarget (   )

Definition at line 838 of file ia64/flush.c. References ASSERT, and Status. Referenced by KiSyncPrefetchVisible(). : This is the target function for issuing PAL_PREFETCH VISIBILITY on the target CPU it executes. Argument: Not used. Return Value: Returns the status from the function HalCallPal --*/ { ULONG_PTR Status; // // Call HalCallPal to drain. // Status = HalCallPal(PAL_PREFETCH_VISIBILITY, 0, 0, 0, 0, 0, 0, 0); ASSERT(Status != PAL_STATUS_ERROR); return Status; }

---

Variable Documentation

KSPIN_LOCK KiCacheFlushLock

Definition at line 39 of file ia64/flush.c. Referenced by KeSweepDcache(), KeSweepIcache(), and KiInitializeKernel().

ULONG NeedPalVisibilitySupport = 1

Definition at line 38 of file ia64/flush.c. Referenced by KiSyncPrefetchVisible().

ULONG ProbePalVisibilitySupport = 1

Definition at line 37 of file ia64/flush.c. Referenced by KiSyncPrefetchVisible().

---