huwcampbell.com
Reactos
1$if(0)
2#pragma once
3
4#include <intrin.h>
5$endif()
6
7#define KeMemoryBarrierWithoutFence() _ReadWriteBarrier()
8
9#if defined(_M_IX86) || defined(_M_AMD64)
10#define PreFetchCacheLine(l, a) _mm_prefetch((char const *) a, l)
11#define PF_TEMPORAL_LEVEL_1 _MM_HINT_T0
12#define PF_TEMPORAL_LEVEL_2 _MM_HINT_T1
13#define PF_TEMPORAL_LEVEL_3 _MM_HINT_T2
14#define PF_NON_TEMPORAL_LEVEL_ALL _MM_HINT_NTA
15#define _AcquireBarrier()
16#define _ReleaseBarrier()
17#elif defined(_M_ARM) || defined(_M_ARM64)
18#define PreFetchCacheLine(l,a) __prefetch((const void *) (a))
19#define PrefetchForWrite(p) __prefetch((const void *) (p))
20#define PF_TEMPORAL_LEVEL_1 0
21#define PF_TEMPORAL_LEVEL_2 1
22#define PF_TEMPORAL_LEVEL_3 2
23#define PF_NON_TEMPORAL_LEVEL_ALL 3
24#define ReadForWriteAccess(p) (*(p))
25#endif
26
27#if !defined(RC_INVOKED)
28
29#if defined(_M_IX86)
30__forceinline
31void
32MemoryBarrier (
33 void)
34{
35 long Barrier;
36 _InterlockedOr(&Barrier, 0);
37}
38#define PrefetchForWrite(p)
39#define ReadForWriteAccess(p) (*(p))
40#elif defined(_M_AMD64)
41#define MemoryBarrier __faststorefence
42#define PrefetchForWrite(p) _m_prefetchw(p)
43#define ReadForWriteAccess(p) (_m_prefetchw(p), *(p))
44#elif defined(_M_ARM)
45# define MemoryBarrier() __dmb(_ARM_BARRIER_SY)
46# define _AcquireBarrier() __dmb(_ARM_BARRIER_ISH)
47# define _ReleaseBarrier() __dmb(_ARM_BARRIER_ISH)
48# define _DataSynchronizationBarrier() __dsb(_ARM_BARRIER_SY)
49# define _InstructionSynchronizationBarrier() __isb(_ARM_BARRIER_SY)
50#elif defined(_M_ARM64)
51# define MemoryBarrier() __dmb(_ARM64_BARRIER_SY)
52# define _AcquireBarrier() __dmb(_ARM64_BARRIER_ISH)
53# define _ReleaseBarrier() __dmb(_ARM64_BARRIER_ISH)
54# define _DataSynchronizationBarrier() __dsb(_ARM64_BARRIER_SY)
55# define _InstructionSynchronizationBarrier() __isb(_ARM64_BARRIER_SY)
56#else
57#error Unsupported architecture
58#endif /* _M_ARM */
59
60#if defined(_M_IX86) || defined(_M_AMD64)
61#define __iso_volatile_load8(p) (*(volatile char*)(p))
62#define __iso_volatile_load16(p) (*(volatile short*)(p))
63#define __iso_volatile_load32(p) (*(volatile int*)(p))
64#define __iso_volatile_load64(p) (*(volatile __int64*)(p))
65#define __iso_volatile_store8(p,v) (*(volatile char*)(p) = (v))
66#define __iso_volatile_store16(p,v) (*(volatile short*)(p) = (v))
67#define __iso_volatile_store32(p,v) (*(volatile int*)(p) = (v))
68#define __iso_volatile_store64(p,v) (*(volatile __int64*)(p) = (v))
69#endif
70
71__forceinline
72char
73ReadRaw8 (
74 _In_ _Interlocked_operand_ char const volatile *Source)
75{
76 return *(char *)Source;
77}
78
79__forceinline
80void
81WriteRaw8 (
82 _Out_ _Interlocked_operand_ char volatile *Destination,
83 _In_ char Value)
84{
85 *(char *)Destination = Value;
86}
87
88__forceinline
89short
90ReadRaw16 (
91 _In_ _Interlocked_operand_ short const volatile *Source)
92{
93 return *(short *)Source;
94}
95
96__forceinline
97void
98WriteRaw16 (
99 _Out_ _Interlocked_operand_ short volatile *Destination,
100 _In_ short Value)
101{
102 *(short *)Destination = Value;
103}
104
105__forceinline
106long
107ReadRaw (
108 _In_ _Interlocked_operand_ long const volatile *Source)
109{
110 return *(long *)Source;
111}
112
113__forceinline
114void
115WriteRaw (
116 _Out_ _Interlocked_operand_ long volatile *Destination,
117 _In_ long Value)
118{
119 *(long *)Destination = Value;
120}
121
122__forceinline
123__int64
124ReadRaw64 (
125 _In_ _Interlocked_operand_ __int64 const volatile *Source)
126{
127 return *(__int64 *)Source;
128}
129
130__forceinline
131void
132WriteRaw64 (
133 _Out_ _Interlocked_operand_ __int64 volatile *Destination,
134 _In_ __int64 Value)
135{
136 *(__int64 *)Destination = Value;
137}
138
139__forceinline
140char
141ReadNoFence8 (
142 _In_ _Interlocked_operand_ char const volatile *Source)
143{
144 return __iso_volatile_load8(Source);
145}
146
147__forceinline
148void
149WriteNoFence8 (
150 _Out_ _Interlocked_operand_ char volatile *Destination,
151 _In_ char Value)
152{
153 __iso_volatile_store8(Destination, Value);
154}
155
156__forceinline
157short
158ReadNoFence16 (
159 _In_ _Interlocked_operand_ short const volatile *Source)
160{
161 return __iso_volatile_load16(Source);
162}
163
164__forceinline
165void
166WriteNoFence16 (
167 _Out_ _Interlocked_operand_ short volatile *Destination,
168 _In_ short Value)
169{
170 __iso_volatile_store16(Destination, Value);
171}
172
173__forceinline
174long
175ReadNoFence (
176 _In_ _Interlocked_operand_ long const volatile *Source)
177{
178 return __iso_volatile_load32((const volatile int*)Source);
179}
180
181__forceinline
182void
183WriteNoFence (
184 _Out_ _Interlocked_operand_ long volatile *Destination,
185 _In_ long Value)
186{
187 __iso_volatile_store32((volatile int*)Destination, Value);
188}
189
190__forceinline
191__int64
192ReadNoFence64 (
193 _In_ _Interlocked_operand_ __int64 const volatile *Source)
194{
195 return __iso_volatile_load64(Source);
196}
197
198__forceinline
199void
200WriteNoFence64 (
201 _Out_ _Interlocked_operand_ __int64 volatile *Destination,
202 _In_ __int64 Value)
203{
204 __iso_volatile_store64(Destination, Value);
205}
206
207
208__forceinline
209char
210ReadAcquire8 (
211 _In_ _Interlocked_operand_ char const volatile *Source)
212{
213 char Value = __iso_volatile_load8(Source);
214 _AcquireBarrier();
215 return Value;
216}
217
218__forceinline
219void
220WriteRelease8 (
221 _Out_ _Interlocked_operand_ char volatile *Destination,
222 _In_ char Value)
223{
224 _ReleaseBarrier();
225 __iso_volatile_store8(Destination, Value);
226}
227
228__forceinline
229short
230ReadAcquire16 (
231 _In_ _Interlocked_operand_ short const volatile *Source)
232{
233 short Value = __iso_volatile_load16(Source);
234 _AcquireBarrier();
235 return Value;
236}
237
238__forceinline
239void
240WriteRelease16 (
241 _Out_ _Interlocked_operand_ short volatile *Destination,
242 _In_ short Value)
243{
244 _ReleaseBarrier();
245 __iso_volatile_store16(Destination, Value);
246}
247
248__forceinline
249long
250ReadAcquire (
251 _In_ _Interlocked_operand_ long const volatile *Source)
252{
253 long Value = __iso_volatile_load32((const volatile int*)Source);
254 _AcquireBarrier();
255 return Value;
256}
257
258__forceinline
259void
260WriteRelease (
261 _Out_ _Interlocked_operand_ long volatile *Destination,
262 _In_ long Value)
263{
264 _ReleaseBarrier();
265 __iso_volatile_store32((volatile int*)Destination, Value);
266}
267
268__forceinline
269__int64
270ReadAcquire64 (
271 _In_ _Interlocked_operand_ __int64 const volatile *Source)
272{
273 __int64 Value = __iso_volatile_load64(Source);
274 _AcquireBarrier();
275 return Value;
276}
277
278__forceinline
279void
280WriteRelease64 (
281 _Out_ _Interlocked_operand_ __int64 volatile *Destination,
282 _In_ __int64 Value)
283{
284 _ReleaseBarrier();
285 __iso_volatile_store64(Destination, Value);
286}
287
288
289__forceinline
290unsigned char
291ReadUCharAcquire (
292 _In_ _Interlocked_operand_ unsigned char const volatile *Source)
293{
294 return (unsigned char)ReadAcquire8((char*)Source);
295}
296
297__forceinline
298unsigned char
299ReadUCharNoFence (
300 _In_ _Interlocked_operand_ unsigned char const volatile *Source)
301{
302 return (unsigned char)ReadNoFence8((char*)Source);
303}
304
305__forceinline
306unsigned char
307ReadUCharRaw (
308 _In_ _Interlocked_operand_ unsigned char const volatile *Source)
309{
310 return (unsigned char)ReadRaw8((char*)Source);
311}
312
313__forceinline
314void
315WriteUCharRelease (
316 _Out_ _Interlocked_operand_ unsigned char volatile *Destination,
317 _In_ unsigned char Value)
318{
319 WriteRelease8((char*)Destination, (char)Value);
320}
321
322__forceinline
323void
324WriteUCharNoFence (
325 _Out_ _Interlocked_operand_ unsigned char volatile *Destination,
326 _In_ unsigned char Value)
327{
328 WriteNoFence8((char*)Destination, (char)Value);
329}
330
331__forceinline
332void
333WriteUCharRaw (
334 _Out_ _Interlocked_operand_ unsigned char volatile *Destination,
335 _In_ unsigned char Value)
336{
337 WriteRaw8((char*)Destination, (char)Value);
338}
339
340__forceinline
341BOOLEAN
342ReadBooleanAcquire (
343 _In_ _Interlocked_operand_ BOOLEAN const volatile *Source)
344{
345 return (BOOLEAN)ReadAcquire8((char*)Source);
346}
347
348__forceinline
349unsigned char
350ReadBooleanNoFence (
351 _In_ _Interlocked_operand_ BOOLEAN const volatile *Source)
352{
353 return (BOOLEAN)ReadNoFence8((char*)Source);
354}
355
356__forceinline
357void
358WriteBooleanRelease (
359 _Out_ _Interlocked_operand_ BOOLEAN volatile *Destination,
360 _In_ BOOLEAN Value)
361{
362 WriteRelease8((char*)Destination, (char)Value);
363}
364
365__forceinline
366void
367WriteBooleanNoFence (
368 _Out_ _Interlocked_operand_ BOOLEAN volatile *Destination,
369 _In_ BOOLEAN Value)
370{
371 WriteNoFence8((char*)Destination, (char)Value);
372}
373
374__forceinline
375unsigned short
376ReadUShortAcquire (
377 _In_ _Interlocked_operand_ unsigned short const volatile *Source)
378{
379 return (unsigned short)ReadAcquire16((short*)Source);
380}
381
382__forceinline
383unsigned short
384ReadUShortNoFence (
385 _In_ _Interlocked_operand_ unsigned short const volatile *Source)
386{
387 return (unsigned short)ReadNoFence16((short*)Source);
388}
389
390__forceinline
391unsigned short
392ReadUShortRaw (
393 _In_ _Interlocked_operand_ unsigned short const volatile *Source)
394{
395 return (unsigned short)ReadRaw16((short*)Source);
396}
397
398__forceinline
399void
400WriteUShortRelease (
401 _Out_ _Interlocked_operand_ unsigned short volatile *Destination,
402 _In_ unsigned short Value)
403{
404 WriteRelease16((short*)Destination, (short)Value);
405}
406
407__forceinline
408void
409WriteUShortNoFence (
410 _Out_ _Interlocked_operand_ unsigned short volatile *Destination,
411 _In_ unsigned short Value)
412{
413 WriteNoFence16((short*)Destination, (short)Value);
414}
415
416__forceinline
417void
418WriteUShortRaw (
419 _Out_ _Interlocked_operand_ unsigned short volatile *Destination,
420 _In_ unsigned short Value)
421{
422 WriteRaw16((short*)Destination, (short)Value);
423}
424
425__forceinline
426unsigned long
427ReadULongAcquire (
428 _In_ _Interlocked_operand_ unsigned long const volatile *Source)
429{
430 return (unsigned long)ReadAcquire((long*)Source);
431}
432
433__forceinline
434unsigned long
435ReadULongNoFence (
436 _In_ _Interlocked_operand_ unsigned long const volatile *Source)
437{
438 return (unsigned long)ReadNoFence((long*)Source);
439}
440
441__forceinline
442unsigned long
443ReadULongRaw (
444 _In_ _Interlocked_operand_ unsigned long const volatile *Source)
445{
446 return (unsigned long)ReadRaw((long*)Source);
447}
448
449__forceinline
450void
451WriteULongRelease (
452 _Out_ _Interlocked_operand_ unsigned long volatile *Destination,
453 _In_ unsigned long Value)
454{
455 WriteRelease((long*)Destination, (long)Value);
456}
457
458__forceinline
459void
460WriteULongNoFence (
461 _Out_ _Interlocked_operand_ unsigned long volatile *Destination,
462 _In_ unsigned long Value)
463{
464 WriteNoFence((long*)Destination, (long)Value);
465}
466
467__forceinline
468void
469WriteULongRaw (
470 _Out_ _Interlocked_operand_ unsigned long volatile *Destination,
471 _In_ unsigned long Value)
472{
473 WriteRaw((long*)Destination, (long)Value);
474}
475
476__forceinline
477unsigned __int64
478ReadULong64Acquire (
479 _In_ _Interlocked_operand_ unsigned __int64 const volatile *Source)
480{
481 return (unsigned __int64)ReadAcquire64((__int64*)Source);
482}
483
484__forceinline
485unsigned __int64
486ReadULong64NoFence (
487 _In_ _Interlocked_operand_ unsigned __int64 const volatile *Source)
488{
489 return (unsigned __int64)ReadNoFence64((__int64*)Source);
490}
491
492__forceinline
493unsigned __int64
494ReadULong64Raw (
495 _In_ _Interlocked_operand_ unsigned __int64 const volatile *Source)
496{
497 return (unsigned __int64)ReadRaw64((__int64*)Source);
498}
499
500__forceinline
501void
502WriteULong64Release (
503 _Out_ _Interlocked_operand_ unsigned __int64 volatile *Destination,
504 _In_ unsigned __int64 Value)
505{
506 WriteRelease64((__int64*)Destination, (__int64)Value);
507}
508
509__forceinline
510void
511WriteULong64NoFence (
512 _Out_ _Interlocked_operand_ unsigned __int64 volatile *Destination,
513 _In_ unsigned __int64 Value)
514{
515 WriteNoFence64((__int64*)Destination, (__int64)Value);
516}
517
518__forceinline
519void
520WriteULong64Raw (
521 _Out_ _Interlocked_operand_ unsigned __int64 volatile *Destination,
522 _In_ unsigned __int64 Value)
523{
524 WriteRaw64((__int64*)Destination, (__int64)Value);
525}
526
527#ifdef _WIN64
528
529__forceinline
530void*
531ReadPointerAcquire (
532 _In_ _Interlocked_operand_ void* const volatile *Source)
533{
534 return (void*)ReadAcquire64((__int64*)Source);
535}
536
537__forceinline
538void*
539ReadPointerNoFence (
540 _In_ _Interlocked_operand_ void* const volatile *Source)
541{
542 return (void*)ReadNoFence64((__int64*)Source);
543}
544
545__forceinline
546void*
547ReadPointerRaw (
548 _In_ _Interlocked_operand_ void* const volatile *Source)
549{
550 return (void*)ReadRaw64((__int64*)Source);
551}
552
553__forceinline
554void
555WritePointerRelease (
556 _Out_ _Interlocked_operand_ void* volatile *Destination,
557 _In_ void* Value)
558{
559 WriteRelease64((__int64*)Destination, (__int64)Value);
560}
561
562__forceinline
563void
564WritePointerNoFence (
565 _Out_ _Interlocked_operand_ void* volatile *Destination,
566 _In_ void* Value)
567{
568 WriteNoFence64((__int64*)Destination, (__int64)Value);
569}
570
571__forceinline
572void
573WritePointerRaw (
574 _Out_ _Interlocked_operand_ void* volatile *Destination,
575 _In_ void* Value)
576{
577 WriteRaw64((__int64*)Destination, (__int64)Value);
578}
579
580#define ReadLongPtrAcquire ReadAcquire64
581#define ReadLongPtrNoFence ReadNoFence64
582#define ReadLongPtrRaw ReadRaw64
583#define WriteLongPtrRelease WriteRelease64
584#define WriteLongPtrNoFence WriteNoFence64
585#define WriteLongPtrRaw WriteRaw64
586#define ReadULongPtrAcquire ReadULong64Acquire
587#define ReadULongPtrNoFence ReadULong64NoFence
588#define ReadULongPtrRaw ReadULong64Raw
589#define WriteULongPtrRelease WriteULong64Release
590#define WriteULongPtrNoFence WriteULong64NoFence
591#define WriteULongPtrRaw WriteULong64Raw
592
593#else // _WIN64
594
595__forceinline
596void*
597ReadPointerAcquire (
598 _In_ _Interlocked_operand_ void* const volatile *Source)
599{
600 return (void*)ReadAcquire((long*)Source);
601}
602
603__forceinline
604void*
605ReadPointerNoFence (
606 _In_ _Interlocked_operand_ void* const volatile *Source)
607{
608 return (void*)ReadNoFence((long*)Source);
609}
610
611__forceinline
612void*
613ReadPointerRaw (
614 _In_ _Interlocked_operand_ void* const volatile *Source)
615{
616 return (void*)ReadRaw((long*)Source);
617}
618
619__forceinline
620void
621WritePointerRelease (
622 _Out_ _Interlocked_operand_ void* volatile *Destination,
623 _In_ void* Value)
624{
625 WriteRelease((long*)Destination, (long)Value);
626}
627
628__forceinline
629void
630WritePointerNoFence (
631 _Out_ _Interlocked_operand_ void* volatile *Destination,
632 _In_opt_ void* Value)
633{
634 WriteNoFence((long*)Destination, (long)Value);
635}
636
637__forceinline
638void
639WritePointerRaw (
640 _Out_ _Interlocked_operand_ void* volatile *Destination,
641 _In_opt_ void* Value)
642{
643 WriteRaw((long*)Destination, (long)Value);
644}
645
646#define ReadLongPtrAcquire ReadAcquire
647#define ReadLongPtrNoFence ReadNoFence
648#define ReadLongPtrRaw ReadRaw
649#define WriteLongPtrRelease WriteRelease
650#define WriteLongPtrNoFence WriteNoFence
651#define WriteLongPtrRaw WriteRaw
652#define ReadULongPtrAcquire ReadULongAcquire
653#define ReadULongPtrNoFence ReadULongNoFence
654#define ReadULongPtrRaw ReadULongRaw
655#define WriteULongPtrRelease WriteULongRelease
656#define WriteULongPtrNoFence WriteULongNoFence
657#define WriteULongPtrRaw WriteULongRaw
658
659#endif // _WIN64
660
661#define ReadSizeTAcquire ReadULongPtrAcquire
662#define ReadSizeTNoFence ReadULongPtrNoFence
663#define ReadSizeTRaw ReadULongPtrRaw
664#define WriteSizeTRelease WriteULongPtrRelease
665#define WriteSizeTNoFence WriteULongPtrNoFence
666#define WriteSizeTRaw WriteULongPtrRaw
667
668/* Overloaded functions for C++ */
669#if defined(__cplusplus)
670extern "C++" {
671
672 template<typename T>
673 __forceinline
674 T
675 ReadRaw (
676 _In_ _Interlocked_operand_ T const volatile *Source)
677 {
678 return *(T*)Source;
679 }
680
681 template<typename T>
682 __forceinline
683 void
684 WriteRaw (
685 _Out_ _Interlocked_operand_ T volatile *Destination,
686 _In_ T Value)
687 {
688 *(T*)Destination = Value;
689 }
690
691 template<typename T>
692 __forceinline
693 T
694 ReadNoFence (
695 _In_ _Interlocked_operand_ T const volatile *Source)
696 {
697 switch (sizeof(T))
698 {
699 case 1: return (T)ReadNoFence8((char const volatile *)Source);
700 case 2: return (T)ReadNoFence16((short const volatile *)Source);
701 case 4: return (T)ReadNoFence((long const volatile *)Source);
702 case 8: return (T)ReadNoFence64((__int64 const volatile *)Source);
703 }
704 }
705
706 template<typename T>
707 __forceinline
708 void
709 WriteNoFence (
710 _Out_ _Interlocked_operand_ T volatile *Destination,
711 _In_ T Value)
712 {
713 switch (sizeof(T))
714 {
715 case 1: WriteNoFence8((char volatile *)Destination, (char)Value);
716 case 2: WriteNoFence16((short volatile *)Destination, (short)Value);
717 case 4: WriteNoFence((long volatile *)Destination, (long)Value);
718 case 8: WriteNoFence64((__int64 volatile *)Destination, (__int64)Value);
719 }
720 }
721
722} // extern "C++"
723#endif // __cplusplus
724
725#endif // !defined(RC_INVOKED)
726
727#undef _AcquireBarrier
728#undef _ReleaseBarrier