[XDK] Add Read/Write helper functions

sdk/include/xdk/memaccess.h

···
+$if(0)
+#pragma once
+
+#include <intrin.h>
+$endif()
+
+#define KeMemoryBarrierWithoutFence() _ReadWriteBarrier()
+
+#if defined(_M_IX86) || defined(_M_AMD64)
+#define PreFetchCacheLine(l, a)               _mm_prefetch((char const *) a, l)
+#define PF_TEMPORAL_LEVEL_1 _MM_HINT_T0
+#define PF_TEMPORAL_LEVEL_2 _MM_HINT_T1
+#define PF_TEMPORAL_LEVEL_3 _MM_HINT_T2
+#define PF_NON_TEMPORAL_LEVEL_ALL _MM_HINT_NTA
+#define _AcquireBarrier()
+#define _ReleaseBarrier()
+#elif defined(_M_ARM) || defined(_M_ARM64)
+#define PreFetchCacheLine(l,a)               __prefetch((const void *) (a))
+#define PrefetchForWrite(p)                  __prefetch((const void *) (p))
+#define PF_TEMPORAL_LEVEL_1         0
+#define PF_TEMPORAL_LEVEL_2         1
+#define PF_TEMPORAL_LEVEL_3         2
+#define PF_NON_TEMPORAL_LEVEL_ALL   3
+#define ReadForWriteAccess(p)                (*(p))
+#endif
+
+#if !defined(RC_INVOKED)
+
+#if defined(_M_IX86) 
+__forceinline
+void
+MemoryBarrier (
+    void)
+{
+    long Barrier;
+    _InterlockedOr(&Barrier, 0);
+}
+#define PrefetchForWrite(p)
+#define ReadForWriteAccess(p)                 (*(p))
+#elif defined(_M_AMD64)
+#define MemoryBarrier                         __faststorefence
+#define PrefetchForWrite(p)                   _m_prefetchw(p)
+#define ReadForWriteAccess(p)                 (_m_prefetchw(p), *(p))
+#elif defined(_M_ARM)
+# define MemoryBarrier()                      __dmb(_ARM_BARRIER_SY)
+# define _AcquireBarrier()                    __dmb(_ARM_BARRIER_ISH)
+# define _ReleaseBarrier()                    __dmb(_ARM_BARRIER_ISH)
+# define _DataSynchronizationBarrier()        __dsb(_ARM_BARRIER_SY)
+# define _InstructionSynchronizationBarrier() __isb(_ARM_BARRIER_SY)
+#elif defined(_M_ARM64)
+# define MemoryBarrier()                      __dmb(_ARM64_BARRIER_SY)
+# define _AcquireBarrier()                    __dmb(_ARM64_BARRIER_ISH)
+# define _ReleaseBarrier()                    __dmb(_ARM64_BARRIER_ISH)
+# define _DataSynchronizationBarrier()        __dsb(_ARM64_BARRIER_SY)
+# define _InstructionSynchronizationBarrier() __isb(_ARM64_BARRIER_SY)
+#else
+#error Unsupported architecture
+#endif /* _M_ARM */
+
+#if defined(_M_IX86) || defined(_M_AMD64)
+#define __iso_volatile_load8(p) (*(volatile char*)(p))
+#define __iso_volatile_load16(p) (*(volatile short*)(p))
+#define __iso_volatile_load32(p) (*(volatile int*)(p))
+#define __iso_volatile_load64(p) (*(volatile __int64*)(p))
+#define __iso_volatile_store8(p,v) (*(volatile char*)(p) = (v))
+#define __iso_volatile_store16(p,v) (*(volatile short*)(p) = (v))
+#define __iso_volatile_store32(p,v) (*(volatile int*)(p) = (v))
+#define __iso_volatile_store64(p,v) (*(volatile __int64*)(p) = (v))
+#endif
+
+__forceinline
+char
+ReadRaw8 (
+    _In_ _Interlocked_operand_ char const volatile *Source)
+{
+    return *(char *)Source;
+}
+
+__forceinline
+void
+WriteRaw8 (
+    _Out_ _Interlocked_operand_ char volatile *Destination,
+    _In_ char Value)
+{
+    *(char *)Destination = Value;
+}
+
+__forceinline
+short
+ReadRaw16 (
+    _In_ _Interlocked_operand_ short const volatile *Source)
+{
+    return *(short *)Source;
+}
+
+__forceinline
+void
+WriteRaw16 (
+    _Out_ _Interlocked_operand_ short volatile *Destination,
+    _In_ short Value)
+{
+    *(short *)Destination = Value;
+}
+
+__forceinline
+long
+ReadRaw (
+    _In_ _Interlocked_operand_ long const volatile *Source)
+{
+    return *(long *)Source;
+}
+
+__forceinline
+void
+WriteRaw (
+    _Out_ _Interlocked_operand_ long volatile *Destination,
+    _In_ long Value)
+{
+    *(long *)Destination = Value;
+}
+
+__forceinline
+__int64
+ReadRaw64 (
+    _In_ _Interlocked_operand_ __int64 const volatile *Source)
+{
+    return *(__int64 *)Source;
+}
+
+__forceinline
+void
+WriteRaw64 (
+    _Out_ _Interlocked_operand_ __int64 volatile *Destination,
+    _In_ __int64 Value)
+{
+    *(__int64 *)Destination = Value;
+}
+
+__forceinline
+char
+ReadNoFence8 (
+    _In_ _Interlocked_operand_ char const volatile *Source)
+{
+    return __iso_volatile_load8(Source);
+}
+
+__forceinline
+void
+WriteNoFence8 (
+    _Out_ _Interlocked_operand_ char volatile *Destination,
+    _In_ char Value)
+{
+    __iso_volatile_store8(Destination, Value);
+}
+
+__forceinline
+short
+ReadNoFence16 (
+    _In_ _Interlocked_operand_ short const volatile *Source)
+{
+    return __iso_volatile_load16(Source);
+}
+
+__forceinline
+void
+WriteNoFence16 (
+    _Out_ _Interlocked_operand_ short volatile *Destination,
+    _In_ short Value)
+{
+    __iso_volatile_store16(Destination, Value);
+}
+
+__forceinline
+long
+ReadNoFence (
+    _In_ _Interlocked_operand_ long const volatile *Source)
+{
+    return __iso_volatile_load32((const volatile int*)Source);
+}
+
+__forceinline
+void
+WriteNoFence (
+    _Out_ _Interlocked_operand_ long volatile *Destination,
+    _In_ long Value)
+{
+    __iso_volatile_store32((volatile int*)Destination, Value);
+}
+
+__forceinline
+__int64
+ReadNoFence64 (
+    _In_ _Interlocked_operand_ __int64 const volatile *Source)
+{
+    return __iso_volatile_load64(Source);
+}
+
+__forceinline
+void
+WriteNoFence64 (
+    _Out_ _Interlocked_operand_ __int64 volatile *Destination,
+    _In_ __int64 Value)
+{
+    __iso_volatile_store64(Destination, Value);
+}
+
+
+__forceinline
+char
+ReadAcquire8 (
+    _In_ _Interlocked_operand_ char const volatile *Source)
+{
+    char Value = __iso_volatile_load8(Source);
+    _AcquireBarrier();
+    return Value;
+}
+
+__forceinline
+void
+WriteRelease8 (
+    _Out_ _Interlocked_operand_ char volatile *Destination,
+    _In_ char Value)
+{
+    _ReleaseBarrier();
+    __iso_volatile_store8(Destination, Value);
+}
+
+__forceinline
+short
+ReadAcquire16 (
+    _In_ _Interlocked_operand_ short const volatile *Source)
+{
+    short Value = __iso_volatile_load16(Source);
+    _AcquireBarrier();
+    return Value;
+}
+
+__forceinline
+void
+WriteRelease16 (
+    _Out_ _Interlocked_operand_ short volatile *Destination,
+    _In_ short Value)
+{
+    _ReleaseBarrier();
+    __iso_volatile_store16(Destination, Value);
+}
+
+__forceinline
+long
+ReadAcquire (
+    _In_ _Interlocked_operand_ long const volatile *Source)
+{
+    long Value = __iso_volatile_load32((const volatile int*)Source);
+    _AcquireBarrier();
+    return Value;
+}
+
+__forceinline
+void
+WriteRelease (
+    _Out_ _Interlocked_operand_ long volatile *Destination,
+    _In_ long Value)
+{
+    _ReleaseBarrier();
+    __iso_volatile_store32((volatile int*)Destination, Value);
+}
+
+__forceinline
+__int64
+ReadAcquire64 (
+    _In_ _Interlocked_operand_ __int64 const volatile *Source)
+{
+    __int64 Value = __iso_volatile_load64(Source);
+    _AcquireBarrier();
+    return Value;
+}
+
+__forceinline
+void
+WriteRelease64 (
+    _Out_ _Interlocked_operand_ __int64 volatile *Destination,
+    _In_ __int64 Value)
+{
+    _ReleaseBarrier();
+    __iso_volatile_store64(Destination, Value);
+}
+
+
+__forceinline
+unsigned char
+ReadUCharAcquire (
+    _In_ _Interlocked_operand_ unsigned char const volatile *Source)
+{
+    return (unsigned char)ReadAcquire8((char*)Source);
+}
+
+__forceinline
+unsigned char
+ReadUCharNoFence (
+    _In_ _Interlocked_operand_ unsigned char const volatile *Source)
+{
+    return (unsigned char)ReadNoFence8((char*)Source);
+}
+
+__forceinline
+unsigned char
+ReadUCharRaw (
+    _In_ _Interlocked_operand_ unsigned char const volatile *Source)
+{
+    return (unsigned char)ReadRaw8((char*)Source);
+}
+
+__forceinline
+void
+WriteUCharRelease (
+    _Out_ _Interlocked_operand_ unsigned char volatile *Destination,
+    _In_ unsigned char Value)
+{
+    WriteRelease8((char*)Destination, (char)Value);
+}
+
+__forceinline
+void
+WriteUCharNoFence (
+    _Out_ _Interlocked_operand_ unsigned char volatile *Destination,
+    _In_ unsigned char Value)
+{
+    WriteNoFence8((char*)Destination, (char)Value);
+}
+
+__forceinline
+void
+WriteUCharRaw (
+    _Out_ _Interlocked_operand_ unsigned char volatile *Destination,
+    _In_ unsigned char Value)
+{
+    WriteRaw8((char*)Destination, (char)Value);
+}
+
+__forceinline
+BOOLEAN
+ReadBooleanAcquire (
+    _In_ _Interlocked_operand_ BOOLEAN const volatile *Source)
+{
+    return (BOOLEAN)ReadAcquire8((char*)Source);
+}
+
+__forceinline
+unsigned char
+ReadBooleanNoFence (
+    _In_ _Interlocked_operand_ BOOLEAN const volatile *Source)
+{
+    return (BOOLEAN)ReadNoFence8((char*)Source);
+}
+
+__forceinline
+void
+WriteBooleanRelease (
+    _Out_ _Interlocked_operand_ BOOLEAN volatile *Destination,
+    _In_ BOOLEAN Value)
+{
+    WriteRelease8((char*)Destination, (char)Value);
+}
+
+__forceinline
+void
+WriteBooleanNoFence (
+    _Out_ _Interlocked_operand_ BOOLEAN volatile *Destination,
+    _In_ BOOLEAN Value)
+{
+    WriteNoFence8((char*)Destination, (char)Value);
+}
+
+__forceinline
+unsigned short
+ReadUShortAcquire (
+    _In_ _Interlocked_operand_ unsigned short const volatile *Source)
+{
+    return (unsigned short)ReadAcquire16((short*)Source);
+}
+
+__forceinline
+unsigned short
+ReadUShortNoFence (
+    _In_ _Interlocked_operand_ unsigned short const volatile *Source)
+{
+    return (unsigned short)ReadNoFence16((short*)Source);
+}
+
+__forceinline
+unsigned short
+ReadUShortRaw (
+    _In_ _Interlocked_operand_ unsigned short const volatile *Source)
+{
+    return (unsigned short)ReadRaw16((short*)Source);
+}
+
+__forceinline
+void
+WriteUShortRelease (
+    _Out_ _Interlocked_operand_ unsigned short volatile *Destination,
+    _In_ unsigned short Value)
+{
+    WriteRelease16((short*)Destination, (short)Value);
+}
+
+__forceinline
+void
+WriteUShortNoFence (
+    _Out_ _Interlocked_operand_ unsigned short volatile *Destination,
+    _In_ unsigned short Value)
+{
+    WriteNoFence16((short*)Destination, (short)Value);
+}
+
+__forceinline
+void
+WriteUShortRaw (
+    _Out_ _Interlocked_operand_ unsigned short volatile *Destination,
+    _In_ unsigned short Value)
+{
+    WriteRaw16((short*)Destination, (short)Value);
+}
+
+__forceinline
+unsigned long
+ReadULongAcquire (
+    _In_ _Interlocked_operand_ unsigned long const volatile *Source)
+{
+    return (unsigned long)ReadAcquire((long*)Source);
+}
+
+__forceinline
+unsigned long
+ReadULongNoFence (
+    _In_ _Interlocked_operand_ unsigned long const volatile *Source)
+{
+    return (unsigned long)ReadNoFence((long*)Source);
+}
+
+__forceinline
+unsigned long
+ReadULongRaw (
+    _In_ _Interlocked_operand_ unsigned long const volatile *Source)
+{
+    return (unsigned long)ReadRaw((long*)Source);
+}
+
+__forceinline
+void
+WriteULongRelease (
+    _Out_ _Interlocked_operand_ unsigned long volatile *Destination,
+    _In_ unsigned long Value)
+{
+    WriteRelease((long*)Destination, (long)Value);
+}
+
+__forceinline
+void
+WriteULongNoFence (
+    _Out_ _Interlocked_operand_ unsigned long volatile *Destination,
+    _In_ unsigned long Value)
+{
+    WriteNoFence((long*)Destination, (long)Value);
+}
+
+__forceinline
+void
+WriteULongRaw (
+    _Out_ _Interlocked_operand_ unsigned long volatile *Destination,
+    _In_ unsigned long Value)
+{
+    WriteRaw((long*)Destination, (long)Value);
+}
+
+__forceinline
+unsigned __int64
+ReadULong64Acquire (
+    _In_ _Interlocked_operand_ unsigned __int64 const volatile *Source)
+{
+    return (unsigned __int64)ReadAcquire64((__int64*)Source);
+}
+
+__forceinline
+unsigned __int64
+ReadULong64NoFence (
+    _In_ _Interlocked_operand_ unsigned __int64 const volatile *Source)
+{
+    return (unsigned __int64)ReadNoFence64((__int64*)Source);
+}
+
+__forceinline
+unsigned __int64
+ReadULong64Raw (
+    _In_ _Interlocked_operand_ unsigned __int64 const volatile *Source)
+{
+    return (unsigned __int64)ReadRaw64((__int64*)Source);
+}
+
+__forceinline
+void
+WriteULong64Release (
+    _Out_ _Interlocked_operand_ unsigned __int64 volatile *Destination,
+    _In_ unsigned __int64 Value)
+{
+    WriteRelease64((__int64*)Destination, (__int64)Value);
+}
+
+__forceinline
+void
+WriteULong64NoFence (
+    _Out_ _Interlocked_operand_ unsigned __int64 volatile *Destination,
+    _In_ unsigned __int64 Value)
+{
+    WriteNoFence64((__int64*)Destination, (__int64)Value);
+}
+
+__forceinline
+void
+WriteULong64Raw (
+    _Out_ _Interlocked_operand_ unsigned __int64 volatile *Destination,
+    _In_ unsigned __int64 Value)
+{
+    WriteRaw64((__int64*)Destination, (__int64)Value);
+}
+
+#ifdef _WIN64
+
+__forceinline
+void*
+ReadPointerAcquire (
+    _In_ _Interlocked_operand_ void* const volatile *Source)
+{
+    return (void*)ReadAcquire64((__int64*)Source);
+}
+
+__forceinline
+void*
+ReadPointerNoFence (
+    _In_ _Interlocked_operand_ void* const volatile *Source)
+{
+    return (void*)ReadNoFence64((__int64*)Source);
+}
+
+__forceinline
+void*
+ReadPointerRaw (
+    _In_ _Interlocked_operand_ void* const volatile *Source)
+{
+    return (void*)ReadRaw64((__int64*)Source);
+}
+
+__forceinline
+void
+WritePointerRelease (
+    _Out_ _Interlocked_operand_ void* volatile *Destination,
+    _In_ void* Value)
+{
+    WriteRelease64((__int64*)Destination, (__int64)Value);
+}
+
+__forceinline
+void
+WritePointerNoFence (
+    _Out_ _Interlocked_operand_ void* volatile *Destination,
+    _In_ void* Value)
+{
+    WriteNoFence64((__int64*)Destination, (__int64)Value);
+}
+
+__forceinline
+void
+WritePointerRaw (
+    _Out_ _Interlocked_operand_ void* volatile *Destination,
+    _In_ void* Value)
+{
+    WriteRaw64((__int64*)Destination, (__int64)Value);
+}
+
+#define ReadLongPtrAcquire ReadAcquire64
+#define ReadLongPtrNoFence ReadNoFence64
+#define ReadLongPtrRaw ReadRaw64
+#define WriteLongPtrRelease WriteRelease64
+#define WriteLongPtrNoFence WriteNoFence64
+#define WriteLongPtrRaw WriteRaw64
+#define ReadULongPtrAcquire ReadULong64Acquire
+#define ReadULongPtrNoFence ReadULong64NoFence
+#define ReadULongPtrRaw ReadULong64Raw
+#define WriteULongPtrRelease WriteULong64Release
+#define WriteULongPtrNoFence WriteULong64NoFence
+#define WriteULongPtrRaw WriteULong64Raw
+
+#else // _WIN64
+
+__forceinline
+void*
+ReadPointerAcquire (
+    _In_ _Interlocked_operand_ void* const volatile *Source)
+{
+    return (void*)ReadAcquire((long*)Source);
+}
+
+__forceinline
+void*
+ReadPointerNoFence (
+    _In_ _Interlocked_operand_ void* const volatile *Source)
+{
+    return (void*)ReadNoFence((long*)Source);
+}
+
+__forceinline
+void*
+ReadPointerRaw (
+    _In_ _Interlocked_operand_ void* const volatile *Source)
+{
+    return (void*)ReadRaw((long*)Source);
+}
+
+__forceinline
+void
+WritePointerRelease (
+    _Out_ _Interlocked_operand_ void* volatile *Destination,
+    _In_ void* Value)
+{
+    WriteRelease((long*)Destination, (long)Value);
+}
+
+__forceinline
+void
+WritePointerNoFence (
+    _Out_ _Interlocked_operand_ void* volatile *Destination,
+    _In_opt_ void* Value)
+{
+    WriteNoFence((long*)Destination, (long)Value);
+}
+
+__forceinline
+void
+WritePointerRaw (
+    _Out_ _Interlocked_operand_ void* volatile *Destination,
+    _In_opt_ void* Value)
+{
+    WriteRaw((long*)Destination, (long)Value);
+}
+
+#define ReadLongPtrAcquire ReadAcquire
+#define ReadLongPtrNoFence ReadNoFence
+#define ReadLongPtrRaw ReadRaw
+#define WriteLongPtrRelease WriteRelease
+#define WriteLongPtrNoFence WriteNoFence
+#define WriteLongPtrRaw WriteRaw
+#define ReadULongPtrAcquire ReadULongAcquire
+#define ReadULongPtrNoFence ReadULongNoFence
+#define ReadULongPtrRaw ReadULongRaw
+#define WriteULongPtrRelease WriteULongRelease
+#define WriteULongPtrNoFence WriteULongNoFence
+#define WriteULongPtrRaw WriteULongRaw
+
+#endif // _WIN64
+
+#define ReadSizeTAcquire ReadULongPtrAcquire
+#define ReadSizeTNoFence ReadULongPtrNoFence
+#define ReadSizeTRaw ReadULongPtrRaw
+#define WriteSizeTRelease WriteULongPtrRelease
+#define WriteSizeTNoFence WriteULongPtrNoFence
+#define WriteSizeTRaw WriteULongPtrRaw
+
+/* Overloaded functions for C++ */
+#if defined(__cplusplus)
+extern "C++" {
+
+    template<typename T>
+    __forceinline
+    T
+    ReadRaw (
+        _In_ _Interlocked_operand_ T const volatile *Source)
+    {
+        return *(T*)Source;
+    }
+
+    template<typename T>
+    __forceinline
+    void
+    WriteRaw (
+        _Out_ _Interlocked_operand_ T volatile *Destination,
+        _In_ T Value)
+    {
+        *(T*)Destination = Value;
+    }
+
+    template<typename T>
+    __forceinline
+    T
+    ReadNoFence (
+        _In_ _Interlocked_operand_ T const volatile *Source)
+    {
+        switch (sizeof(T))
+        {
+            case 1: return (T)ReadNoFence8((char const volatile *)Source);
+            case 2: return (T)ReadNoFence16((short const volatile *)Source);
+            case 4: return (T)ReadNoFence((long const volatile *)Source);
+            case 8: return (T)ReadNoFence64((__int64 const volatile *)Source);
+        }
+    }
+
+    template<typename T>
+    __forceinline
+    void
+    WriteNoFence (
+        _Out_ _Interlocked_operand_ T volatile *Destination,
+        _In_ T Value)
+    {
+        switch (sizeof(T))
+        {
+            case 1: WriteNoFence8((char volatile *)Destination, (char)Value);
+            case 2: WriteNoFence16((short volatile *)Destination, (short)Value);
+            case 4: WriteNoFence((long volatile *)Destination, (long)Value);
+            case 8: WriteNoFence64((__int64 volatile *)Destination, (__int64)Value);
+        }
+    }
+
+} // extern "C++"
+#endif // __cplusplus
+
+#endif // !defined(RC_INVOKED)
+
+#undef _AcquireBarrier
+#undef _ReleaseBarrier

sdk/include/xdk/wdm.template.h

···
 
 
 $define (_WDMDDK_)
+$include (memaccess.h)
 $include (interlocked.h)
 $include (rtltypes.h)
 $include (ketypes.h)

sdk/include/xdk/winnt.template.h

···
 $define(USHORT=WORD)
 $define(UCHAR=BYTE)
 $include(ntbasedef.h)
+$include(memaccess.h)
 $include(interlocked.h)
 $include(ketypes.h)
 $include(extypes.h)

sdk/include/xdk/winnt_old.h

···
     return *((PVOID *)GetCurrentFiber());
 }
 
-/* TODO: Other architectures than X86 */
-#if defined(_M_IX86)
-#define PF_TEMPORAL_LEVEL_1
-#define PF_NON_TEMPORAL_LEVEL_ALL
-#define PreFetchCacheLine(l, a)
-#elif defined (_M_AMD64)
-#define PreFetchCacheLine(l, a)
-#elif defined(_M_PPC)
-#define PreFetchCacheLine(l, a)
-#elif defined(_M_ARM)
-#define PreFetchCacheLine(l, a)
-#elif defined(_M_ARM64)
-#define PreFetchCacheLine(l, a)
-#else
-#error Unknown architecture
-#endif
-
-/* TODO: Other architectures than X86 */
-#if defined(_M_IX86)
-#if defined(_MSC_VER)
-FORCEINLINE
-VOID
-MemoryBarrier(VOID)
-{
-    LONG Barrier;
-    __asm { xchg Barrier, eax }
-}
-#else
-FORCEINLINE
-VOID
-MemoryBarrier(VOID)
-{
-    LONG Barrier, *Dummy = &Barrier;
-    (VOID)Dummy;
-    __asm__ __volatile__("xchgl %%eax, %[Barrier]" : : [Barrier] "m" (Barrier) : "memory");
-}
-#endif
-#elif defined (_M_AMD64)
-#define MemoryBarrier __faststorefence
-#elif defined(_M_PPC)
-#define MemoryBarrier()
-#elif defined(_M_ARM)
-#define MemoryBarrier()
-#elif defined(_M_ARM64)
-#define MemoryBarrier()
-#else
-#error Unknown architecture
-#endif
-
 #if defined(_M_IX86) || defined(_M_AMD64)
 
 #define YieldProcessor _mm_pause