prov/opx: Add RISC-V support

Signed-off-by: Chris Taylor <ctaylor@tactcomplabs.com>
swelch · Feb 21, 2024 · c52e280 · c52e280
1 parent 7f23127
commit c52e280
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 5 deletions.
diff --git a/prov/opx/configure.m4 b/prov/opx/configure.m4
@@ -49,7 +49,7 @@ AC_DEFUN([FI_OPX_CONFIGURE],[
 	dnl and is not supported for non-x86 processors.
 	AS_IF([test "x$macos" = "x1"],[opx_happy=0],
 		[test "x$freebsd" = "x1"],[opx_happy=0],
-		[test x$host_cpu != xx86_64],[opx_happy=0],
+		[test x$host_cpu != xx86_64 && test x$host_cpu != xriscv && test x$host_cpu != xriscv64],[opx_happy=0],
 		[test x"$enable_opx" != x"no"],[
 
 		AC_MSG_CHECKING([for opx provider])

diff --git a/prov/opx/include/rdma/opx/fi_opx_compiler.h b/prov/opx/include/rdma/opx/fi_opx_compiler.h
@@ -59,14 +59,33 @@
 
 static inline void fi_opx_compiler_msync_writes()
 {
-	asm volatile ("sfence" :  :  : "memory");
+#if defined(__riscv)
+
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+ 	asm volatile ("fence ow,ow" :  :  : "memory");
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
+ 	asm volatile ("sfence" :  :  : "memory");
+#endif
 }
 
 static inline void fi_opx_compiler_msync_reads()
 {
-	asm volatile ("lfence" :  :  : "memory");
-}
+#if defined(__riscv)
 
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+	asm volatile ("fence ir,ir" :  :  : "memory");
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
+ 	asm volatile ("lfence" :  :  : "memory");
+#endif
+}
 
 #define fi_opx_compiler_barrier()		__asm__ __volatile__ ( "" ::: "memory" )
 
@@ -87,28 +106,61 @@ void fi_opx_compiler_store_u64(volatile uint64_t * const variable, const uint64_
 static inline
 void fi_opx_compiler_inc_u64(volatile uint64_t * const variable)
 {
+#if defined(__riscv)
+
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+        (*variable) += 1;
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
 	__asm__ __volatile__ ("lock ; incq %0"
 				: "=m" (*variable)
 				: "m" (*variable));
+#endif
 	return;
 }
 
 static inline
 uint64_t fi_opx_compiler_fetch_and_inc_u64(volatile uint64_t * const variable)
 {
 	uint64_t value = 1;
+#if defined(__riscv)
+
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+	const uint64_t tmp = (*variable);
+	(*variable) = value;
+	value = tmp;
+	(*variable) = (*variable) + value;
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
 	__asm__ __volatile__ ("lock ; xadd %0,%1"
 				: "=r" (value), "=m" (*variable)
 				: "0" (value), "m" (*variable));
+#endif
 	return value;
 }
 
 static inline
 void fi_opx_compiler_dec_u64(volatile uint64_t * const variable)
 {
+#if defined(__riscv)
+
+#if defined(__riscv_xlen) && (__riscv_xlen == 64)
+	(*variable) -= 1;
+#else
+#error "Unsupported CPU type"
+#endif
+
+#else
 	__asm__ __volatile__ ("lock ; decq %0"
 				: "=m" (*variable)
 				: "m" (*variable));
+#endif
 	return;
 }
 

diff --git a/prov/opx/include/rdma/opx/fi_opx_hfi1_packet.h b/prov/opx/include/rdma/opx/fi_opx_hfi1_packet.h
@@ -43,7 +43,12 @@
 #include "rdma/fabric.h"	/* only for 'fi_addr_t' ... which is a typedef to uint64_t */
 #include "rdma/opx/fi_opx_addr.h"
 
-
+#if defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64)
+#ifndef PAGE_SIZE
+/* 4K pages default */
+#define PAGE_SIZE 4096
+#endif
+#endif
 
 #define FI_OPX_ADDR_SEP_RX_MAX			(4)
 #define FI_OPX_HFI1_PACKET_MTU			(8192)

diff --git a/prov/opx/include/rdma/opx/fi_opx_timer.h b/prov/opx/include/rdma/opx/fi_opx_timer.h
@@ -62,6 +62,14 @@ static inline uint64_t fi_opx_timer_get_cycles()
 	cycles = ((uint64_t)a) | (((uint64_t)d) << 32);
 	return cycles;
 }
+#elif defined(__riscv) && defined(__riscv_xlen) && (__riscv_xlen == 64)
+__attribute__((always_inline))
+static inline uint64_t fi_opx_timer_get_cycles()
+{
+        uint64_t dst = 0;
+        asm volatile ("rdcycle %0" : "=r" (dst) );
+        return dst;
+}
 #else
 #error "Cycle timer not defined for this platform"
 #endif