From 7047f4804047c074a6fc56edd04486b3c1034ff6 Mon Sep 17 00:00:00 2001 From: mahailiang Date: Thu, 26 Dec 2024 10:26:15 +0800 Subject: [PATCH] add sw_64 support --- config/opal_config_asm.m4 | 5 + opal/include/opal/sys/Makefile.am | 1 + opal/include/opal/sys/architecture.h | 1 + opal/include/opal/sys/atomic.h | 2 + opal/include/opal/sys/cma.h | 4 + opal/include/opal/sys/sw_64/Makefile.am | 23 ++ opal/include/opal/sys/sw_64/atomic.h | 379 +++++++++++++++++ .../pmix3x/pmix/config/pmix_config_asm.m4 | 5 + .../pmix/src/atomics/sys/Makefile.include | 1 + .../pmix/src/atomics/sys/architecture.h | 1 + .../pmix/pmix3x/pmix/src/atomics/sys/atomic.h | 2 + .../pmix/pmix3x/pmix/src/atomics/sys/cma.h | 3 + .../src/atomics/sys/sw_64/Makefile.include | 23 ++ .../pmix/src/atomics/sys/sw_64/atomic.h | 380 ++++++++++++++++++ 14 files changed, 830 insertions(+) create mode 100644 opal/include/opal/sys/sw_64/Makefile.am create mode 100644 opal/include/opal/sys/sw_64/atomic.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sw_64/Makefile.include create mode 100644 opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sw_64/atomic.h diff --git a/config/opal_config_asm.m4 b/config/opal_config_asm.m4 index d8640e0..8698b06 100644 --- a/config/opal_config_asm.m4 +++ b/config/opal_config_asm.m4 @@ -1049,6 +1049,11 @@ AC_DEFUN([OPAL_CONFIG_ASM],[ OPAL_GCC_INLINE_ASSIGN="" OPAL_ASM_SUPPORT_64BIT=0 case "${host}" in + sw_64-*) + opal_cv_asm_arch="SW_64" + OPAL_ASM_SUPPORT_64BIT=1 + OPAL_GCC_INLINE_ASSIGN='"mov 0,%0" : "=&r"(ret)' + ;; x86_64-*x32) opal_cv_asm_arch="X86_64" OPAL_ASM_SUPPORT_64BIT=1 diff --git a/opal/include/opal/sys/Makefile.am b/opal/include/opal/sys/Makefile.am index 9387ed6..9d11693 100644 --- a/opal/include/opal/sys/Makefile.am +++ b/opal/include/opal/sys/Makefile.am @@ -37,5 +37,6 @@ include opal/sys/arm64/Makefile.am include opal/sys/ia32/Makefile.am include opal/sys/powerpc/Makefile.am include opal/sys/sparcv9/Makefile.am +include opal/sys/sw_64/Makefile.am include opal/sys/sync_builtin/Makefile.am include opal/sys/gcc_builtin/Makefile.am diff --git a/opal/include/opal/sys/architecture.h b/opal/include/opal/sys/architecture.h index 8a9fc53..b279079 100644 --- a/opal/include/opal/sys/architecture.h +++ b/opal/include/opal/sys/architecture.h @@ -45,6 +45,7 @@ #define OPAL_S390 0110 #define OPAL_S390X 0111 #define OPAL_RISCV64 0120 +#define OPAL_SW_64 0121 #define OPAL_BUILTIN_SYNC 0200 #define OPAL_BUILTIN_GCC 0202 #define OPAL_BUILTIN_NO 0203 diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h index 3b165f0..e21de29 100644 --- a/opal/include/opal/sys/atomic.h +++ b/opal/include/opal/sys/atomic.h @@ -181,6 +181,8 @@ enum { #include "opal/sys/sparcv9/atomic.h" #elif OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 #include "opal/sys/sparcv9/atomic.h" +#elif OPAL_ASSEMBLY_ARCH == OPAL_SW_64 +#include "opal/sys/sw_64/atomic.h" #endif #ifndef DOXYGEN diff --git a/opal/include/opal/sys/cma.h b/opal/include/opal/sys/cma.h index 8040ccb..8817eb8 100644 --- a/opal/include/opal/sys/cma.h +++ b/opal/include/opal/sys/cma.h @@ -97,6 +97,10 @@ #define __NR_process_vm_readv 270 #define __NR_process_vm_writev 271 +#if OPAL_ASSEMBLY_ARCH == OPAL_SW_64 +#define __NR_process_vm_readv 504 +#define __NR_process_vm_writev 505 + #else #error "Unsupported architecture for process_vm_readv and process_vm_writev syscalls" #endif diff --git a/opal/include/opal/sys/sw_64/Makefile.am b/opal/include/opal/sys/sw_64/Makefile.am new file mode 100644 index 0000000..afe7abc --- /dev/null +++ b/opal/include/opal/sys/sw_64/Makefile.am @@ -0,0 +1,23 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2008 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This makefile.am does not stand on its own - it is included from opal/include/Makefile.am + +headers += \ + opal/sys/sw_64/atomic.h + diff --git a/opal/include/opal/sys/sw_64/atomic.h b/opal/include/opal/sys/sw_64/atomic.h new file mode 100644 index 0000000..93d6c8c --- /dev/null +++ b/opal/include/opal/sys/sw_64/atomic.h @@ -0,0 +1,379 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2010 IBM Corporation. All rights reserved. + * Copyright (c) 2010 ARM ltd. All rights reserved. + * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#if !defined(OPAL_SYS_ARCH_ATOMIC_H) + +#define OPAL_SYS_ARCH_ATOMIC_H 1 + +#if OPAL_GCC_INLINE_ASSEMBLY + +#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 +#define OPAL_HAVE_ATOMIC_SWAP_32 1 +#define OPAL_HAVE_ATOMIC_MATH_32 1 +#define OPAL_HAVE_ATOMIC_MATH_64 1 +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 +#define OPAL_HAVE_ATOMIC_SWAP_64 1 +#define OPAL_HAVE_ATOMIC_ADD_32 1 +#define OPAL_HAVE_ATOMIC_AND_32 1 +#define OPAL_HAVE_ATOMIC_OR_32 1 +#define OPAL_HAVE_ATOMIC_XOR_32 1 +#define OPAL_HAVE_ATOMIC_SUB_32 1 +#define OPAL_HAVE_ATOMIC_ADD_64 1 +#define OPAL_HAVE_ATOMIC_AND_64 1 +#define OPAL_HAVE_ATOMIC_OR_64 1 +#define OPAL_HAVE_ATOMIC_XOR_64 1 +#define OPAL_HAVE_ATOMIC_SUB_64 1 + +#define MB() __asm__ __volatile__ ("memb" : : : "memory") + +/********************************************************************** + * + * Memory Barriers + * + *********************************************************************/ + +static inline void opal_atomic_mb (void) +{ + MB(); +} + +static inline void opal_atomic_rmb (void) +{ + MB(); +} + +static inline void opal_atomic_wmb (void) +{ + MB(); +} + +static inline void opal_atomic_isync (void) +{ +} + +/********************************************************************** + * + * Atomic math operations + * + *********************************************************************/ + +static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +{ + int32_t prev, tmp, cmp; + bool ret; + unsigned long ptr; + + __asm__ __volatile__ (" ldi %1, %4 \n"//ptr = addr + "1: lldw %0, 0(%1) \n"//prev = *ptr + " cmpeq %0, %5, %3 \n" + " wr_f %3 \n"//if(prev != oldval) cmp = 0; else cmp = 1 + " mov %6, %2 \n"//tmp = newval + " lstw %2, 0(%1) \n"//*ptr = tmp + " rd_f %2 \n" + " beq %3, 2f \n"//if(cmp == 0) goto 2 + " beq %2, 1b \n"//if(tmp == 0) goto 1, if store option failed goto 1 + "2: \n" + : "=&r" (prev), "=&r" (ptr), "=&r" (tmp), "=&r" (cmp) + : "r" (addr), "r" (*oldval), "r" (newval) + : "memory"); + + ret = (prev == *oldval); + *oldval = prev; + return ret; +} + +static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval) +{ + int32_t ret, tmp; + unsigned long ptr; + + __asm__ __volatile__ (" ldi %2, %3 \n"//ptr = addr + "1: lldw %0, 0(%2) \n"//ret = *ptr + " ldi %1, 1 \n"//tmp = 1 + " wr_f %1 \n" + " mov %4, %1 \n"//tmp = newval + " lstw %1, 0(%2) \n"//*ptr = tmp(with atomic check) + " rd_f %1 \n"//tmp = status value + " beq %1, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (ret), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (newval) + : "memory"); + + return ret; +} + +#define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32 +#define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32 + +static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +{ + int64_t prev, tmp, cmp; + bool ret; + unsigned long ptr; + + __asm__ __volatile__ (" ldi %1, %4 \n"//ptr = addr + "1: lldl %0, 0(%1) \n"//prev = *ptr + " cmpeq %0, %5, %3 \n" + " wr_f %3 \n"//if(prev != oldval) cmp = 0; else cmp = 1 + " mov %6, %2 \n"//tmp = newval + " lstl %2, 0(%1) \n"//*ptr = tmp + " rd_f %2 \n" + " beq %3, 2f \n"//if(cmp == 0) goto 2 + " beq %2, 1b \n"//if(tmp == 0) goto 1, if store option failed goto 1 + "2: \n" + : "=&r" (prev), "=&r" (ptr), "=&r" (tmp), "=&r" (cmp) + : "r" (addr), "r" (*oldval), "r" (newval) + : "memory"); + + ret = (prev == *oldval); + *oldval = prev; + return ret; +} + +static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval) +{ + int64_t ret, tmp; + unsigned long ptr; + + __asm__ __volatile__ (" ldi %2, %3 \n"//ptr = addr + "1: lldl %0, 0(%2) \n"//ret = *ptr + " ldi %1, 1 \n"//tmp = 1 + " wr_f %1 \n" + " mov %4, %1 \n"//tmp = newval + " lstl %1, 0(%2) \n"//*ptr = tmp(with atomic check) + " rd_f %1 \n"//tmp = status value + " beq %1, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (ret), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (newval) + : "memory"); + + return ret; +} + +#define opal_atomic_compare_exchange_strong_acq_64 opal_atomic_compare_exchange_strong_64 +#define opal_atomic_compare_exchange_strong_rel_64 opal_atomic_compare_exchange_strong_64 + +static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " addw %1, %5, %0 \n"//newval = old + value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " subw %1, %5, %0 \n"//newval = old - value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " and %1, %5, %0 \n"//newval = old & value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " bis %1, %5, %0 \n"//newval = old | value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " xor %1, %5, %0 \n"//newval = old ^ value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " addl %1, %5, %0 \n"//newval = old + value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " subl %1, %5, %0 \n"//newval = old - value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " and %1, %5, %0 \n"//newval = old & value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " bis %1, %5, %0 \n"//newval = old | value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " xor %1, %5, %0 \n"//newval = old ^ value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +#endif /* OPAL_GCC_INLINE_ASSEMBLY */ + +#endif /* ! OPAL_SYS_ARCH_ATOMIC_H */ diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_config_asm.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_config_asm.m4 index 64e286c..2df06c7 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix_config_asm.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_config_asm.m4 @@ -1051,6 +1051,11 @@ AC_DEFUN([PMIX_CONFIG_ASM],[ PMIX_ASM_SUPPORT_64BIT=0 fi case "${host}" in + sw_64-*) + pmix_cv_asm_arch="SW_64" + PMIX_ASM_SUPPORT_64BIT=1 + PMIX_GCC_INLINE_ASSIGN='"mov 0,%0" : "=&r"(ret)' + ; x86_64-*x32) pmix_cv_asm_arch="X86_64" PMIX_ASM_SUPPORT_64BIT=1 diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/Makefile.include b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/Makefile.include index c7070dd..c8c5119 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/Makefile.include +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/Makefile.include @@ -38,4 +38,5 @@ include atomics/sys/arm/Makefile.include include atomics/sys/arm64/Makefile.include include atomics/sys/ia32/Makefile.include include atomics/sys/powerpc/Makefile.include +include atomics/sys/sw_64/Makefile.include include atomics/sys/gcc_builtin/Makefile.include diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/architecture.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/architecture.h index c69e540..8bc5d16 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/architecture.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/architecture.h @@ -37,6 +37,7 @@ #define PMIX_POWERPC64 0051 #define PMIX_ARM 0100 #define PMIX_ARM64 0101 +#define PMIX_SW_64 0121 #define PMIX_BUILTIN_GCC 0202 #define PMIX_BUILTIN_NO 0203 #define PMIX_BUILTIN_C11 0204 diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic.h index 783870c..bc251c0 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic.h @@ -178,6 +178,8 @@ enum { #include "src/atomics/sys/powerpc/atomic.h" #elif PMIX_ASSEMBLY_ARCH == PMIX_POWERPC64 #include "src/atomics/sys/powerpc/atomic.h" +#elif PMIX_ASSEMBLY_ARCH == PMIX_SW_64 +#include "src/atomics/sys/sw_64/atomic.h" #endif #ifndef DOXYGEN diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/cma.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/cma.h index 8a5bc2b..bb4b052 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/cma.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/cma.h @@ -62,6 +62,9 @@ #define __NR_process_vm_readv 270 #define __NR_process_vm_writev 271 +#if PMIX_ASSEMBLY_ARCH == PMIX_SW_64 +#define __NR_process_vm_readv 504 +#define __NR_process_vm_writev 505 #else #error "Unsupported architecture for process_vm_readv and process_vm_writev syscalls" diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sw_64/Makefile.include b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sw_64/Makefile.include new file mode 100644 index 0000000..f943b4f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sw_64/Makefile.include @@ -0,0 +1,23 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2008 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2017 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This makefile.am does not stand on its own - it is included from pmix/include/Makefile.am + +headers += \ + atomics/sys/sw_64/atomic.h diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sw_64/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sw_64/atomic.h new file mode 100644 index 0000000..d592ac2 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sw_64/atomic.h @@ -0,0 +1,380 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2010 IBM Corporation. All rights reserved. + * Copyright (c) 2010 ARM ltd. All rights reserved. + * Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#if !defined(PMIX_SYS_ARCH_ATOMIC_H) + +#define PMIX_SYS_ARCH_ATOMIC_H 1 + +#if PMIX_GCC_INLINE_ASSEMBLY + +#define PMIX_HAVE_ATOMIC_MEM_BARRIER 1 +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 +#define PMIX_HAVE_ATOMIC_SWAP_32 1 +#define PMIX_HAVE_ATOMIC_MATH_32 1 +#define PMIX_HAVE_ATOMIC_MATH_64 1 +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 +#define PMIX_HAVE_ATOMIC_SWAP_64 1 +#define PMIX_HAVE_ATOMIC_ADD_32 1 +#define PMIX_HAVE_ATOMIC_AND_32 1 +#define PMIX_HAVE_ATOMIC_OR_32 1 +#define PMIX_HAVE_ATOMIC_XOR_32 1 +#define PMIX_HAVE_ATOMIC_SUB_32 1 +#define PMIX_HAVE_ATOMIC_ADD_64 1 +#define PMIX_HAVE_ATOMIC_AND_64 1 +#define PMIX_HAVE_ATOMIC_OR_64 1 +#define PMIX_HAVE_ATOMIC_XOR_64 1 +#define PMIX_HAVE_ATOMIC_SUB_64 1 + +#define PMIXMB() __asm__ __volatile__ ("memb" : : : "memory") + +/********************************************************************** + * + * Memory Barriers + * + *********************************************************************/ + +static inline void pmix_atomic_mb (void) +{ + PMIXMB(); +} + +static inline void pmix_atomic_rmb (void) +{ + PMIXMB(); +} + +static inline void pmix_atomic_wmb (void) +{ + PMIXMB(); +} + +static inline void pmix_atomic_isync (void) +{ +} + +/********************************************************************** + * + * Atomic math operations + * + *********************************************************************/ + +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) +{ + int32_t prev, tmp, cmp; + bool ret; + unsigned long ptr; + + __asm__ __volatile__ (" ldi %1, %4 \n"//ptr = addr + "1: lldw %0, 0(%1) \n"//prev = *ptr + " cmpeq %0, %5, %3 \n" + " wr_f %3 \n"//if(prev != oldval) cmp = 0; else cmp = 1 + " mov %6, %2 \n"//tmp = newval + " lstw %2, 0(%1) \n"//*ptr = tmp + " rd_f %2 \n" + " beq %3, 2f \n"//if(cmp == 0) goto 2 + " beq %2, 1b \n"//if(tmp == 0) goto 1, if store option failed goto 1 + "2: \n" + : "=&r" (prev), "=&r" (ptr), "=&r" (tmp), "=&r" (cmp) + : "r" (addr), "r" (*oldval), "r" (newval) + : "memory"); + + ret = (prev == *oldval); + *oldval = prev; + return ret; +} + +static inline int32_t pmix_atomic_swap_32(pmix_atomic_int32_t *addr, int32_t newval) +{ + int32_t ret, tmp; + unsigned long ptr; + + __asm__ __volatile__ (" ldi %2, %3 \n"//ptr = addr + "1: lldw %0, 0(%2) \n"//ret = *ptr + " ldi %1, 1 \n"//tmp = 1 + " wr_f %1 \n" + " mov %4, %1 \n"//tmp = newval + " lstw %1, 0(%2) \n"//*ptr = tmp(with atomic check) + " rd_f %1 \n"//tmp = status value + " beq %1, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (ret), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (newval) + : "memory"); + + return ret; +} + +#define pmix_atomic_compare_exchange_strong_acq_32 pmix_atomic_compare_exchange_strong_32 +#define pmix_atomic_compare_exchange_strong_rel_32 pmix_atomic_compare_exchange_strong_32 + +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) +{ + int64_t prev, tmp, cmp; + bool ret; + unsigned long ptr; + + __asm__ __volatile__ (" ldi %1, %4 \n"//ptr = addr + "1: lldl %0, 0(%1) \n"//prev = *ptr + " cmpeq %0, %5, %3 \n" + " wr_f %3 \n"//if(prev != oldval) cmp = 0; else cmp = 1 + " mov %6, %2 \n"//tmp = newval + " lstl %2, 0(%1) \n"//*ptr = tmp + " rd_f %2 \n" + " beq %3, 2f \n"//if(cmp == 0) goto 2 + " beq %2, 1b \n"//if(tmp == 0) goto 1, if store option failed goto 1 + "2: \n" + : "=&r" (prev), "=&r" (ptr), "=&r" (tmp), "=&r" (cmp) + : "r" (addr), "r" (*oldval), "r" (newval) + : "memory"); + + ret = (prev == *oldval); + *oldval = prev; + return ret; +} + +static inline int64_t pmix_atomic_swap_64 (pmix_atomic_int64_t *addr, int64_t newval) +{ + int64_t ret, tmp; + unsigned long ptr; + + __asm__ __volatile__ (" ldi %2, %3 \n"//ptr = addr + "1: lldl %0, 0(%2) \n"//ret = *ptr + " ldi %1, 1 \n"//tmp = 1 + " wr_f %1 \n" + " mov %4, %1 \n"//tmp = newval + " lstl %1, 0(%2) \n"//*ptr = tmp(with atomic check) + " rd_f %1 \n"//tmp = status value + " beq %1, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (ret), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (newval) + : "memory"); + + return ret; +} + +#define pmix_atomic_compare_exchange_strong_acq_64 pmix_atomic_compare_exchange_strong_64 +#define pmix_atomic_compare_exchange_strong_rel_64 pmix_atomic_compare_exchange_strong_64 + +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " addw %1, %5, %0 \n"//newval = old + value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " subw %1, %5, %0 \n"//newval = old - value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " and %1, %5, %0 \n"//newval = old & value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " bis %1, %5, %0 \n"//newval = old | value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldw %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " xor %1, %5, %0 \n"//newval = old ^ value + " lstw %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " addl %1, %5, %0 \n"//newval = old + value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " subl %1, %5, %0 \n"//newval = old - value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " and %1, %5, %0 \n"//newval = old & value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " bis %1, %5, %0 \n"//newval = old | value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t newval, old, tmp; + unsigned long ptr; + + __asm__ __volatile__(" ldi %3, %4 \n"//ptr = addr + "1: lldl %1, 0(%3) \n"//old = *ptr + " ldi %2, 1 \n"//tmp = 1 + " wr_f %2 \n" + " xor %1, %5, %0 \n"//newval = old ^ value + " lstl %0, 0(%3) \n"//*ptr = newval(with atomic check) + " rd_f %2 \n"//tmp = status value + " beq %2, 1b \n"//if(tmp == 0) goto 1, start again on atomic error + : "=&r" (newval), "=&r" (old), "=&r" (tmp), "=&r" (ptr) + : "r" (addr), "r" (value) + : "memory"); + + return old; +} + +#endif /* PMIX_GCC_INLINE_ASSEMBLY */ + +#endif /* ! PMIX_SYS_ARCH_ATOMIC_H */ -- 2.27.0