mirror of
https://github.com/sfilippone/psblas3.git
synced 2026-10-24 14:51:49 +00:00
[UPDATE] Transient version usefull for debug on local server
This commit is contained in:
Stack-1
@@ -9,12 +9,14 @@ program psb_comm_cg_test
|
||||
implicit none
|
||||
|
||||
type(psb_ctxt_type) :: ctxt
|
||||
type(psb_ctxt_type) :: desc_ctxt
|
||||
type(psb_dspmat_type) :: a
|
||||
type(psb_desc_type) :: desc_a
|
||||
type(psb_d_vect_type) :: b, x
|
||||
type(psb_dprec_type) :: prec
|
||||
|
||||
integer(psb_ipk_) :: info, iam, np
|
||||
integer(psb_ipk_) :: desc_me, desc_np
|
||||
integer(psb_ipk_) :: idim, itmax, itrace, istop, iter
|
||||
integer(psb_ipk_) :: scheme_idx, prec_idx, rep, nrep, nwarm
|
||||
integer(psb_ipk_), parameter :: n_schemes=3, n_precs=2
|
||||
@@ -42,7 +44,7 @@ program psb_comm_cg_test
|
||||
nrep = 5
|
||||
nwarm = 1
|
||||
! Keep itrace positive to avoid modulo-by-zero paths in convergence logging.
|
||||
itrace = 1
|
||||
itrace = 0
|
||||
istop = 2
|
||||
eps = 1.d-6
|
||||
scheme_type = (/ psb_comm_isend_irecv_, psb_comm_ineighbor_alltoallv_, &
|
||||
@@ -127,6 +129,14 @@ program psb_comm_cg_test
|
||||
! call probe_ieee('after psb_d_gen_pde3d')
|
||||
if (info /= psb_success_) goto 9999
|
||||
|
||||
! desc_ctxt = desc_a%get_context()
|
||||
! call psb_info(desc_ctxt, desc_me, desc_np)
|
||||
! if (desc_np == -1) then
|
||||
! info = psb_err_context_error_
|
||||
! write(psb_err_unit,*) 'Invalid descriptor context after psb_d_gen_pde3d'
|
||||
! goto 9999
|
||||
! end if
|
||||
|
||||
do prec_idx = 1, n_precs
|
||||
do scheme_idx = 1, n_schemes
|
||||
do rep = 1, nrep
|
||||
@@ -169,14 +179,6 @@ program psb_comm_cg_test
|
||||
setup_time(prec_idx,scheme_idx,rep) = prec_init_time(prec_idx,scheme_idx,rep) + &
|
||||
& prec_bld_time(prec_idx,scheme_idx,rep) + comm_set_time(prec_idx,scheme_idx,rep)
|
||||
|
||||
do iter = 1, nwarm
|
||||
call psb_geaxpby(dzero,b,dzero,x,desc_a,info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
call psb_krylov('CG',a,prec,b,x,eps,desc_a,info,&
|
||||
& itmax=itmax,itrace=itrace,istop=istop)
|
||||
if (info /= psb_success_) goto 9999
|
||||
end do
|
||||
|
||||
call psb_geaxpby(dzero,b,dzero,x,desc_a,info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
|
||||
@@ -212,20 +214,20 @@ program psb_comm_cg_test
|
||||
final_error(prec_idx,scheme_idx,rep) = err
|
||||
solve_info(prec_idx,scheme_idx,rep) = info
|
||||
|
||||
if (iam == psb_root_) then
|
||||
select type(ch => x%v%comm_handle)
|
||||
type is(psb_comm_neighbor_handle)
|
||||
write(psb_out_unit,'("DIAG_COMM scheme=",a,", prec=",a,", rep=",i0)') &
|
||||
& trim(scheme_name(scheme_idx)), trim(prec_name(prec_idx)), rep
|
||||
write(psb_out_unit,'("DIAG_COMM counters: init=",i0,", start=",i0,", wait=",i0,", realloc=",i0)') &
|
||||
& ch%diag_init_calls, ch%diag_start_calls, ch%diag_wait_calls, &
|
||||
& ch%diag_buffer_reallocs
|
||||
write(psb_out_unit,'("DIAG_COMM state: ready=",l1,", bsz=",i0)') &
|
||||
& ch%persistent_request_ready, ch%persistent_buffer_size
|
||||
class default
|
||||
continue
|
||||
end select
|
||||
end if
|
||||
! if (iam == psb_root_) then
|
||||
! select type(ch => x%v%comm_handle)
|
||||
! type is(psb_comm_neighbor_handle)
|
||||
! write(psb_out_unit,'("DIAG_COMM scheme=",a,", prec=",a,", rep=",i0)') &
|
||||
! & trim(scheme_name(scheme_idx)), trim(prec_name(prec_idx)), rep
|
||||
! write(psb_out_unit,'("DIAG_COMM counters: init=",i0,", start=",i0,", wait=",i0,", realloc=",i0)') &
|
||||
! & ch%diag_init_calls, ch%diag_start_calls, ch%diag_wait_calls, &
|
||||
! & ch%diag_buffer_reallocs
|
||||
! write(psb_out_unit,'("DIAG_COMM state: ready=",l1,", bsz=",i0)') &
|
||||
! & ch%persistent_request_ready, ch%persistent_buffer_size
|
||||
! class default
|
||||
! continue
|
||||
! end select
|
||||
! end if
|
||||
|
||||
if (info /= psb_success_) goto 9999
|
||||
end do
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
#SBATCH --job-name=psb_spmv_overlap
|
||||
#SBATCH --partition=boost_usr_prod
|
||||
#SBATCH --time=01:00:00
|
||||
#SBATCH --nodes=2
|
||||
#SBATCH --ntasks=64
|
||||
#SBATCH --ntasks-per-node=32
|
||||
#SBATCH --cpus-per-task=1
|
||||
#SBATCH --threads-per-core=1
|
||||
#SBATCH --gpus-per-node=4
|
||||
#SBATCH --export=ALL
|
||||
#SBATCH -A CNHPC_1736213
|
||||
#SBATCH --output=psb_spmv_overlap_%j.out
|
||||
#SBATCH --error=psb_spmv_overlap_%j.err
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Environment tuned like the existing comm test script.
|
||||
export UCX_VFS_ENABLE=n
|
||||
export UCX_VFS_USE_FUSE=n
|
||||
export UCX_STATS_DEST=none
|
||||
export UCX_LOG_LEVEL=error
|
||||
export UCX_TLS=dc,sm,self
|
||||
export UCX_NET_DEVICES=mlx5_0:1
|
||||
export OMPI_MCA_coll=^hcoll,han
|
||||
export OMPI_MCA_coll_hcoll_enable=0
|
||||
export UCX_MEMTYPE_CACHE=n
|
||||
export UCX_CLOSE_TIMEOUT=10s
|
||||
|
||||
EXEC=./test/comm/spmv/runs/spmv_overlap
|
||||
IDIM_LIST=${IDIM_LIST:-"20 40 60 80 100 140 180 220 260 300"}
|
||||
TIMES=${TIMES:-100}
|
||||
BUILD_IF_MISSING=${BUILD_IF_MISSING:-1}
|
||||
|
||||
if [[ ! -x "$EXEC" ]]; then
|
||||
echo "Executable not found: $EXEC" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Running SpMV overlap comm test"
|
||||
echo " EXEC=$EXEC"
|
||||
echo " IDIM_LIST=$IDIM_LIST"
|
||||
echo " TIMES=$TIMES"
|
||||
echo " BUILD_IF_MISSING=$BUILD_IF_MISSING"
|
||||
|
||||
for IDIM in $IDIM_LIST; do
|
||||
echo ""
|
||||
echo "=== Running IDIM=$IDIM TIMES=$TIMES ==="
|
||||
IDIM="$IDIM" TIMES="$TIMES" srun --exclusive -N2 -n64 "$EXEC"
|
||||
done
|
||||
@@ -524,29 +524,47 @@ contains
|
||||
real(psb_dpk_) :: alpha, beta
|
||||
|
||||
type(psb_dspmat_type) :: a
|
||||
type(psb_d_vect_type) :: x_baseline, x_neighbor, x_persistent
|
||||
type(psb_d_vect_type) :: y_baseline, y_neighbor, y_persistent
|
||||
type(psb_d_vect_type) :: x_isend, x_neighbor, x_persistent
|
||||
type(psb_d_vect_type) :: y_ov_isend, y_ov_neighbor, y_ov_persistent
|
||||
type(psb_d_vect_type) :: y_no_isend, y_no_neighbor, y_no_persistent
|
||||
type(psb_desc_type) :: desc_a
|
||||
|
||||
character(len=:), allocatable :: output_file_name
|
||||
character(len=32) :: idim_str
|
||||
character(len=64) :: env_buf
|
||||
real(psb_dpk_), allocatable :: x_global(:), y_global(:)
|
||||
integer(psb_ipk_) :: my_rank, np, info, err_act
|
||||
integer :: env_len, env_status, ios
|
||||
integer(psb_ipk_) :: n_global, idim
|
||||
integer(psb_ipk_) :: i, times
|
||||
real(psb_dpk_) :: t0, t1, dt
|
||||
real(psb_dpk_) :: tsum_baseline, tsum_neighbor, tsum_persistent
|
||||
real(psb_dpk_) :: err_bn, err_bp, tol
|
||||
logical :: tnd
|
||||
real(psb_dpk_) :: t_ov_isend, t_ov_neighbor, t_ov_persistent
|
||||
real(psb_dpk_) :: t_no_isend, t_no_neighbor, t_no_persistent
|
||||
real(psb_dpk_) :: err_isend, err_neighbor, err_persistent, tol
|
||||
real(psb_dpk_) :: avg_ov, avg_no, speedup, gain_pct
|
||||
|
||||
info = psb_success_
|
||||
tol = 1.0d-10
|
||||
times = 100
|
||||
tsum_baseline = 0.0_psb_dpk_
|
||||
tsum_neighbor = 0.0_psb_dpk_
|
||||
tsum_persistent = 0.0_psb_dpk_
|
||||
tnd = .false.
|
||||
t_ov_isend = 0.0_psb_dpk_
|
||||
t_ov_neighbor = 0.0_psb_dpk_
|
||||
t_ov_persistent = 0.0_psb_dpk_
|
||||
t_no_isend = 0.0_psb_dpk_
|
||||
t_no_neighbor = 0.0_psb_dpk_
|
||||
t_no_persistent = 0.0_psb_dpk_
|
||||
idim = 10
|
||||
call psb_erractionsave(err_act)
|
||||
|
||||
call get_environment_variable('IDIM', env_buf, length=env_len, status=env_status)
|
||||
if ((env_status == 0) .and. (env_len > 0)) then
|
||||
read(env_buf(1:env_len), *, iostat=ios) idim
|
||||
if ((ios /= 0) .or. (idim < 2)) idim = 10
|
||||
end if
|
||||
|
||||
call get_environment_variable('TIMES', env_buf, length=env_len, status=env_status)
|
||||
if ((env_status == 0) .and. (env_len > 0)) then
|
||||
read(env_buf(1:env_len), *, iostat=ios) times
|
||||
if ((ios /= 0) .or. (times < 1)) times = 100
|
||||
end if
|
||||
|
||||
n_global = idim * idim * idim
|
||||
alpha = done
|
||||
beta = dzero
|
||||
@@ -555,7 +573,7 @@ contains
|
||||
|
||||
|
||||
call psb_barrier(ctxt)
|
||||
call psb_d_gen_pde3d(ctxt,idim,a,y_baseline,x_baseline,desc_a,"CSR",info,partition=1)
|
||||
call psb_d_gen_pde3d(ctxt,idim,a,y_ov_isend,x_isend,desc_a,"CSR",info,partition=1)
|
||||
|
||||
if (info /= psb_success_) goto 9999
|
||||
call psb_barrier(ctxt)
|
||||
@@ -571,96 +589,200 @@ contains
|
||||
|
||||
call psb_geall(x_neighbor, desc_a, info)
|
||||
call psb_geall(x_persistent, desc_a, info)
|
||||
call psb_geall(y_neighbor, desc_a, info)
|
||||
call psb_geall(y_persistent, desc_a, info)
|
||||
call psb_geall(y_ov_neighbor, desc_a, info)
|
||||
call psb_geall(y_ov_persistent, desc_a, info)
|
||||
call psb_geall(y_no_isend, desc_a, info)
|
||||
call psb_geall(y_no_neighbor, desc_a, info)
|
||||
call psb_geall(y_no_persistent, desc_a, info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
|
||||
call psb_scatter(x_global, x_baseline, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(x_global, x_isend, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(x_global, x_neighbor, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(x_global, x_persistent, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_baseline, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_neighbor, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_persistent, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_ov_isend, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_ov_neighbor, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_ov_persistent, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_no_isend, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_no_neighbor, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_no_persistent, desc_a, info, root=psb_root_)
|
||||
if (info /= psb_success_) goto 9999
|
||||
|
||||
! Set communication schemes on the x vectors used by psb_spmm.
|
||||
call psb_comm_set(psb_comm_isend_irecv_, x_baseline%v%comm_handle, info)
|
||||
call psb_comm_set(psb_comm_isend_irecv_, x_isend%v%comm_handle, info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
call psb_comm_set(psb_comm_ineighbor_alltoallv_, x_neighbor%v%comm_handle, info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
call psb_comm_set(psb_comm_persistent_ineighbor_alltoallv_, x_persistent%v%comm_handle, info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
|
||||
! Warm-up all schemes once.
|
||||
call psb_spmm(alpha, a, x_baseline, beta, y_baseline, desc_a, info, doswap=.true.)
|
||||
call psb_spmm(alpha, a, x_neighbor, beta, y_neighbor, desc_a, info, doswap=.true.)
|
||||
call psb_spmm(alpha, a, x_persistent, beta, y_persistent, desc_a, info, doswap=.true.)
|
||||
! Warm-up all schemes once: overlap and non-overlap paths.
|
||||
call psb_spmm(alpha, a, x_isend, beta, y_ov_isend, desc_a, info, doswap=.true.)
|
||||
call psb_halo(x_isend, desc_a, info)
|
||||
call psb_spmm(alpha, a, x_isend, beta, y_no_isend, desc_a, info, doswap=.false.)
|
||||
|
||||
call psb_spmm(alpha, a, x_neighbor, beta, y_ov_neighbor, desc_a, info, doswap=.true.)
|
||||
call psb_halo(x_neighbor, desc_a, info)
|
||||
call psb_spmm(alpha, a, x_neighbor, beta, y_no_neighbor, desc_a, info, doswap=.false.)
|
||||
|
||||
call psb_spmm(alpha, a, x_persistent, beta, y_ov_persistent, desc_a, info, doswap=.true.)
|
||||
call psb_halo(x_persistent, desc_a, info)
|
||||
call psb_spmm(alpha, a, x_persistent, beta, y_no_persistent, desc_a, info, doswap=.false.)
|
||||
if (info /= psb_success_) goto 9999
|
||||
|
||||
! Restore vectors so timed loops start from same initial state.
|
||||
call psb_scatter(x_global, x_baseline, desc_a, info, root=psb_root_)
|
||||
! -----------------------------
|
||||
! isend/irecv scheme
|
||||
! -----------------------------
|
||||
call psb_scatter(x_global, x_isend, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_ov_isend, desc_a, info, root=psb_root_)
|
||||
call psb_barrier(ctxt)
|
||||
t0 = psb_wtime()
|
||||
do i = 1, times
|
||||
call psb_spmm(alpha, a, x_isend, beta, y_ov_isend, desc_a, info, doswap=.true.)
|
||||
end do
|
||||
t1 = psb_wtime()
|
||||
dt = t1 - t0
|
||||
call psb_amx(ctxt, dt)
|
||||
t_ov_isend = dt
|
||||
|
||||
call psb_scatter(x_global, x_isend, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_no_isend, desc_a, info, root=psb_root_)
|
||||
call psb_barrier(ctxt)
|
||||
t0 = psb_wtime()
|
||||
do i = 1, times
|
||||
call psb_halo(x_isend, desc_a, info)
|
||||
call psb_spmm(alpha, a, x_isend, beta, y_no_isend, desc_a, info, doswap=.false.)
|
||||
end do
|
||||
t1 = psb_wtime()
|
||||
dt = t1 - t0
|
||||
call psb_amx(ctxt, dt)
|
||||
t_no_isend = dt
|
||||
|
||||
! -----------------------------
|
||||
! ineighbor_alltoallv scheme
|
||||
! -----------------------------
|
||||
call psb_scatter(x_global, x_neighbor, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_ov_neighbor, desc_a, info, root=psb_root_)
|
||||
call psb_barrier(ctxt)
|
||||
t0 = psb_wtime()
|
||||
do i = 1, times
|
||||
call psb_spmm(alpha, a, x_neighbor, beta, y_ov_neighbor, desc_a, info, doswap=.true.)
|
||||
end do
|
||||
t1 = psb_wtime()
|
||||
dt = t1 - t0
|
||||
call psb_amx(ctxt, dt)
|
||||
t_ov_neighbor = dt
|
||||
|
||||
call psb_scatter(x_global, x_neighbor, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_no_neighbor, desc_a, info, root=psb_root_)
|
||||
call psb_barrier(ctxt)
|
||||
t0 = psb_wtime()
|
||||
do i = 1, times
|
||||
call psb_halo(x_neighbor, desc_a, info)
|
||||
call psb_spmm(alpha, a, x_neighbor, beta, y_no_neighbor, desc_a, info, doswap=.false.)
|
||||
end do
|
||||
t1 = psb_wtime()
|
||||
dt = t1 - t0
|
||||
call psb_amx(ctxt, dt)
|
||||
t_no_neighbor = dt
|
||||
|
||||
! ----------------------------------------
|
||||
! persistent_ineighbor_alltoallv scheme
|
||||
! ----------------------------------------
|
||||
call psb_scatter(x_global, x_persistent, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_baseline, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_neighbor, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_persistent, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_ov_persistent, desc_a, info, root=psb_root_)
|
||||
call psb_barrier(ctxt)
|
||||
t0 = psb_wtime()
|
||||
do i = 1, times
|
||||
call psb_spmm(alpha, a, x_persistent, beta, y_ov_persistent, desc_a, info, doswap=.true.)
|
||||
end do
|
||||
t1 = psb_wtime()
|
||||
dt = t1 - t0
|
||||
call psb_amx(ctxt, dt)
|
||||
t_ov_persistent = dt
|
||||
|
||||
call psb_scatter(x_global, x_persistent, desc_a, info, root=psb_root_)
|
||||
call psb_scatter(y_global, y_no_persistent, desc_a, info, root=psb_root_)
|
||||
call psb_barrier(ctxt)
|
||||
t0 = psb_wtime()
|
||||
do i = 1, times
|
||||
call psb_halo(x_persistent, desc_a, info)
|
||||
call psb_spmm(alpha, a, x_persistent, beta, y_no_persistent, desc_a, info, doswap=.false.)
|
||||
end do
|
||||
t1 = psb_wtime()
|
||||
dt = t1 - t0
|
||||
call psb_amx(ctxt, dt)
|
||||
t_no_persistent = dt
|
||||
|
||||
if (info /= psb_success_) goto 9999
|
||||
|
||||
! Baseline (isend/irecv) overlapped SpMV.
|
||||
t0 = psb_wtime()
|
||||
do i = 1, times
|
||||
call psb_spmm(alpha, a, x_baseline, beta, y_baseline, desc_a, info, doswap=.true.)
|
||||
end do
|
||||
t1 = psb_wtime()
|
||||
dt = t1 - t0
|
||||
call psb_amx(ctxt, dt)
|
||||
tsum_baseline = tsum_baseline + dt
|
||||
|
||||
! Neighbor alltoallv overlapped SpMV.
|
||||
t0 = psb_wtime()
|
||||
do i = 1, times
|
||||
call psb_spmm(alpha, a, x_neighbor, beta, y_neighbor, desc_a, info, doswap=.true.)
|
||||
end do
|
||||
t1 = psb_wtime()
|
||||
dt = t1 - t0
|
||||
call psb_amx(ctxt, dt)
|
||||
tsum_neighbor = tsum_neighbor + dt
|
||||
|
||||
! Persistent-neighbor overlapped SpMV.
|
||||
t0 = psb_wtime()
|
||||
do i = 1, times
|
||||
call psb_spmm(alpha, a, x_persistent, beta, y_persistent, desc_a, info, doswap=.true.)
|
||||
end do
|
||||
t1 = psb_wtime()
|
||||
dt = t1 - t0
|
||||
call psb_amx(ctxt, dt)
|
||||
tsum_persistent = tsum_persistent + dt
|
||||
|
||||
err_bn = maxval(abs(y_baseline%get_vect() - y_neighbor%get_vect()))
|
||||
err_bp = maxval(abs(y_baseline%get_vect() - y_persistent%get_vect()))
|
||||
call psb_amx(ctxt, err_bn)
|
||||
call psb_amx(ctxt, err_bp)
|
||||
err_isend = maxval(abs(y_ov_isend%get_vect() - y_no_isend%get_vect()))
|
||||
err_neighbor = maxval(abs(y_ov_neighbor%get_vect() - y_no_neighbor%get_vect()))
|
||||
err_persistent = maxval(abs(y_ov_persistent%get_vect() - y_no_persistent%get_vect()))
|
||||
call psb_amx(ctxt, err_isend)
|
||||
call psb_amx(ctxt, err_neighbor)
|
||||
call psb_amx(ctxt, err_persistent)
|
||||
|
||||
if (my_rank == 0) then
|
||||
write(psb_out_unit,'(" Avg baseline time : ",es12.5)') tsum_baseline / real(times, psb_dpk_)
|
||||
write(psb_out_unit,'(" Tot baseline time : ",es12.5)') tsum_baseline
|
||||
write(psb_out_unit,'(" Avg neighbor time : ",es12.5)') tsum_neighbor / real(times, psb_dpk_)
|
||||
write(psb_out_unit,'(" Tot neighbor time : ",es12.5)') tsum_neighbor
|
||||
write(psb_out_unit,'(" Avg pers-neigh time: ",es12.5)') tsum_persistent / real(times, psb_dpk_)
|
||||
write(psb_out_unit,'(" Tot pers-neigh time: ",es12.5)') tsum_persistent
|
||||
write(psb_out_unit,'(" Check baseline vs neighbor err = ",es12.5)') err_bn
|
||||
write(psb_out_unit,'(" Check baseline vs persistent err = ",es12.5)') err_bp
|
||||
if ((err_bn > tol) .or. (err_bp > tol)) then
|
||||
write(psb_out_unit,'(/,"SpMV overlap benchmark")')
|
||||
write(psb_out_unit,'(" idim : ",i0)') idim
|
||||
write(psb_out_unit,'(" global unknowns : ",i0)') n_global
|
||||
write(psb_out_unit,'(" repetitions : ",i0)') times
|
||||
write(psb_out_unit,'(" timing metric : max over MPI ranks")')
|
||||
write(psb_out_unit,'(" gain(%) = 100*(1 - overlap/no_overlap)")')
|
||||
|
||||
write(psb_out_unit,'(/,"Scheme: isend_irecv")')
|
||||
avg_ov = t_ov_isend / real(times, psb_dpk_)
|
||||
avg_no = t_no_isend / real(times, psb_dpk_)
|
||||
speedup = t_no_isend / max(t_ov_isend, tiny(done))
|
||||
gain_pct = 100.0_psb_dpk_ * (done - (t_ov_isend / max(t_no_isend, tiny(done))))
|
||||
write(psb_out_unit,'(" total overlap : ",es12.5)') t_ov_isend
|
||||
write(psb_out_unit,'(" total no_overlap : ",es12.5)') t_no_isend
|
||||
write(psb_out_unit,'(" avg overlap : ",es12.5)') avg_ov
|
||||
write(psb_out_unit,'(" avg no_overlap : ",es12.5)') avg_no
|
||||
write(psb_out_unit,'(" speedup (no/ov) : ",f10.4)') speedup
|
||||
write(psb_out_unit,'(" gain (%) : ",f10.4)') gain_pct
|
||||
write(psb_out_unit,'(" overlap vs no_overlap err = ",es12.5)') err_isend
|
||||
|
||||
write(psb_out_unit,'(/,"Scheme: ineighbor_alltoallv")')
|
||||
avg_ov = t_ov_neighbor / real(times, psb_dpk_)
|
||||
avg_no = t_no_neighbor / real(times, psb_dpk_)
|
||||
speedup = t_no_neighbor / max(t_ov_neighbor, tiny(done))
|
||||
gain_pct = 100.0_psb_dpk_ * (done - (t_ov_neighbor / max(t_no_neighbor, tiny(done))))
|
||||
write(psb_out_unit,'(" total overlap : ",es12.5)') t_ov_neighbor
|
||||
write(psb_out_unit,'(" total no_overlap : ",es12.5)') t_no_neighbor
|
||||
write(psb_out_unit,'(" avg overlap : ",es12.5)') avg_ov
|
||||
write(psb_out_unit,'(" avg no_overlap : ",es12.5)') avg_no
|
||||
write(psb_out_unit,'(" speedup (no/ov) : ",f10.4)') speedup
|
||||
write(psb_out_unit,'(" gain (%) : ",f10.4)') gain_pct
|
||||
write(psb_out_unit,'(" overlap vs no_overlap err = ",es12.5)') err_neighbor
|
||||
|
||||
write(psb_out_unit,'(/,"Scheme: persistent_ineighbor_alltoallv")')
|
||||
avg_ov = t_ov_persistent / real(times, psb_dpk_)
|
||||
avg_no = t_no_persistent / real(times, psb_dpk_)
|
||||
speedup = t_no_persistent / max(t_ov_persistent, tiny(done))
|
||||
gain_pct = 100.0_psb_dpk_ * (done - (t_ov_persistent / max(t_no_persistent, tiny(done))))
|
||||
write(psb_out_unit,'(" total overlap : ",es12.5)') t_ov_persistent
|
||||
write(psb_out_unit,'(" total no_overlap : ",es12.5)') t_no_persistent
|
||||
write(psb_out_unit,'(" avg overlap : ",es12.5)') avg_ov
|
||||
write(psb_out_unit,'(" avg no_overlap : ",es12.5)') avg_no
|
||||
write(psb_out_unit,'(" speedup (no/ov) : ",f10.4)') speedup
|
||||
write(psb_out_unit,'(" gain (%) : ",f10.4)') gain_pct
|
||||
write(psb_out_unit,'(" overlap vs no_overlap err = ",es12.5)') err_persistent
|
||||
|
||||
if ((err_isend > tol) .or. (err_neighbor > tol) .or. (err_persistent > tol)) then
|
||||
write(psb_out_unit,'(" WARNING: mismatch exceeds tolerance ",es12.5)') tol
|
||||
end if
|
||||
end if
|
||||
|
||||
call psb_gefree(x_baseline, desc_a, info)
|
||||
call psb_gefree(x_isend, desc_a, info)
|
||||
call psb_gefree(x_neighbor, desc_a, info)
|
||||
call psb_gefree(x_persistent, desc_a, info)
|
||||
call psb_gefree(y_baseline, desc_a, info)
|
||||
call psb_gefree(y_neighbor, desc_a, info)
|
||||
call psb_gefree(y_persistent, desc_a, info)
|
||||
call psb_gefree(y_ov_isend, desc_a, info)
|
||||
call psb_gefree(y_ov_neighbor, desc_a, info)
|
||||
call psb_gefree(y_ov_persistent, desc_a, info)
|
||||
call psb_gefree(y_no_isend, desc_a, info)
|
||||
call psb_gefree(y_no_neighbor, desc_a, info)
|
||||
call psb_gefree(y_no_persistent, desc_a, info)
|
||||
call psb_spfree(a, desc_a, info)
|
||||
call psb_cdfree(desc_a, info)
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
!
|
||||
program psb_comm_test
|
||||
use psb_base_mod
|
||||
use psb_error_mod, only: psb_set_debug_level, psb_debug_ext_
|
||||
use psi_mod
|
||||
use psb_comm_factory_mod, only: psb_comm_set, psb_comm_free
|
||||
use psb_comm_schemes_mod, only: psb_comm_ineighbor_alltoallv_, psb_comm_persistent_ineighbor_alltoallv_, &
|
||||
@@ -30,6 +31,7 @@ program psb_comm_test
|
||||
integer(psb_ipk_) :: iters
|
||||
character(len=32) :: arg
|
||||
character(len=16) :: mode
|
||||
logical :: debug_swapdata
|
||||
|
||||
! ---- descriptor / context ----
|
||||
type(psb_ctxt_type) :: ctxt
|
||||
@@ -58,13 +60,16 @@ program psb_comm_test
|
||||
real(psb_dpk_) :: t0, t1, dt, tsum_baseline, tsum_neighbor, tsum_neighbor_persistent
|
||||
integer(psb_lpk_), allocatable :: glob_col(:)
|
||||
character(len=40) :: name
|
||||
real(psb_dpk_) :: huge_d
|
||||
|
||||
name = 'test_halo_new'
|
||||
tol = 1.0d-12
|
||||
huge_d = huge(1.0_psb_dpk_)
|
||||
n_pass = 0
|
||||
n_total = 0
|
||||
iters = 5
|
||||
mode = 'both'
|
||||
debug_swapdata = .false.
|
||||
|
||||
! ---- parse command-line argument for idim ----
|
||||
idim = 10
|
||||
@@ -92,9 +97,15 @@ program psb_comm_test
|
||||
call get_command_argument(i+1, arg)
|
||||
read(arg, *) mode
|
||||
end if
|
||||
else if (trim(arg) == '--debug') then
|
||||
debug_swapdata = .true.
|
||||
end if
|
||||
end do
|
||||
|
||||
if (debug_swapdata) then
|
||||
call psb_set_debug_level(psb_debug_ext_)
|
||||
end if
|
||||
|
||||
run_baseline = .false.
|
||||
run_neighbor = .false.
|
||||
run_persistent = .false.
|
||||
@@ -180,11 +191,35 @@ program psb_comm_test
|
||||
! 3. Allocate two D vectors (scratch) and fill owned entries
|
||||
! ==================================================================
|
||||
call psb_geall(v_baseline, desc_a, info)
|
||||
if (info /= psb_success_) then
|
||||
write(psb_err_unit,*) my_rank, 'geall baseline error:', info
|
||||
call psb_abort(ctxt)
|
||||
end if
|
||||
call psb_geall(v_neighbor, desc_a, info)
|
||||
if (info /= psb_success_) then
|
||||
write(psb_err_unit,*) my_rank, 'geall neighbor error:', info
|
||||
call psb_abort(ctxt)
|
||||
end if
|
||||
call psb_geall(v_neighbor_persistent, desc_a, info)
|
||||
if (info /= psb_success_) then
|
||||
write(psb_err_unit,*) my_rank, 'geall persistent-neighbor error:', info
|
||||
call psb_abort(ctxt)
|
||||
end if
|
||||
call psb_geasb(v_baseline, desc_a, info, scratch=.true.)
|
||||
if (info /= psb_success_) then
|
||||
write(psb_err_unit,*) my_rank, 'geasb baseline error:', info
|
||||
call psb_abort(ctxt)
|
||||
end if
|
||||
call psb_geasb(v_neighbor, desc_a, info, scratch=.true.)
|
||||
if (info /= psb_success_) then
|
||||
write(psb_err_unit,*) my_rank, 'geasb neighbor error:', info
|
||||
call psb_abort(ctxt)
|
||||
end if
|
||||
call psb_geasb(v_neighbor_persistent, desc_a, info, scratch=.true.)
|
||||
if (info /= psb_success_) then
|
||||
write(psb_err_unit,*) my_rank, 'geasb persistent-neighbor error:', info
|
||||
call psb_abort(ctxt)
|
||||
end if
|
||||
|
||||
! Fill owned entries with the global index value
|
||||
allocate(vals(ncol))
|
||||
@@ -207,6 +242,10 @@ program psb_comm_test
|
||||
do i = 1, ncol
|
||||
expected(i) = real(glob_col(i), psb_dpk_)
|
||||
end do
|
||||
allocate(result_baseline(ncol), result_neighbor(ncol), result_persistent(ncol))
|
||||
result_baseline = huge_d
|
||||
result_neighbor = huge_d
|
||||
result_persistent = huge_d
|
||||
|
||||
! ==================================================================
|
||||
! 6. Baseline halo exchange (Isend/Irecv in one call)
|
||||
@@ -342,16 +381,33 @@ program psb_comm_test
|
||||
! ==================================================================
|
||||
! 8. Extract results and compare
|
||||
! ==================================================================
|
||||
result_baseline = v_baseline%get_vect()
|
||||
result_neighbor = v_neighbor%get_vect()
|
||||
result_persistent = v_neighbor_persistent%get_vect()
|
||||
result_baseline = v_baseline%v%v
|
||||
result_neighbor = v_neighbor%v%v
|
||||
result_persistent = v_neighbor_persistent%v%v
|
||||
|
||||
! Debug: Check if results are properly populated
|
||||
if (my_rank == 0 .and. debug_swapdata) then
|
||||
write(psb_out_unit,'("DEBUG: ncol=",i0," nrow=",i0)') ncol, nrow
|
||||
write(psb_out_unit,'("DEBUG: size(result_baseline)=",i0)') size(result_baseline)
|
||||
if (ncol > 0) then
|
||||
write(psb_out_unit,'("DEBUG: result_baseline(1:min(5,ncol))=",5(es12.5,1x))') &
|
||||
& result_baseline(1:min(5,ncol))
|
||||
write(psb_out_unit,'("DEBUG: expected(1:min(5,ncol))=",5(es12.5,1x))') &
|
||||
& expected(1:min(5,ncol))
|
||||
end if
|
||||
end if
|
||||
|
||||
if (run_baseline .and. run_neighbor) then
|
||||
n_total = n_total + 1
|
||||
err = maxval(abs(result_baseline(1:ncol) - result_neighbor(1:ncol)))
|
||||
err = huge_d
|
||||
if (ncol > 0) then
|
||||
err = maxval(abs(result_baseline(1:ncol) - result_neighbor(1:ncol)))
|
||||
else
|
||||
err = 0.0_psb_dpk_
|
||||
end if
|
||||
call psb_amx(ctxt, err)
|
||||
if (my_rank == 0) then
|
||||
if (err < tol) then
|
||||
if ((err >= 0.0_psb_dpk_) .and. (err < tol)) then
|
||||
write(psb_out_unit,'(" [PASS] cross-check baseline vs neighbor : err = ",es12.5)') err
|
||||
n_pass = n_pass + 1
|
||||
else
|
||||
@@ -362,10 +418,15 @@ program psb_comm_test
|
||||
|
||||
if (run_baseline) then
|
||||
n_total = n_total + 1
|
||||
err = maxval(abs(result_baseline(1:ncol) - expected(1:ncol)))
|
||||
err = huge_d
|
||||
if (ncol > 0) then
|
||||
err = maxval(abs(result_baseline(1:ncol) - expected(1:ncol)))
|
||||
else
|
||||
err = 0.0_psb_dpk_
|
||||
end if
|
||||
call psb_amx(ctxt, err)
|
||||
if (my_rank == 0) then
|
||||
if (err < tol) then
|
||||
if ((err >= 0.0_psb_dpk_) .and. (err < tol)) then
|
||||
write(psb_out_unit,'(" [PASS] baseline absolute correctness : err = ",es12.5)') err
|
||||
n_pass = n_pass + 1
|
||||
else
|
||||
@@ -376,10 +437,15 @@ program psb_comm_test
|
||||
|
||||
if (run_neighbor) then
|
||||
n_total = n_total + 1
|
||||
err = maxval(abs(result_neighbor(1:ncol) - expected(1:ncol)))
|
||||
err = huge_d
|
||||
if (ncol > 0) then
|
||||
err = maxval(abs(result_neighbor(1:ncol) - expected(1:ncol)))
|
||||
else
|
||||
err = 0.0_psb_dpk_
|
||||
end if
|
||||
call psb_amx(ctxt, err)
|
||||
if (my_rank == 0) then
|
||||
if (err < tol) then
|
||||
if ((err >= 0.0_psb_dpk_) .and. (err < tol)) then
|
||||
write(psb_out_unit,'(" [PASS] neighbor absolute correctness : err = ",es12.5)') err
|
||||
n_pass = n_pass + 1
|
||||
else
|
||||
@@ -390,10 +456,15 @@ program psb_comm_test
|
||||
|
||||
if (run_baseline .and. run_persistent) then
|
||||
n_total = n_total + 1
|
||||
err = maxval(abs(result_baseline(1:ncol) - result_persistent(1:ncol)))
|
||||
err = huge_d
|
||||
if (ncol > 0) then
|
||||
err = maxval(abs(result_baseline(1:ncol) - result_persistent(1:ncol)))
|
||||
else
|
||||
err = 0.0_psb_dpk_
|
||||
end if
|
||||
call psb_amx(ctxt, err)
|
||||
if (my_rank == 0) then
|
||||
if (err < tol) then
|
||||
if ((err >= 0.0_psb_dpk_) .and. (err < tol)) then
|
||||
write(psb_out_unit,'(" [PASS] cross-check baseline vs pers-nei : err = ",es12.5)') err
|
||||
n_pass = n_pass + 1
|
||||
else
|
||||
@@ -404,10 +475,15 @@ program psb_comm_test
|
||||
|
||||
if (run_persistent) then
|
||||
n_total = n_total + 1
|
||||
err = maxval(abs(result_persistent(1:ncol) - expected(1:ncol)))
|
||||
err = huge_d
|
||||
if (ncol > 0) then
|
||||
err = maxval(abs(result_persistent(1:ncol) - expected(1:ncol)))
|
||||
else
|
||||
err = 0.0_psb_dpk_
|
||||
end if
|
||||
call psb_amx(ctxt, err)
|
||||
if (my_rank == 0) then
|
||||
if (err < tol) then
|
||||
if ((err >= 0.0_psb_dpk_) .and. (err < tol)) then
|
||||
write(psb_out_unit,'(" [PASS] pers-neigh absolute correctness : err = ",es12.5)') err
|
||||
n_pass = n_pass + 1
|
||||
else
|
||||
@@ -426,12 +502,17 @@ program psb_comm_test
|
||||
call psi_swapdata(psb_comm_status_start_, dzero, v_neighbor%v, desc_a, info, data=psb_comm_halo_)
|
||||
call psi_swapdata(psb_comm_status_wait_, dzero, v_neighbor%v, desc_a, info, data=psb_comm_halo_)
|
||||
|
||||
result_neighbor = v_neighbor%get_vect()
|
||||
result_neighbor = v_neighbor%v%v
|
||||
n_total = n_total + 1
|
||||
err = maxval(abs(result_neighbor(1:ncol) - expected(1:ncol)))
|
||||
err = huge_d
|
||||
if (ncol > 0) then
|
||||
err = maxval(abs(result_neighbor(1:ncol) - expected(1:ncol)))
|
||||
else
|
||||
err = 0.0_psb_dpk_
|
||||
end if
|
||||
call psb_amx(ctxt, err)
|
||||
if (my_rank == 0) then
|
||||
if (err < tol) then
|
||||
if ((err >= 0.0_psb_dpk_) .and. (err < tol)) then
|
||||
write(psb_out_unit,'(" [PASS] neighbor topology reuse : err = ",es12.5)') err
|
||||
n_pass = n_pass + 1
|
||||
else
|
||||
@@ -450,12 +531,17 @@ program psb_comm_test
|
||||
call psi_swapdata(psb_comm_status_start_, dzero, v_neighbor_persistent%v, desc_a, info, data=psb_comm_halo_)
|
||||
call psi_swapdata(psb_comm_status_wait_, dzero, v_neighbor_persistent%v, desc_a, info, data=psb_comm_halo_)
|
||||
|
||||
result_persistent = v_neighbor_persistent%get_vect()
|
||||
result_persistent = v_neighbor_persistent%v%v
|
||||
n_total = n_total + 1
|
||||
err = maxval(abs(result_persistent(1:ncol) - expected(1:ncol)))
|
||||
err = huge_d
|
||||
if (ncol > 0) then
|
||||
err = maxval(abs(result_persistent(1:ncol) - expected(1:ncol)))
|
||||
else
|
||||
err = 0.0_psb_dpk_
|
||||
end if
|
||||
call psb_amx(ctxt, err)
|
||||
if (my_rank == 0) then
|
||||
if (err < tol) then
|
||||
if ((err >= 0.0_psb_dpk_) .and. (err < tol)) then
|
||||
write(psb_out_unit,'(" [PASS] pers-neigh buffer reuse : err = ",es12.5)') err
|
||||
n_pass = n_pass + 1
|
||||
else
|
||||
@@ -467,6 +553,7 @@ program psb_comm_test
|
||||
! ==================================================================
|
||||
! 9. Summary
|
||||
! ==================================================================
|
||||
call psb_barrier(ctxt)
|
||||
if (my_rank == 0) then
|
||||
write(psb_out_unit,'("================================================")')
|
||||
write(psb_out_unit,'(" Results: ",i0," / ",i0," tests passed")') n_pass, n_total
|
||||
@@ -477,6 +564,7 @@ program psb_comm_test
|
||||
end if
|
||||
write(psb_out_unit,'("================================================")')
|
||||
end if
|
||||
call psb_barrier(ctxt)
|
||||
|
||||
! ==================================================================
|
||||
! 10. Cleanup
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env bash
|
||||
#SBATCH --job-name=psb_swapdata_test
|
||||
#SBATCH --partition=boost_usr_prod
|
||||
#SBATCH --time=02:00:00
|
||||
#SBATCH --nodes=4
|
||||
#SBATCH --ntasks=128
|
||||
#SBATCH --ntasks-per-node=32
|
||||
#SBATCH --cpus-per-task=1
|
||||
#SBATCH --threads-per-core=1
|
||||
#SBATCH --gpus-per-node=4
|
||||
#SBATCH --export=ALL
|
||||
#SBATCH -A CNHPC_1736213
|
||||
#SBATCH --output=psb_comm_swapdata_%j.out
|
||||
#SBATCH --error=psb_comm_swapdata_%j.err
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Environment tuned like the main comm test script.
|
||||
export UCX_VFS_ENABLE=n
|
||||
export UCX_VFS_USE_FUSE=n
|
||||
export UCX_STATS_DEST=none
|
||||
export UCX_LOG_LEVEL=error
|
||||
export UCX_TLS=dc,sm,self
|
||||
export UCX_NET_DEVICES=mlx5_0:1
|
||||
export OMPI_MCA_coll=^hcoll,han
|
||||
export OMPI_MCA_coll_hcoll_enable=0
|
||||
export UCX_MEMTYPE_CACHE=n
|
||||
export UCX_CLOSE_TIMEOUT=10s
|
||||
|
||||
EXEC=./test/comm/swapdata/runs/psb_comm_test
|
||||
DIM_START=${DIM_START:-20}
|
||||
DIM_END=${DIM_END:-300}
|
||||
DIM_STEP=${DIM_STEP:-20}
|
||||
ITERS=${ITERS:-10}
|
||||
MODE=${MODE:-both}
|
||||
DEBUG=${DEBUG:-0}
|
||||
|
||||
EXTRA_ARGS=()
|
||||
if [[ "$DEBUG" != "0" ]]; then
|
||||
EXTRA_ARGS+=(--debug)
|
||||
fi
|
||||
|
||||
if [[ ! -x "$EXEC" ]]; then
|
||||
echo "Executable not found: $EXEC" >&2
|
||||
echo "Build the test first with 'make' in test/comm/swapdata/" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Running swapdata comm test"
|
||||
echo " EXEC=$EXEC"
|
||||
echo " DIM_START=$DIM_START"
|
||||
echo " DIM_END=$DIM_END"
|
||||
echo " DIM_STEP=$DIM_STEP"
|
||||
echo " ITERS=$ITERS"
|
||||
echo " MODE=$MODE"
|
||||
echo " DEBUG=$DEBUG"
|
||||
|
||||
for DIM in $(seq "$DIM_START" "$DIM_STEP" "$DIM_END"); do
|
||||
echo ""
|
||||
echo "=== Running DIM=$DIM ==="
|
||||
srun --exclusive -N2 -n64 "$EXEC" --dim "$DIM" --iters "$ITERS" --mode "$MODE" "${EXTRA_ARGS[@]}"
|
||||
done
|
||||
Reference in New Issue
Block a user