mirror of
https://github.com/sfilippone/amg4psblas.git
synced 2026-10-06 22:55:12 +00:00
[ADD] Added first test for communication schemes profiling on additive and multiplicative AMG
This commit is contained in:
Stack-1
@@ -17,7 +17,14 @@ SGEN2D=amg_s_pde2d_poisson_mod.o amg_s_pde2d_exp_mod.o \
|
||||
SGEN3D=amg_s_pde3d_poisson_mod.o amg_s_pde3d_exp_mod.o \
|
||||
amg_s_pde3d_gauss_mod.o amg_s_pde3d_box_mod.o
|
||||
|
||||
all: amg_s_pde3d amg_d_pde3d amg_s_pde2d amg_d_pde2d
|
||||
all: amg_s_pde3d amg_d_pde3d amg_s_pde2d amg_d_pde2d amg_d_comm_test
|
||||
|
||||
# Comparison of the PSBLAS communication schemes over an AMG hierarchy.
|
||||
# Reuses the 3D PDE generators of this directory rather than duplicating them.
|
||||
amg_d_comm_test: amg_d_comm_test.o amg_d_genpde_mod.o $(DGEN3D)
|
||||
$(FLINK) $(LINKOPT) amg_d_comm_test.o amg_d_genpde_mod.o $(DGEN3D) \
|
||||
-o amg_d_comm_test $(AMG_LIBS) $(PSBLAS_LIBS) $(AMG_LDLIBS) $(LDLIBS)
|
||||
/bin/mv amg_d_comm_test $(EXEDIR)
|
||||
|
||||
amg_d_pde3d: amg_d_pde3d.o amg_d_genpde_mod.o $(DGEN3D) data_input.o
|
||||
$(FLINK) $(LINKOPT) amg_d_pde3d.o amg_d_genpde_mod.o $(DGEN3D) data_input.o \
|
||||
@@ -42,6 +49,7 @@ amg_s_pde2d: amg_s_pde2d.o amg_s_genpde_mod.o $(SGEN2D) data_input.o
|
||||
|
||||
amg_d_pde3d.o amg_s_pde3d.o amg_d_pde2d.o amg_s_pde2d.o: data_input.o
|
||||
|
||||
amg_d_comm_test.o: amg_d_genpde_mod.o $(DGEN3D)
|
||||
amg_d_pde3d.o: amg_d_genpde_mod.o $(DGEN3D)
|
||||
amg_s_pde3d.o: amg_s_genpde_mod.o $(SGEN3D)
|
||||
amg_d_pde2d.o: amg_d_genpde_mod.o $(DGEN2D)
|
||||
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/bin/bash
|
||||
# ============================================================================
|
||||
# amg_comm_strong.sbatch STRONG scaling, communication schemes on AMG
|
||||
#
|
||||
# Fixed total problem size; MPI ranks grow across scale points. Every scale
|
||||
# point runs inside ONE reserved allocation, so the queue is paid once.
|
||||
#
|
||||
# The executable sweeps, in a single invocation:
|
||||
# * uniform the 5 schemes applied to the whole hierarchy
|
||||
# * sensitivity baseline everywhere, one level at a time moved onto another
|
||||
# scheme -- the marginal value of a scheme AT a level, which
|
||||
# is what a per-level policy is built on
|
||||
# so there is no per-scheme or per-level loop here, and no reason to come back
|
||||
# for a second allocation.
|
||||
#
|
||||
# Both CSVs are written per scale point; nothing needs re-running to plot.
|
||||
# ============================================================================
|
||||
#SBATCH --job-name=amg_comm_strong
|
||||
#SBATCH --output=amg_comm_strong_%j.out
|
||||
#SBATCH --error=amg_comm_strong_%j.err
|
||||
#SBATCH --nodes=8 # reserve the maximum up front
|
||||
#SBATCH --ntasks-per-node=80
|
||||
#SBATCH --cpus-per-task=1 # CPU-only: one core per rank
|
||||
#SBATCH --time=01:00:00
|
||||
#SBATCH --qos=acc_debug
|
||||
# Account intentionally not hardcoded. Pass it at submit time:
|
||||
# sbatch -A <account> amg_comm_strong.sh
|
||||
# or export SBATCH_ACCOUNT=<account> once in your cluster profile.
|
||||
|
||||
# ============================================================================
|
||||
# USER CONFIGURATION
|
||||
# ============================================================================
|
||||
DIM=200 # FIXED problem size (idim^3 unknowns)
|
||||
NREP=7 # repetitions per configuration
|
||||
NLEV=5 # max levels requested of the hierarchy
|
||||
ITMAX=500
|
||||
RANK_POINTS="80 160 320 640" # total ranks per scale point (multiples of 80)
|
||||
|
||||
EXE=$SLURM_SUBMIT_DIR/runs/amg_d_comm_test
|
||||
|
||||
# ============================================================================
|
||||
# ENVIRONMENT
|
||||
# ============================================================================
|
||||
module purge
|
||||
module load bsc/1.0
|
||||
module load gcc/12.3.0
|
||||
module load ucx/1.16.0-gcc
|
||||
module load openmpi/5.0.5-gcc
|
||||
module load openblas/0.3.27-gcc
|
||||
|
||||
export OMPI_MCA_coll_hcoll_enable=0
|
||||
|
||||
RESDIR=$SLURM_SUBMIT_DIR/results_amg_comm_${SLURM_JOB_ID}
|
||||
mkdir -p $RESDIR
|
||||
|
||||
echo "=== AMG communication schemes, STRONG scaling (CPU-only) ==="
|
||||
echo " fixed_dim=$DIM nrep=$NREP max_levels=$NLEV itmax=$ITMAX"
|
||||
echo " reserved_nodes=$SLURM_NNODES rank_points=[$RANK_POINTS]"
|
||||
echo " exe=$EXE"
|
||||
echo "============================================================"
|
||||
|
||||
if [ ! -x "$EXE" ]; then
|
||||
echo "FATAL: $EXE not found or not executable. Build it before submitting:"
|
||||
echo " cd samples/advanced/pdegen && make amg_d_comm_test"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
|
||||
for NRANKS in $RANK_POINTS; do
|
||||
NNODES=$(( (NRANKS + 79) / 80 ))
|
||||
STEP_DIR=$RESDIR/${NRANKS}ranks
|
||||
mkdir -p $STEP_DIR
|
||||
|
||||
echo ""
|
||||
echo ">>> STRONG point: $NRANKS ranks ($NNODES nodes), fixed dim=$DIM"
|
||||
|
||||
srun -N $NNODES -n $NRANKS --ntasks-per-node=80 --cpus-per-task=1 \
|
||||
$EXE $DIM $NREP $NLEV $ITMAX \
|
||||
--mode=all --csv=$STEP_DIR/amg_comm.csv \
|
||||
> $STEP_DIR/run.out 2>&1
|
||||
RC=$?
|
||||
echo ">>> exit=$RC output=$STEP_DIR/run.out"
|
||||
|
||||
# Fail loudly rather than silently producing a dataset nobody can trust:
|
||||
# if a scheme did not reach every level, the comparison is meaningless.
|
||||
if ! grep -q "SCHEME PROPAGATION: OK" $STEP_DIR/run.out; then
|
||||
echo "!!! WARNING: scheme propagation not confirmed at $NRANKS ranks"
|
||||
FAILED=1
|
||||
fi
|
||||
# All configurations must converge identically; a differing iteration count
|
||||
# means the halo exchange changed the arithmetic, not just its schedule.
|
||||
NITERS=$(grep -oE "it +[0-9]+" $STEP_DIR/run.out | awk '{print $2}' | sort -u | wc -l)
|
||||
if [ "$NITERS" != "1" ]; then
|
||||
echo "!!! WARNING: iteration count is not constant at $NRANKS ranks ($NITERS distinct values)"
|
||||
FAILED=1
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
if [ "$FAILED" = "0" ]; then
|
||||
echo "=== AMG COMM STRONG DONE, all checks passed. Results: $RESDIR ==="
|
||||
else
|
||||
echo "=== AMG COMM STRONG DONE WITH WARNINGS. Results: $RESDIR ==="
|
||||
fi
|
||||
find $RESDIR -name "*.csv" -printf " %p (%s bytes)\n"
|
||||
@@ -0,0 +1,467 @@
|
||||
!
|
||||
! Comparison of the PSBLAS communication schemes over an AMG hierarchy.
|
||||
!
|
||||
! Companion to test/comm/cg in PSBLAS, which does the same for an unpreconditioned
|
||||
! CG: here the solve runs with a multilevel preconditioner, so halo exchanges
|
||||
! happen on every level of the hierarchy and not only on the fine one.
|
||||
!
|
||||
! Two measurement modes, both produced by a single invocation so that a batch
|
||||
! allocation yields the whole dataset:
|
||||
!
|
||||
! uniform the same scheme on every level, one run per scheme. Answers
|
||||
! "which scheme is best overall".
|
||||
! sensitivity baseline everywhere except one level, swept over levels and
|
||||
! schemes. Answers "what does scheme S buy me AT level k", which
|
||||
! is the quantity a per-level policy is built on and which cannot
|
||||
! be recovered from the uniform runs.
|
||||
!
|
||||
! The scheme cannot simply be set on desc_a and left at that. Two facts on the
|
||||
! PSBLAS side make the ordering in run_one mandatory:
|
||||
!
|
||||
! * psb_desc_type%comm_type defaults to psb_comm_isend_irecv_ and is NOT
|
||||
! carried over by clone/transfer, so the coarse descriptors built inside
|
||||
! prec%build come out on the baseline scheme whatever desc_a says;
|
||||
! * the communication handle is latched lazily on the vector at its first
|
||||
! exchange (psi_dswapdata_vect) and never re-read afterwards, so handles
|
||||
! left over from a previous repetition must be freed explicitly.
|
||||
!
|
||||
! Run with:
|
||||
! mpirun -np <P> ./amg_d_comm_test [idim] [nrep] [nlev] [itmax]
|
||||
! [--remap] [--csv=<path>] [--mode=all|uniform|sensitivity]
|
||||
!
|
||||
program amg_d_comm_test
|
||||
use psb_base_mod
|
||||
use psb_util_mod
|
||||
use psb_linsolve_mod
|
||||
use amg_prec_mod
|
||||
use amg_d_genpde_mod
|
||||
use amg_d_pde3d_poisson_mod
|
||||
use psb_comm_factory_mod, only: psb_comm_free
|
||||
use psb_comm_schemes_mod, only: psb_comm_isend_irecv_, psb_comm_ineighbor_alltoallv_, &
|
||||
& psb_comm_persistent_ineighbor_alltoallv_, psb_comm_rma_pull_, psb_comm_rma_push_
|
||||
|
||||
implicit none
|
||||
|
||||
integer(psb_ipk_), parameter :: n_schemes = 5, max_lev = 32
|
||||
|
||||
type(psb_ctxt_type) :: ctxt
|
||||
type(psb_dspmat_type) :: a
|
||||
type(psb_desc_type) :: desc_a
|
||||
type(psb_d_vect_type) :: b, x
|
||||
type(amg_dprec_type) :: prec
|
||||
|
||||
integer(psb_ipk_) :: info, my_rank, np
|
||||
integer(psb_ipk_) :: idim, itmax, itrace, istop
|
||||
integer(psb_ipk_) :: s_idx, rep, nrep, nlev, lev, nlev_built, csv_unit
|
||||
integer(psb_ipk_) :: scheme_type(n_schemes)
|
||||
integer(psb_ipk_) :: scheme_of_level(max_lev)
|
||||
character(len=25) :: scheme_name(n_schemes)
|
||||
real(psb_dpk_) :: eps
|
||||
character(len=5) :: afmt
|
||||
character(len=256):: csv_file, lev_file
|
||||
character(len=16) :: run_mode
|
||||
logical :: do_remap, ok_all, want_csv, header_done
|
||||
! per-level exchange accounting, filled after every solve
|
||||
integer(psb_ipk_) :: lev_unit
|
||||
integer(psb_ipk_) :: lev_rows(max_lev), lev_halo(max_lev)
|
||||
|
||||
info = psb_success_
|
||||
afmt = 'CSR'
|
||||
idim = 40
|
||||
nrep = 5
|
||||
nlev = 3
|
||||
itmax = 1000
|
||||
itrace = -1
|
||||
istop = 2
|
||||
eps = 1.0e-6_psb_dpk_
|
||||
csv_file = ''
|
||||
run_mode = 'all'
|
||||
do_remap = .false.
|
||||
csv_unit = 77
|
||||
lev_unit = 78
|
||||
header_done = .false.
|
||||
lev_rows(:) = 0
|
||||
lev_halo(:) = 0
|
||||
|
||||
scheme_type = (/ psb_comm_isend_irecv_, psb_comm_ineighbor_alltoallv_, &
|
||||
& psb_comm_persistent_ineighbor_alltoallv_, psb_comm_rma_pull_, psb_comm_rma_push_ /)
|
||||
scheme_name(1) = 'isend_irecv'
|
||||
scheme_name(2) = 'ineighbor_alltoallv'
|
||||
scheme_name(3) = 'persistent_ineighbor_a2av'
|
||||
scheme_name(4) = 'rma_pull'
|
||||
scheme_name(5) = 'rma_push'
|
||||
|
||||
call read_int_arg(1, idim, 40)
|
||||
call read_int_arg(2, nrep, 5)
|
||||
call read_int_arg(3, nlev, 3)
|
||||
call read_int_arg(4, itmax,1000)
|
||||
call parse_flags(do_remap, csv_file, run_mode)
|
||||
want_csv = (len_trim(csv_file) > 0)
|
||||
|
||||
call psb_init(ctxt)
|
||||
call psb_info(ctxt, my_rank, np)
|
||||
call amg_set_do_remap(do_remap)
|
||||
|
||||
if (my_rank == psb_root_) then
|
||||
write(psb_out_unit,*) 'Welcome to PSBLAS version: ', psb_version_string_
|
||||
write(psb_out_unit,'("AMG communication-scheme comparison")')
|
||||
write(psb_out_unit,'("Grid dimensions : ",i0," x ",i0," x ",i0)') idim,idim,idim
|
||||
write(psb_out_unit,'("Number of processors : ",i0)') np
|
||||
write(psb_out_unit,'("Preconditioner : ML, V-cycle, JACOBI, max levels ",i0)') nlev
|
||||
write(psb_out_unit,'("Iterative method : CG, itmax ",i0,", eps ",es9.2)') itmax, eps
|
||||
write(psb_out_unit,'("Repetitions : ",i0)') nrep
|
||||
write(psb_out_unit,'("Remap active : ",l1)') amg_get_do_remap()
|
||||
write(psb_out_unit,'("Mode : ",a)') trim(run_mode)
|
||||
if (want_csv) write(psb_out_unit,'("CSV output : ",a)') trim(csv_file)
|
||||
write(psb_out_unit,'(" ")')
|
||||
end if
|
||||
|
||||
if (want_csv .and. (my_rank == psb_root_)) then
|
||||
open(unit=csv_unit,file=trim(csv_file),status='replace',action='write',iostat=info)
|
||||
if (info /= 0) then
|
||||
write(psb_err_unit,'("Cannot open CSV file ",a)') trim(csv_file)
|
||||
goto 9999
|
||||
end if
|
||||
write(csv_unit,'(a)') 'mode,scheme,target_level,nranks,idim,nlev_built,rep,'// &
|
||||
& 'prec_init_s,prec_bld_s,comm_set_s,krylov_s,total_s,iters,final_err,remap'
|
||||
! Second file, one row per (run, level): this is the per-level exchange
|
||||
! breakdown the policy is meant to be built on.
|
||||
lev_file = trim(csv_file)//'.levels.csv'
|
||||
open(unit=lev_unit,file=trim(lev_file),status='replace',action='write',iostat=info)
|
||||
if (info /= 0) then
|
||||
write(psb_err_unit,'("Cannot open CSV file ",a)') trim(lev_file)
|
||||
goto 9999
|
||||
end if
|
||||
write(lev_unit,'(a)') 'mode,scheme,target_level,nranks,idim,rep,level,'// &
|
||||
& 'loc_rows,halo_width'
|
||||
header_done = .true.
|
||||
end if
|
||||
|
||||
call psb_barrier(ctxt)
|
||||
call amg_gen_pde3d(ctxt,idim,a,b,x,desc_a,afmt,&
|
||||
& a1_poisson,a2_poisson,a3_poisson,&
|
||||
& b1_poisson,b2_poisson,b3_poisson,c_poisson,g_poisson,info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
|
||||
ok_all = .true.
|
||||
nlev_built = 0
|
||||
|
||||
! ---------------------------------------------------------------- uniform
|
||||
if ((trim(run_mode) == 'all').or.(trim(run_mode) == 'uniform')) then
|
||||
if (my_rank == psb_root_) then
|
||||
write(psb_out_unit,'(104("="))')
|
||||
write(psb_out_unit,'("UNIFORM: same scheme on every level")')
|
||||
write(psb_out_unit,'(104("="))')
|
||||
end if
|
||||
do s_idx = 1, n_schemes
|
||||
scheme_of_level(:) = scheme_type(s_idx)
|
||||
do rep = 1, nrep
|
||||
call run_one('uniform', trim(scheme_name(s_idx)), izero, &
|
||||
& scheme_of_level, (rep == 1), info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
end do
|
||||
end do
|
||||
end if
|
||||
|
||||
! ------------------------------------------------------------ sensitivity
|
||||
!
|
||||
! Baseline everywhere, one level at a time moved onto another scheme. The
|
||||
! difference against the all-baseline uniform run is the marginal value of
|
||||
! that scheme at that level, which is exactly what a per-level policy needs.
|
||||
!
|
||||
if ((trim(run_mode) == 'all').or.(trim(run_mode) == 'sensitivity')) then
|
||||
if (nlev_built <= 0) then
|
||||
! Need one build to learn how many levels the hierarchy actually has.
|
||||
scheme_of_level(:) = scheme_type(1)
|
||||
call run_one('probe', trim(scheme_name(1)), izero, scheme_of_level, .false., info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
end if
|
||||
if (my_rank == psb_root_) then
|
||||
write(psb_out_unit,'(" ")')
|
||||
write(psb_out_unit,'(104("="))')
|
||||
write(psb_out_unit,'("SENSITIVITY: baseline everywhere except one level")')
|
||||
write(psb_out_unit,'(104("="))')
|
||||
end if
|
||||
do lev = 1, nlev_built
|
||||
do s_idx = 2, n_schemes
|
||||
scheme_of_level(:) = scheme_type(1)
|
||||
scheme_of_level(lev) = scheme_type(s_idx)
|
||||
do rep = 1, nrep
|
||||
call run_one('sensitivity', trim(scheme_name(s_idx)), lev, &
|
||||
& scheme_of_level, .false., info)
|
||||
if (info /= psb_success_) goto 9999
|
||||
end do
|
||||
end do
|
||||
end do
|
||||
end if
|
||||
|
||||
if (my_rank == psb_root_) then
|
||||
write(psb_out_unit,'(" ")')
|
||||
if (ok_all) then
|
||||
write(psb_out_unit,'("SCHEME PROPAGATION: OK on every level and every configuration")')
|
||||
else
|
||||
write(psb_out_unit,'("SCHEME PROPAGATION: FAILED, see the messages above")')
|
||||
end if
|
||||
if (want_csv) then
|
||||
close(csv_unit)
|
||||
close(lev_unit)
|
||||
write(psb_out_unit,'("CSV written to ",a)') trim(csv_file)
|
||||
end if
|
||||
write(psb_out_unit,'(104("="))')
|
||||
end if
|
||||
|
||||
call psb_gefree(b,desc_a,info)
|
||||
call psb_gefree(x,desc_a,info)
|
||||
call psb_spfree(a,desc_a,info)
|
||||
call psb_cdfree(desc_a,info)
|
||||
call psb_exit(ctxt)
|
||||
stop
|
||||
|
||||
9999 call psb_error(ctxt)
|
||||
stop 1
|
||||
|
||||
contains
|
||||
|
||||
!
|
||||
! One full init/build/solve with the given per-level scheme assignment.
|
||||
! Timings are max-reduced across ranks before being recorded.
|
||||
!
|
||||
subroutine run_one(mode, sname, target_lev, sch_of_lev, verbose, info)
|
||||
character(len=*), intent(in) :: mode, sname
|
||||
integer(psb_ipk_), intent(in) :: target_lev
|
||||
integer(psb_ipk_), intent(in) :: sch_of_lev(:)
|
||||
logical, intent(in) :: verbose
|
||||
integer(psb_ipk_), intent(out) :: info
|
||||
|
||||
real(psb_dpk_) :: t_start, t_init, t_bld, t_comm, t_kry, err
|
||||
integer(psb_ipk_) :: iter, lv
|
||||
|
||||
info = psb_success_
|
||||
|
||||
! Fine level: must be set before the build, so that desc_a and every vector
|
||||
! lazily initialised during the solve pick this scheme up.
|
||||
call desc_a%set_comm_scheme(sch_of_lev(1), info)
|
||||
if (info /= psb_success_) return
|
||||
if (allocated(x%v%comm_handle)) call psb_comm_free(x%v%comm_handle, info)
|
||||
if (info /= psb_success_) return
|
||||
if (allocated(b%v%comm_handle)) call psb_comm_free(b%v%comm_handle, info)
|
||||
if (info /= psb_success_) return
|
||||
|
||||
call psb_geaxpby(dzero,b,dzero,x,desc_a,info)
|
||||
if (info /= psb_success_) return
|
||||
call psb_barrier(ctxt)
|
||||
|
||||
t_start = psb_wtime()
|
||||
call prec%init(ctxt,'ML',info)
|
||||
if (info /= psb_success_) return
|
||||
! Spelled out rather than left to the defaults, so that the configuration
|
||||
! the timings refer to is readable here and not in amg_dprecinit.
|
||||
call prec%set('max_levs', nlev, info)
|
||||
if (info == psb_success_) call prec%set('ml_cycle','VCYCLE',info)
|
||||
if (info == psb_success_) call prec%set('smoother_type','JACOBI',info)
|
||||
if (info == psb_success_) call prec%set('smoother_sweeps',ione,info)
|
||||
if (info /= psb_success_) return
|
||||
t_init = psb_wtime() - t_start
|
||||
call psb_amx(ctxt,t_init)
|
||||
|
||||
t_start = psb_wtime()
|
||||
call prec%build(a,desc_a,info)
|
||||
if (info /= psb_success_) return
|
||||
t_bld = psb_wtime() - t_start
|
||||
call psb_amx(ctxt,t_bld)
|
||||
|
||||
nlev_built = prec%get_nlevs()
|
||||
|
||||
! Hierarchy: the coarse descriptors only exist now, and they were born on
|
||||
! the default scheme. This must happen before the first apply.
|
||||
t_start = psb_wtime()
|
||||
call set_hierarchy_scheme(prec, sch_of_lev, info)
|
||||
if (info /= psb_success_) return
|
||||
t_comm = psb_wtime() - t_start
|
||||
call psb_amx(ctxt,t_comm)
|
||||
|
||||
call psb_geaxpby(dzero,b,dzero,x,desc_a,info)
|
||||
if (info /= psb_success_) return
|
||||
call psb_barrier(ctxt)
|
||||
|
||||
t_start = psb_wtime()
|
||||
call psb_krylov('CG',a,prec,b,x,eps,desc_a,info,&
|
||||
& itmax=itmax,iter=iter,err=err,itrace=itrace,istop=istop)
|
||||
t_kry = psb_wtime() - t_start
|
||||
call psb_amx(ctxt,t_kry)
|
||||
if (info /= psb_success_) return
|
||||
|
||||
call check_hierarchy_scheme(prec, sch_of_lev, sname, my_rank, ok_all)
|
||||
|
||||
! Per-level structure. The exchange time per level would need counters
|
||||
! inside psi_swapdata: adding any component to psb_desc_type or to
|
||||
! psb_comm_handle_type makes gfortran 13.3 die with an ICE in
|
||||
! amg_?_matchboxp_mod (gfc_get_function_type), so that is left out and the
|
||||
! per-level question is answered by the sensitivity mode instead.
|
||||
do lv = 1, nlev_built
|
||||
lev_rows(lv) = 0
|
||||
lev_halo(lv) = 0
|
||||
if (prec%precv(lv)%desc_ac%is_asb()) then
|
||||
lev_rows(lv) = prec%precv(lv)%desc_ac%get_local_rows()
|
||||
lev_halo(lv) = prec%precv(lv)%desc_ac%get_local_cols() - lev_rows(lv)
|
||||
end if
|
||||
end do
|
||||
|
||||
if (my_rank == psb_root_) then
|
||||
if (verbose) call report_hierarchy(prec)
|
||||
if (want_csv .and. header_done .and. (trim(mode) /= 'probe')) then
|
||||
do lv = 1, nlev_built
|
||||
write(lev_unit,'(a,",",a,",",i0,",",i0,",",i0,",",i0,",",i0,",",i0,",",i0)') &
|
||||
& trim(mode), trim(sname), target_lev, np, idim, rep, lv, &
|
||||
& lev_rows(lv), lev_halo(lv)
|
||||
end do
|
||||
end if
|
||||
if (trim(mode) /= 'probe') then
|
||||
write(psb_out_unit,'(a14,1x,a26,1x,"lev ",i2,1x,"it ",i5,1x,"err ",es12.5,&
|
||||
&1x,"bld ",es11.4,1x,"solve ",es11.4)') &
|
||||
& trim(mode), trim(sname), target_lev, iter, err, t_bld, t_kry
|
||||
end if
|
||||
if (want_csv .and. header_done .and. (trim(mode) /= 'probe')) then
|
||||
write(csv_unit,'(a,",",a,",",i0,",",i0,",",i0,",",i0,",",i0,5(",",es16.9),",",i0,",",es16.9,",",l1)') &
|
||||
& trim(mode), trim(sname), target_lev, np, idim, nlev_built, rep, &
|
||||
& t_init, t_bld, t_comm, t_kry, t_init+t_bld+t_comm+t_kry, &
|
||||
& iter, err, do_remap
|
||||
end if
|
||||
end if
|
||||
|
||||
call prec%free(info)
|
||||
end subroutine run_one
|
||||
|
||||
!
|
||||
! Write the per-level schemes onto every descriptor of the hierarchy and drop
|
||||
! the handles already latched on the per-level work vectors.
|
||||
!
|
||||
! base_desc is a pointer (level 1 aliases desc_a); desc_ac is the coarse
|
||||
! descriptor produced by the aggregation during prec%build. Both must be set:
|
||||
! the smoother exchanges go through base_desc, the coarse-grid ones through
|
||||
! desc_ac.
|
||||
!
|
||||
subroutine set_hierarchy_scheme(prec, sch_of_lev, info)
|
||||
type(amg_dprec_type), intent(inout) :: prec
|
||||
integer(psb_ipk_), intent(in) :: sch_of_lev(:)
|
||||
integer(psb_ipk_), intent(out) :: info
|
||||
integer(psb_ipk_) :: lv, iv, sch
|
||||
|
||||
info = psb_success_
|
||||
if (.not.allocated(prec%precv)) return
|
||||
|
||||
do lv = 1, size(prec%precv)
|
||||
sch = sch_of_lev(min(lv,size(sch_of_lev)))
|
||||
if (associated(prec%precv(lv)%base_desc)) then
|
||||
call prec%precv(lv)%base_desc%set_comm_scheme(sch, info)
|
||||
if (info /= psb_success_) return
|
||||
end if
|
||||
call prec%precv(lv)%desc_ac%set_comm_scheme(sch, info)
|
||||
if (info /= psb_success_) return
|
||||
|
||||
if (allocated(prec%precv(lv)%wrk)) then
|
||||
call free_vect_handle(prec%precv(lv)%wrk%vtx, info)
|
||||
if (info == psb_success_) call free_vect_handle(prec%precv(lv)%wrk%vty, info)
|
||||
if (info == psb_success_) call free_vect_handle(prec%precv(lv)%wrk%vx2l, info)
|
||||
if (info == psb_success_) call free_vect_handle(prec%precv(lv)%wrk%vy2l, info)
|
||||
if (info /= psb_success_) return
|
||||
if (allocated(prec%precv(lv)%wrk%wv)) then
|
||||
do iv = 1, size(prec%precv(lv)%wrk%wv)
|
||||
call free_vect_handle(prec%precv(lv)%wrk%wv(iv), info)
|
||||
if (info /= psb_success_) return
|
||||
end do
|
||||
end if
|
||||
end if
|
||||
end do
|
||||
end subroutine set_hierarchy_scheme
|
||||
|
||||
subroutine free_vect_handle(v, info)
|
||||
type(psb_d_vect_type), intent(inout) :: v
|
||||
integer(psb_ipk_), intent(out) :: info
|
||||
info = psb_success_
|
||||
if (.not.allocated(v%v)) return
|
||||
if (allocated(v%v%comm_handle)) call psb_comm_free(v%v%comm_handle, info)
|
||||
end subroutine free_vect_handle
|
||||
|
||||
!
|
||||
! After a solve, every work vector that took part in an exchange must carry a
|
||||
! handle of the requested type. A level still holding the baseline while the
|
||||
! others moved is the signature of an incomplete propagation.
|
||||
!
|
||||
subroutine check_hierarchy_scheme(prec, sch_of_lev, sname, my_rank, ok_all)
|
||||
type(amg_dprec_type), intent(in) :: prec
|
||||
integer(psb_ipk_), intent(in) :: sch_of_lev(:), my_rank
|
||||
character(len=*), intent(in) :: sname
|
||||
logical, intent(inout) :: ok_all
|
||||
integer(psb_ipk_) :: lv, got, want
|
||||
|
||||
if (.not.allocated(prec%precv)) return
|
||||
do lv = 1, size(prec%precv)
|
||||
if (.not.allocated(prec%precv(lv)%wrk)) cycle
|
||||
if (.not.allocated(prec%precv(lv)%wrk%vtx%v)) cycle
|
||||
if (.not.allocated(prec%precv(lv)%wrk%vtx%v%comm_handle)) cycle
|
||||
want = sch_of_lev(min(lv,size(sch_of_lev)))
|
||||
got = prec%precv(lv)%wrk%vtx%v%comm_handle%comm_type
|
||||
if (got /= want) then
|
||||
ok_all = .false.
|
||||
write(psb_err_unit,'(" [FAIL] ",a,": rank ",i0," level ",i0,": expected ",i0," got ",i0)') &
|
||||
& sname, my_rank, lv, want, got
|
||||
end if
|
||||
end do
|
||||
end subroutine check_hierarchy_scheme
|
||||
|
||||
!
|
||||
! Per-level size and halo width. The halo is the gap between local columns and
|
||||
! local rows, i.e. how many remote entries this rank receives: it is the
|
||||
! quantity whose ratio to the local rows degrades going coarse, which is the
|
||||
! regime the schemes are expected to behave differently in.
|
||||
!
|
||||
subroutine report_hierarchy(prec)
|
||||
type(amg_dprec_type), intent(in) :: prec
|
||||
integer(psb_ipk_) :: lv, nr, nc
|
||||
|
||||
if (.not.allocated(prec%precv)) return
|
||||
write(psb_out_unit,'(" ")')
|
||||
write(psb_out_unit,'(" level loc rows loc cols halo width halo/rows")')
|
||||
do lv = 1, size(prec%precv)
|
||||
if (.not.prec%precv(lv)%desc_ac%is_asb()) cycle
|
||||
nr = prec%precv(lv)%desc_ac%get_local_rows()
|
||||
nc = prec%precv(lv)%desc_ac%get_local_cols()
|
||||
write(psb_out_unit,'(i7,i14,i14,i15,f12.3)') lv, nr, nc, nc-nr, &
|
||||
& real(nc-nr,psb_dpk_)/real(max(nr,1),psb_dpk_)
|
||||
end do
|
||||
write(psb_out_unit,'(" ")')
|
||||
end subroutine report_hierarchy
|
||||
|
||||
subroutine parse_flags(do_remap, csv_file, run_mode)
|
||||
logical, intent(inout) :: do_remap
|
||||
character(len=*), intent(inout) :: csv_file, run_mode
|
||||
character(len=256) :: larg
|
||||
integer(psb_ipk_) :: k
|
||||
|
||||
do k = 1, command_argument_count()
|
||||
call get_command_argument(k, larg)
|
||||
if (index(larg,'--remap') == 1) then
|
||||
do_remap = .true.
|
||||
else if (index(larg,'--csv=') == 1) then
|
||||
csv_file = trim(larg(7:))
|
||||
else if (index(larg,'--mode=') == 1) then
|
||||
run_mode = trim(larg(8:))
|
||||
end if
|
||||
end do
|
||||
end subroutine parse_flags
|
||||
|
||||
subroutine read_int_arg(pos, val, fallback)
|
||||
integer(psb_ipk_), intent(in) :: pos, fallback
|
||||
integer(psb_ipk_), intent(inout) :: val
|
||||
character(len=256) :: larg
|
||||
integer(psb_ipk_) :: ios
|
||||
|
||||
call get_command_argument(pos, larg)
|
||||
if (len_trim(larg) <= 0) return
|
||||
if (index(larg,'--') == 1) return
|
||||
read(larg,*,iostat=ios) val
|
||||
if ((ios /= 0).or.(val <= 0)) val = fallback
|
||||
end subroutine read_int_arg
|
||||
|
||||
end program amg_d_comm_test
|
||||
Executable
+152
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Aggregate the CSVs written by amg_d_comm_test.
|
||||
|
||||
Reads one or more result directories (the per-scale-point directories the sbatch
|
||||
script creates) and prints, per rank count:
|
||||
|
||||
* the uniform ranking, i.e. which scheme wins over the whole hierarchy;
|
||||
* the noise floor, measured rather than assumed: levels whose halo width is
|
||||
zero are replicated and exchange nothing, so any apparent effect of changing
|
||||
their scheme is pure measurement scatter. That number is the threshold below
|
||||
which nothing else in the table can be believed;
|
||||
* the per-level sensitivity, with effects inside the noise floor marked so
|
||||
they are not read as signal.
|
||||
|
||||
Usage:
|
||||
./analyse_amg_comm.py results_amg_comm_123456/*/amg_comm.csv
|
||||
./analyse_amg_comm.py runs/amg_comm.csv
|
||||
"""
|
||||
|
||||
import csv
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
def mean(v):
|
||||
return sum(v) / len(v)
|
||||
|
||||
|
||||
def stdev(v):
|
||||
if len(v) < 2:
|
||||
return 0.0
|
||||
m = mean(v)
|
||||
return (sum((x - m) ** 2 for x in v) / (len(v) - 1)) ** 0.5
|
||||
|
||||
|
||||
def load(path):
|
||||
with open(path) as fh:
|
||||
return list(csv.DictReader(fh))
|
||||
|
||||
|
||||
def levels_path(path):
|
||||
return path + ".levels.csv"
|
||||
|
||||
|
||||
def analyse(path):
|
||||
rows = load(path)
|
||||
if not rows:
|
||||
print(f"{path}: empty")
|
||||
return
|
||||
|
||||
# Halo width per level: identical across runs, take it from the first one.
|
||||
halo = {}
|
||||
try:
|
||||
for r in load(levels_path(path)):
|
||||
halo[int(r["level"])] = int(r["halo_width"])
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
nranks = rows[0]["nranks"]
|
||||
idim = rows[0]["idim"]
|
||||
nlev = rows[0]["nlev_built"]
|
||||
print("=" * 78)
|
||||
print(f"{path}")
|
||||
print(f" ranks={nranks} idim={idim} levels_built={nlev}")
|
||||
if halo:
|
||||
desc = " ".join(f"L{k}:{v}" for k, v in sorted(halo.items()))
|
||||
print(f" halo width per level: {desc}")
|
||||
print("=" * 78)
|
||||
|
||||
solve = defaultdict(list)
|
||||
iters = set()
|
||||
for r in rows:
|
||||
key = (r["mode"], r["scheme"], int(r["target_level"]))
|
||||
solve[key].append(float(r["krylov_s"]))
|
||||
iters.add(int(r["iters"]))
|
||||
|
||||
if len(iters) != 1:
|
||||
print(f" !! iteration count is not constant: {sorted(iters)}")
|
||||
print(" the configurations are not solving the same problem;")
|
||||
print(" the timings below are not comparable.")
|
||||
base_key = ("uniform", "isend_irecv", 0)
|
||||
if base_key not in solve:
|
||||
print(" !! no uniform baseline run found, cannot compute deltas")
|
||||
return
|
||||
base = mean(solve[base_key])
|
||||
|
||||
print("\nUNIFORM (same scheme on every level)")
|
||||
print(f" {'scheme':<28}{'solve [s]':>14}{'sd':>12}{'vs base':>10}")
|
||||
uni = [(k, v) for k, v in solve.items() if k[0] == "uniform"]
|
||||
for k, v in sorted(uni, key=lambda kv: mean(kv[1])):
|
||||
print(f" {k[1]:<28}{mean(v):>14.4e}{stdev(v):>12.2e}"
|
||||
f"{100*(mean(v)-base)/base:>9.1f}%")
|
||||
|
||||
# Noise floor from the levels that cannot possibly be affected.
|
||||
dead = [lv for lv, h in halo.items() if h == 0]
|
||||
floor = None
|
||||
if dead:
|
||||
dev = [abs(100 * (mean(v) - base) / base)
|
||||
for k, v in solve.items()
|
||||
if k[0] == "sensitivity" and k[2] in dead]
|
||||
if dev:
|
||||
floor = max(dev)
|
||||
print(f"\nNOISE FLOOR {floor:.1f}%")
|
||||
print(f" measured on level(s) {dead}, whose halo width is zero:")
|
||||
print(" they are replicated and exchange nothing, so their apparent")
|
||||
print(" effect is scatter. Nothing smaller than this is a result.")
|
||||
|
||||
sens = [(k, v) for k, v in solve.items() if k[0] == "sensitivity"]
|
||||
if sens:
|
||||
print("\nSENSITIVITY (baseline everywhere except one level)")
|
||||
print(f" {'level':>6} {'scheme':<28}{'solve [s]':>14}{'vs base':>10} verdict")
|
||||
for k, v in sorted(sens, key=lambda kv: (kv[0][2], mean(kv[1]))):
|
||||
delta = 100 * (mean(v) - base) / base
|
||||
if floor is not None and abs(delta) <= floor:
|
||||
verdict = "noise"
|
||||
elif k[2] in dead:
|
||||
verdict = "(no exchange at this level)"
|
||||
elif delta < 0:
|
||||
verdict = "GAIN"
|
||||
else:
|
||||
verdict = "cost"
|
||||
print(f" {k[2]:>6} {k[1]:<28}{mean(v):>14.4e}{delta:>9.1f}% {verdict}")
|
||||
|
||||
if floor is not None:
|
||||
gains = [(k[2], k[1], 100 * (mean(v) - base) / base)
|
||||
for k, v in sens
|
||||
if k[2] not in dead and 100 * (mean(v) - base) / base < -floor]
|
||||
print()
|
||||
if gains:
|
||||
print(" Levels where a non-baseline scheme beats the baseline by more")
|
||||
print(" than the noise floor -- the case for a per-level policy:")
|
||||
for lv, sc, d in sorted(gains, key=lambda g: g[2]):
|
||||
print(f" level {lv} {sc} {d:.1f}%")
|
||||
else:
|
||||
print(" No per-level gain above the noise floor in this run.")
|
||||
print()
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) < 2:
|
||||
print(__doc__)
|
||||
return 1
|
||||
for path in argv[1:]:
|
||||
if path.endswith(".levels.csv"):
|
||||
continue
|
||||
analyse(path)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
Reference in New Issue
Block a user