mirror of
https://github.com/sfilippone/amg4psblas.git
synced 2026-10-06 14:44:55 +00:00
[ADD] Added scripts to collect data
This commit is contained in:
Stack-1
Executable
+185
@@ -0,0 +1,185 @@
|
||||
#!/bin/bash
|
||||
# ============================================================================
|
||||
# amg_comm_strong.sbatch STRONG scaling, communication schemes on AMG
|
||||
#
|
||||
# Fixed total problem size; MPI ranks grow across scale points. Every scale
|
||||
# point runs inside ONE reserved allocation, so the queue is paid once.
|
||||
#
|
||||
# The executable sweeps, in a single invocation:
|
||||
# * uniform the 5 schemes applied to the whole hierarchy
|
||||
# * sensitivity baseline everywhere, one level at a time moved onto another
|
||||
# scheme -- the marginal value of a scheme AT a level, which
|
||||
# is what a per-level policy is built on
|
||||
# so there is no per-scheme or per-level loop here, and no reason to come back
|
||||
# for a second allocation.
|
||||
#
|
||||
# Both CSVs are written per scale point; nothing needs re-running to plot.
|
||||
# ============================================================================
|
||||
#SBATCH --job-name=amg_comm_strong
|
||||
#SBATCH --output=amg_comm_strong_%j.out
|
||||
#SBATCH --error=amg_comm_strong_%j.err
|
||||
#SBATCH --nodes=16 # reserve the maximum up front
|
||||
#SBATCH --ntasks-per-node=112 # full GPP node: 2 x Sapphire Rapids 8480+
|
||||
#SBATCH --cpus-per-task=1 # CPU-only: one core per rank
|
||||
#SBATCH --time=02:00:00
|
||||
#SBATCH --qos=gp_ehpc # production; gp_debug caps at 2h / 3584 proc
|
||||
#SBATCH --account=ehpc859 # the GPP project; ehpc580 is ACC and over budget
|
||||
#
|
||||
# Verify the node width before trusting RANK_POINTS below:
|
||||
# sinfo -o "%P %c"
|
||||
# If a GPP node is not 112 cores, ntasks-per-node, RANK_POINTS and the NNODES
|
||||
# arithmetic all have to move together.
|
||||
|
||||
# ============================================================================
|
||||
# USER CONFIGURATION
|
||||
# ============================================================================
|
||||
# STRONG or WEAK. They answer different questions and both are worth having:
|
||||
#
|
||||
# strong fixed problem, growing ranks. Matches the SpMV paper, and shows the
|
||||
# knee where communication starts to dominate. Caveat: as ranks grow
|
||||
# the AMG hierarchy itself changes shape -- fewer rows per rank at
|
||||
# every level, coarse levels degenerating -- so a scale point differs
|
||||
# from the next by more than just the rank count.
|
||||
#
|
||||
# weak fixed rows per rank, problem grown with the ranks. The hierarchy
|
||||
# keeps its shape, so the rank count is isolated from the coarsening.
|
||||
# This is the cleaner design for "how do the schemes behave as the
|
||||
# machine grows", which is the question a per-level policy needs.
|
||||
SCALING=${SCALING:-strong}
|
||||
|
||||
NREP=7 # repetitions per configuration
|
||||
NLEV=5 # max levels requested of the hierarchy
|
||||
ITMAX=500
|
||||
|
||||
RANK_POINTS="112 224 448 896 1792" # 1, 2, 4, 8, 16 full nodes
|
||||
|
||||
# strong: one size for every scale point.
|
||||
# 400^3 = 64M unknowns; at 1792 ranks that is ~36k rows/rank on the fine level
|
||||
# and still ~550 on level 3, so the hierarchy stays meaningful to the end.
|
||||
DIM_STRONG=400
|
||||
|
||||
# weak: idim grows as ranks^(1/3) to hold rows/rank constant at ~36.5k,
|
||||
# matching the strong run at its largest point so the two are comparable there.
|
||||
DIM_WEAK="160 202 254 320 403"
|
||||
|
||||
# Two passes are needed and they answer different questions:
|
||||
# PROFILE=false clean timings -- the numbers that compare schemes
|
||||
# PROFILE=true Score-P attribution -- where the time goes, per level
|
||||
# The binary is built with scorep-mpifort, so profiling is ON unless disabled:
|
||||
# a "clean" run is only clean if these are set explicitly.
|
||||
PROFILE=${PROFILE:-false}
|
||||
|
||||
EXE=$SLURM_SUBMIT_DIR/runs/amg_d_comm_test
|
||||
|
||||
# ============================================================================
|
||||
# ENVIRONMENT
|
||||
# ============================================================================
|
||||
module purge
|
||||
module load bsc/1.0
|
||||
module load gcc/12.3.0
|
||||
module load ucx/1.16.0-gcc
|
||||
module load openmpi/5.0.5-gcc
|
||||
module load openblas/0.3.27-gcc
|
||||
|
||||
export OMPI_MCA_coll_hcoll_enable=0
|
||||
|
||||
if [ "$PROFILE" = "true" ]; then
|
||||
export SCOREP_ENABLE_PROFILING=true
|
||||
export SCOREP_ENABLE_TRACING=false
|
||||
export SCOREP_TOTAL_MEMORY=128M
|
||||
else
|
||||
export SCOREP_ENABLE_PROFILING=false
|
||||
export SCOREP_ENABLE_TRACING=false
|
||||
fi
|
||||
|
||||
RESDIR=$SLURM_SUBMIT_DIR/results_amg_${SCALING}_${SLURM_JOB_ID}
|
||||
mkdir -p $RESDIR
|
||||
|
||||
# Region filter: exclude the high-frequency tiny helpers so the profile reflects
|
||||
# real MPI time rather than instrumentation of the descriptor bookkeeping.
|
||||
FILTER=$RESDIR/scorep.filt
|
||||
cat > $FILTER <<'FILTEOF'
|
||||
SCOREP_REGION_NAMES_BEGIN
|
||||
EXCLUDE
|
||||
psb_indx_map_mod::*
|
||||
psb_desc_mod::*
|
||||
psb_error_mod::*
|
||||
psb_gen_block_map_mod::*
|
||||
psi_penv_mod::*
|
||||
psb_hash_mod::*
|
||||
SCOREP_REGION_NAMES_END
|
||||
FILTEOF
|
||||
|
||||
case "$SCALING" in
|
||||
strong) echo "=== AMG communication schemes, STRONG scaling (CPU-only) ===" ;;
|
||||
weak) echo "=== AMG communication schemes, WEAK scaling (CPU-only) ===" ;;
|
||||
*) echo "FATAL: SCALING must be 'strong' or 'weak', got '$SCALING'"; exit 1 ;;
|
||||
esac
|
||||
echo " nrep=$NREP max_levels=$NLEV itmax=$ITMAX"
|
||||
echo " PROFILE=$PROFILE"
|
||||
echo " reserved_nodes=$SLURM_NNODES rank_points=[$RANK_POINTS]"
|
||||
echo " exe=$EXE"
|
||||
echo "============================================================"
|
||||
|
||||
if [ ! -x "$EXE" ]; then
|
||||
echo "FATAL: $EXE not found or not executable. Build it before submitting:"
|
||||
echo " cd samples/advanced/pdegen && make amg_d_comm_test"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
|
||||
IDX=0
|
||||
for NRANKS in $RANK_POINTS; do
|
||||
IDX=$((IDX+1))
|
||||
NNODES=$(( (NRANKS + 111) / 112 ))
|
||||
|
||||
if [ "$SCALING" = "weak" ]; then
|
||||
DIM=$(echo $DIM_WEAK | cut -d' ' -f$IDX)
|
||||
else
|
||||
DIM=$DIM_STRONG
|
||||
fi
|
||||
|
||||
STEP_DIR=$RESDIR/${NRANKS}ranks
|
||||
mkdir -p $STEP_DIR
|
||||
|
||||
echo ""
|
||||
echo ">>> $SCALING point: $NRANKS ranks ($NNODES nodes), dim=$DIM"
|
||||
|
||||
# Name the experiment after the scale point. Left to itself Score-P writes
|
||||
# scorep-<timestamp>, which carries no indication of the rank count and has
|
||||
# to be matched back by hand afterwards.
|
||||
if [ "$PROFILE" = "true" ]; then
|
||||
export SCOREP_EXPERIMENT_DIRECTORY=$STEP_DIR/scorep_${NRANKS}ranks
|
||||
export SCOREP_FILTERING_FILE=$FILTER
|
||||
fi
|
||||
|
||||
srun -N $NNODES -n $NRANKS --ntasks-per-node=112 --cpus-per-task=1 \
|
||||
$EXE $DIM $NREP $NLEV $ITMAX \
|
||||
--mode=all --csv=$STEP_DIR/amg_comm.csv \
|
||||
> $STEP_DIR/run.out 2>&1
|
||||
RC=$?
|
||||
echo ">>> exit=$RC output=$STEP_DIR/run.out"
|
||||
|
||||
# Fail loudly rather than silently producing a dataset nobody can trust:
|
||||
# if a scheme did not reach every level, the comparison is meaningless.
|
||||
if ! grep -q "SCHEME PROPAGATION: OK" $STEP_DIR/run.out; then
|
||||
echo "!!! WARNING: scheme propagation not confirmed at $NRANKS ranks"
|
||||
FAILED=1
|
||||
fi
|
||||
# All configurations must converge identically; a differing iteration count
|
||||
# means the halo exchange changed the arithmetic, not just its schedule.
|
||||
NITERS=$(grep -oE "it +[0-9]+" $STEP_DIR/run.out | awk '{print $2}' | sort -u | wc -l)
|
||||
if [ "$NITERS" != "1" ]; then
|
||||
echo "!!! WARNING: iteration count is not constant at $NRANKS ranks ($NITERS distinct values)"
|
||||
FAILED=1
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
if [ "$FAILED" = "0" ]; then
|
||||
echo "=== AMG COMM ${SCALING} DONE, all checks passed. Results: $RESDIR ==="
|
||||
else
|
||||
echo "=== AMG COMM ${SCALING} DONE WITH WARNINGS. Results: $RESDIR ==="
|
||||
fi
|
||||
find $RESDIR -name "*.csv" -printf " %p (%s bytes)\n"
|
||||
@@ -1,106 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ============================================================================
|
||||
# amg_comm_strong.sbatch STRONG scaling, communication schemes on AMG
|
||||
#
|
||||
# Fixed total problem size; MPI ranks grow across scale points. Every scale
|
||||
# point runs inside ONE reserved allocation, so the queue is paid once.
|
||||
#
|
||||
# The executable sweeps, in a single invocation:
|
||||
# * uniform the 5 schemes applied to the whole hierarchy
|
||||
# * sensitivity baseline everywhere, one level at a time moved onto another
|
||||
# scheme -- the marginal value of a scheme AT a level, which
|
||||
# is what a per-level policy is built on
|
||||
# so there is no per-scheme or per-level loop here, and no reason to come back
|
||||
# for a second allocation.
|
||||
#
|
||||
# Both CSVs are written per scale point; nothing needs re-running to plot.
|
||||
# ============================================================================
|
||||
#SBATCH --job-name=amg_comm_strong
|
||||
#SBATCH --output=amg_comm_strong_%j.out
|
||||
#SBATCH --error=amg_comm_strong_%j.err
|
||||
#SBATCH --nodes=8 # reserve the maximum up front
|
||||
#SBATCH --ntasks-per-node=80
|
||||
#SBATCH --cpus-per-task=1 # CPU-only: one core per rank
|
||||
#SBATCH --time=01:00:00
|
||||
#SBATCH --qos=acc_debug
|
||||
# Account intentionally not hardcoded. Pass it at submit time:
|
||||
# sbatch -A <account> amg_comm_strong.sh
|
||||
# or export SBATCH_ACCOUNT=<account> once in your cluster profile.
|
||||
|
||||
# ============================================================================
|
||||
# USER CONFIGURATION
|
||||
# ============================================================================
|
||||
DIM=200 # FIXED problem size (idim^3 unknowns)
|
||||
NREP=7 # repetitions per configuration
|
||||
NLEV=5 # max levels requested of the hierarchy
|
||||
ITMAX=500
|
||||
RANK_POINTS="80 160 320 640" # total ranks per scale point (multiples of 80)
|
||||
|
||||
EXE=$SLURM_SUBMIT_DIR/runs/amg_d_comm_test
|
||||
|
||||
# ============================================================================
|
||||
# ENVIRONMENT
|
||||
# ============================================================================
|
||||
module purge
|
||||
module load bsc/1.0
|
||||
module load gcc/12.3.0
|
||||
module load ucx/1.16.0-gcc
|
||||
module load openmpi/5.0.5-gcc
|
||||
module load openblas/0.3.27-gcc
|
||||
|
||||
export OMPI_MCA_coll_hcoll_enable=0
|
||||
|
||||
RESDIR=$SLURM_SUBMIT_DIR/results_amg_comm_${SLURM_JOB_ID}
|
||||
mkdir -p $RESDIR
|
||||
|
||||
echo "=== AMG communication schemes, STRONG scaling (CPU-only) ==="
|
||||
echo " fixed_dim=$DIM nrep=$NREP max_levels=$NLEV itmax=$ITMAX"
|
||||
echo " reserved_nodes=$SLURM_NNODES rank_points=[$RANK_POINTS]"
|
||||
echo " exe=$EXE"
|
||||
echo "============================================================"
|
||||
|
||||
if [ ! -x "$EXE" ]; then
|
||||
echo "FATAL: $EXE not found or not executable. Build it before submitting:"
|
||||
echo " cd samples/advanced/pdegen && make amg_d_comm_test"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
|
||||
for NRANKS in $RANK_POINTS; do
|
||||
NNODES=$(( (NRANKS + 79) / 80 ))
|
||||
STEP_DIR=$RESDIR/${NRANKS}ranks
|
||||
mkdir -p $STEP_DIR
|
||||
|
||||
echo ""
|
||||
echo ">>> STRONG point: $NRANKS ranks ($NNODES nodes), fixed dim=$DIM"
|
||||
|
||||
srun -N $NNODES -n $NRANKS --ntasks-per-node=80 --cpus-per-task=1 \
|
||||
$EXE $DIM $NREP $NLEV $ITMAX \
|
||||
--mode=all --csv=$STEP_DIR/amg_comm.csv \
|
||||
> $STEP_DIR/run.out 2>&1
|
||||
RC=$?
|
||||
echo ">>> exit=$RC output=$STEP_DIR/run.out"
|
||||
|
||||
# Fail loudly rather than silently producing a dataset nobody can trust:
|
||||
# if a scheme did not reach every level, the comparison is meaningless.
|
||||
if ! grep -q "SCHEME PROPAGATION: OK" $STEP_DIR/run.out; then
|
||||
echo "!!! WARNING: scheme propagation not confirmed at $NRANKS ranks"
|
||||
FAILED=1
|
||||
fi
|
||||
# All configurations must converge identically; a differing iteration count
|
||||
# means the halo exchange changed the arithmetic, not just its schedule.
|
||||
NITERS=$(grep -oE "it +[0-9]+" $STEP_DIR/run.out | awk '{print $2}' | sort -u | wc -l)
|
||||
if [ "$NITERS" != "1" ]; then
|
||||
echo "!!! WARNING: iteration count is not constant at $NRANKS ranks ($NITERS distinct values)"
|
||||
FAILED=1
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
if [ "$FAILED" = "0" ]; then
|
||||
echo "=== AMG COMM STRONG DONE, all checks passed. Results: $RESDIR ==="
|
||||
else
|
||||
echo "=== AMG COMM STRONG DONE WITH WARNINGS. Results: $RESDIR ==="
|
||||
fi
|
||||
find $RESDIR -name "*.csv" -printf " %p (%s bytes)\n"
|
||||
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/bin/bash
|
||||
# ============================================================================
|
||||
# collect_results.sh — snapshot a result set together with its provenance
|
||||
#
|
||||
# A directory of CSVs is not a dataset: in a month nobody remembers which code,
|
||||
# which library and which environment produced it, and the run gets repeated.
|
||||
# This records all of that next to the numbers, so a result set can be trusted
|
||||
# without re-running it.
|
||||
#
|
||||
# Score-P experiment directories are folded into the result set too. Their
|
||||
# default names carry a timestamp and nothing else, so on their own nobody can
|
||||
# tell which scale point a profile belongs to; here they are moved inside the
|
||||
# result directory and listed with their times, in run order.
|
||||
#
|
||||
# USE (on the cluster, from samples/advanced/pdegen):
|
||||
# ./collect_results.sh results_amg_comm_44449535 [label] [scorep-dir ...]
|
||||
#
|
||||
# With no scorep-dir given, every scorep-* in the current directory is taken.
|
||||
#
|
||||
# Produces <resdir>/PROVENANCE.md and a tarball ready to transfer.
|
||||
# ============================================================================
|
||||
set -u
|
||||
|
||||
RESDIR=${1:?usage: collect_results.sh <results-dir> [label] [scorep-dir ...]}
|
||||
LABEL=${2:-}
|
||||
shift 2 2>/dev/null || shift $#
|
||||
SCOREP_DIRS=("$@")
|
||||
if [ ${#SCOREP_DIRS[@]} -eq 0 ]; then
|
||||
shopt -s nullglob
|
||||
SCOREP_DIRS=(scorep-*)
|
||||
shopt -u nullglob
|
||||
fi
|
||||
|
||||
if [ ! -d "$RESDIR" ]; then
|
||||
echo "No such directory: $RESDIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
JOBID=$(basename "$RESDIR" | sed 's/.*_//')
|
||||
OUT=$RESDIR/PROVENANCE.md
|
||||
|
||||
# Fold the Score-P experiments into the result set, oldest first: the sbatch
|
||||
# runs the scale points in order, so run order is the only handle available to
|
||||
# match a timestamped profile to its rank count.
|
||||
PROFDIR=$RESDIR/scorep
|
||||
if [ ${#SCOREP_DIRS[@]} -gt 0 ]; then
|
||||
mkdir -p "$PROFDIR"
|
||||
echo "Folding ${#SCOREP_DIRS[@]} Score-P experiment(s) into $PROFDIR:"
|
||||
while IFS= read -r d; do
|
||||
[ -d "$d" ] || continue
|
||||
printf ' %s %s\n' "$(date -r "$d" '+%Y-%m-%d %H:%M:%S')" "$d"
|
||||
mv "$d" "$PROFDIR/"
|
||||
done < <(ls -1dtr "${SCOREP_DIRS[@]}" 2>/dev/null)
|
||||
fi
|
||||
|
||||
{
|
||||
echo "# Result set $JOBID"
|
||||
[ -n "$LABEL" ] && echo -e "\n**$LABEL**"
|
||||
echo
|
||||
echo "Collected: $(date -Iseconds) on $(hostname)"
|
||||
echo
|
||||
|
||||
echo "## Instrumentation"
|
||||
echo
|
||||
# The single most misread property of a result set: whether the binary was
|
||||
# instrumented and whether profiling was actually on during the run.
|
||||
if [ -d "$PROFDIR" ]; then
|
||||
echo "Score-P experiments were collected, so profiling was ACTIVE:"
|
||||
echo "absolute timings in the CSVs include instrumentation overhead and are"
|
||||
echo "comparable between schemes, not against a clean build."
|
||||
echo
|
||||
echo "Profiles in run order (the sbatch runs the scale points in order,"
|
||||
echo "so the n-th profile corresponds to the n-th entry of RANK_POINTS;"
|
||||
echo "check against the timestamps in the per-step run.out to be sure):"
|
||||
echo
|
||||
echo '```'
|
||||
ls -1dtr "$PROFDIR"/*/ 2>/dev/null | while read -r d; do
|
||||
printf '%s %8s %s\n' "$(date -r "$d" '+%Y-%m-%d %H:%M:%S')" \
|
||||
"$(du -sh "$d" | cut -f1)" "$(basename "$d")"
|
||||
done
|
||||
echo '```'
|
||||
echo
|
||||
echo "Read them with:"
|
||||
echo '```'
|
||||
echo "cube_stat -p <profile>/profile.cubex"
|
||||
echo "cube_dump -m time -c all <profile>/profile.cubex"
|
||||
echo '```'
|
||||
else
|
||||
echo "No Score-P experiments collected; profiling was probably off."
|
||||
fi
|
||||
echo
|
||||
echo '```'
|
||||
echo "SCOREP_ENABLE_PROFILING=${SCOREP_ENABLE_PROFILING:-<unset, defaults to true>}"
|
||||
echo "SCOREP_ENABLE_TRACING=${SCOREP_ENABLE_TRACING:-<unset, defaults to false>}"
|
||||
echo '```'
|
||||
echo
|
||||
echo "## MPI runtime settings"
|
||||
echo
|
||||
echo "These change which implementation the schemes actually run on, so they"
|
||||
echo "have to be held constant across a campaign. HCOLL in particular"
|
||||
echo "accelerates collectives, which the two neighborhood schemes use and the"
|
||||
echo "point-to-point and one-sided ones do not: switching it changes what is"
|
||||
echo "being compared, not just how fast it is."
|
||||
echo
|
||||
echo '```'
|
||||
env | grep -E "^(OMPI_MCA_|UCX_|SLURM_CPU_BIND)" | sort || echo "(none set)"
|
||||
echo '```'
|
||||
echo
|
||||
|
||||
echo "## Code"
|
||||
echo
|
||||
echo '```'
|
||||
for repo in "$HOME/Desktop/scorep/amg4psblas" "$HOME/Desktop/scorep/psblas3"; do
|
||||
if [ -d "$repo/.git" ]; then
|
||||
printf '%-12s %s %s\n' "$(basename "$repo")" \
|
||||
"$(git -C "$repo" rev-parse --short HEAD 2>/dev/null)" \
|
||||
"$(git -C "$repo" rev-parse --abbrev-ref HEAD 2>/dev/null)"
|
||||
if [ -n "$(git -C "$repo" status --porcelain 2>/dev/null)" ]; then
|
||||
echo " UNCOMMITTED CHANGES PRESENT"
|
||||
fi
|
||||
else
|
||||
printf '%-12s not a git tree\n' "$(basename "$repo")"
|
||||
fi
|
||||
done
|
||||
echo '```'
|
||||
echo
|
||||
|
||||
echo "## Build configuration"
|
||||
echo
|
||||
echo '```'
|
||||
grep -E "^PSBLASDIR|^FC=|^CC=|^FCOPT" "$HOME/Desktop/scorep/amg4psblas/Make.inc" 2>/dev/null
|
||||
grep -E "^FCUDEFINES|^CUDA_DIR" \
|
||||
"$HOME/opt/psblas-cuda/include/Make.inc.psblas" 2>/dev/null
|
||||
echo '```'
|
||||
echo
|
||||
|
||||
echo "## Job"
|
||||
echo
|
||||
echo '```'
|
||||
sacct -j "$JOBID" --format=JobID,JobName,Partition,QOS,NNodes,NTasks,Elapsed,State 2>/dev/null \
|
||||
|| echo "sacct unavailable"
|
||||
echo '```'
|
||||
echo
|
||||
|
||||
echo "## Parameters"
|
||||
echo
|
||||
echo '```'
|
||||
grep -E "^(DIM|NREP|NLEV|ITMAX|RANK_POINTS)=" amg_comm_strong.sh 2>/dev/null
|
||||
echo '```'
|
||||
echo
|
||||
|
||||
echo "## Contents"
|
||||
echo
|
||||
echo '```'
|
||||
find "$RESDIR" -type f -printf "%10s %p\n" | sort -k2
|
||||
echo '```'
|
||||
} > "$OUT"
|
||||
|
||||
# Keep the exact script that produced the numbers: parameters drift.
|
||||
cp -p amg_comm_strong.sh "$RESDIR/amg_comm_strong.sh.used" 2>/dev/null || true
|
||||
|
||||
TARBALL=${RESDIR%/}.tar.gz
|
||||
tar czf "$TARBALL" "$RESDIR"
|
||||
|
||||
echo "Provenance written to $OUT"
|
||||
echo "Tarball: $TARBALL ($(du -h "$TARBALL" | cut -f1))"
|
||||
echo
|
||||
echo "Transfer with:"
|
||||
echo " rsync -avz <user>@<host>:$(readlink -f "$TARBALL") ."
|
||||
Reference in New Issue
Block a user