You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
97 lines
3.6 KiB
Bash
97 lines
3.6 KiB
Bash
#!/bin/bash
|
|
# ============================================================================
|
|
# strong_spmv.sbatch STRONG scaling, CPU-only SpMV across comm backends
|
|
#
|
|
# Fixed total problem size; MPI ranks grow across scale points. The spmv test
|
|
# sweeps ALL comm backends internally (P2P, NEIGHBOR, PNEIGHBOR, MPI_GET,
|
|
# MPI_PUT) in a single run, so there is no per-mode loop.
|
|
# All scale points run inside ONE reserved allocation (same hardware).
|
|
# ============================================================================
|
|
#SBATCH --job-name=spmv_strong
|
|
#SBATCH --output=spmv_strong_%j.out
|
|
#SBATCH --error=spmv_strong_%j.err
|
|
#SBATCH --nodes=8 # reserve the maximum up front
|
|
#SBATCH --ntasks-per-node=80 # 80 cores per ACC node -> 80 ranks/node
|
|
#SBATCH --cpus-per-task=1 # CPU-only: one core per rank
|
|
#SBATCH --time=00:30:00
|
|
#SBATCH --qos=acc_debug
|
|
#SBATCH --account=ehpc580
|
|
|
|
# ============================================================================
|
|
# USER CONFIGURATION
|
|
# ============================================================================
|
|
DIM=280 # FIXED problem size (idim^3 unknowns)
|
|
TIMES=50 # SpMV repetitions (timed)
|
|
CPU_FMT=CSR # CPU matrix storage format
|
|
PROFILE=true # Score-P profiling on/off
|
|
RANK_POINTS="80 160 320 640" # total ranks per scale point (multiples of 80)
|
|
|
|
EXE=$HOME/Desktop/scorep/psblas3/test/comm/spmv/runs/psb_spmv_kernel
|
|
|
|
# ============================================================================
|
|
# ENVIRONMENT
|
|
# ============================================================================
|
|
module purge
|
|
module load bsc/1.0
|
|
module load nvidia-hpc-sdk/25.3
|
|
module load gcc/12.3.0
|
|
module load ucx/1.16.0-gcc
|
|
module load openmpi/5.0.5-gcc
|
|
module load openblas/0.3.27-gcc
|
|
export PATH="$HOME/scorep-cuda/bin:$PATH"
|
|
export LD_LIBRARY_PATH="$HOME/scorep-cuda/lib:$LD_LIBRARY_PATH"
|
|
|
|
export OMPI_MCA_coll_hcoll_enable=0
|
|
export SCOREP_CUDA_ENABLE=no # CPU-only run: silence Score-P CUDA adapter
|
|
|
|
RESDIR=$SLURM_SUBMIT_DIR/results_spmv_strong_${SLURM_JOB_ID}
|
|
mkdir -p $RESDIR
|
|
|
|
# --- Score-P region filter: exclude the high-frequency tiny USR functions so
|
|
# the profile reflects real MPI/comm time, not instrumentation overhead. ---
|
|
FILTER=$RESDIR/scorep.filt
|
|
cat > $FILTER <<'EOF'
|
|
SCOREP_REGION_NAMES_BEGIN
|
|
EXCLUDE
|
|
psb_indx_map_mod::*
|
|
psb_desc_mod::*
|
|
psb_error_mod::*
|
|
psb_gen_block_map_mod::*
|
|
psi_penv_mod::*
|
|
psb_hash_mod::*
|
|
SCOREP_REGION_NAMES_END
|
|
EOF
|
|
|
|
echo "=== SPMV STRONG scaling (CPU-only) ==="
|
|
echo " fixed_dim=$DIM times=$TIMES cpu_fmt=$CPU_FMT"
|
|
echo " reserved_nodes=$SLURM_NNODES rank_points=[$RANK_POINTS] profile=$PROFILE"
|
|
echo "======================================"
|
|
|
|
for NRANKS in $RANK_POINTS; do
|
|
NNODES=$(( (NRANKS + 79) / 80 )) # 80 ranks per node
|
|
STEP_DIR=$RESDIR/${NRANKS}ranks
|
|
mkdir -p $STEP_DIR
|
|
|
|
if [ "$PROFILE" = "true" ]; then
|
|
export SCOREP_ENABLE_PROFILING=true
|
|
export SCOREP_ENABLE_TRACING=false
|
|
export SCOREP_TOTAL_MEMORY=128M
|
|
export SCOREP_FILTERING_FILE=$FILTER
|
|
export SCOREP_EXPERIMENT_DIRECTORY=$STEP_DIR/scorep_profile
|
|
else
|
|
export SCOREP_ENABLE_PROFILING=false
|
|
export SCOREP_ENABLE_TRACING=false
|
|
fi
|
|
|
|
echo ""
|
|
echo ">>> STRONG point: $NRANKS ranks ($NNODES nodes), fixed dim=$DIM"
|
|
srun -N $NNODES -n $NRANKS --ntasks-per-node=80 --cpus-per-task=1 \
|
|
$EXE --gpu=FALSE --dim=$DIM --times=$TIMES --cpu_fmt=$CPU_FMT \
|
|
> $STEP_DIR/run.out 2>&1
|
|
echo ">>> exit=$? output=$STEP_DIR/run.out"
|
|
done
|
|
|
|
echo ""
|
|
echo "=== SPMV STRONG DONE. Results: $RESDIR ==="
|
|
ls -R $RESDIR
|