From 84d22c8429ae6680a5196c9bf703bae6c92e535c Mon Sep 17 00:00:00 2001 From: Stack-1 Date: Sat, 8 Aug 2026 15:02:35 +0200 Subject: [PATCH] [ADD] Build and run the per-type comm test drivers The four drivers the template generates in each of test/comm/{swapdata, cg,spmv} were copied into the tree but appeared in no make rule, so they were never compiled and never run. Only the generic double-precision driver was built. That is why the complex halo exchange could stay broken without anyone noticing. Each Makefile now builds the four typed executables alongside the generic one, which is kept: psb_spmv_kernel and psb_comm_cg_test are referenced by the sbatch scripts. clean covers the new objects and executables, and there are run/run-typed targets with NP and IDIM as before. In cg and spmv the typed drivers are built CPU-only, without -DPSB_HAVE_CUDA and without libpsb_cuda. The CUDA layer has no complex support upstream: psb_?_cuda_vect_mod declares an external cdot/zdot, which are not BLAS symbols (the complex dots are cdotc/cdotu), and cuda/Makefile never compiles the c/z hdiag modules. Both are present in development too, so this is not something the branch introduced. Building all four the same way also keeps the cross-type comparison honest. For the two types the CUDA layer does support, 'make gpu' builds psb_{s,d}_*_gpu variants from the same sources with the CUDA defines and libraries, into separate objects so switching between the two does not force a full rebuild; 'make both' does both. The spmv drivers declare a module, so the GPU build writes its module files to gpumod/ to avoid the two variants overwriting each other. --- test/comm/cg/Makefile | 50 ++++++++++++++++++++++++++++++-- test/comm/spmv/Makefile | 57 +++++++++++++++++++++++++++++++++++-- test/comm/swapdata/Makefile | 21 ++++++++++++-- 3 files changed, 122 insertions(+), 6 deletions(-) diff --git a/test/comm/cg/Makefile b/test/comm/cg/Makefile index 6bc3ee304..3e86deb81 100644 --- a/test/comm/cg/Makefile +++ b/test/comm/cg/Makefile @@ -15,10 +15,34 @@ IDIM ?= 40 PROGSRC=psb_comm_cg_test.F90 TOBJS=psb_comm_cg_test.o +# Per-type drivers generated from the template (X_comm_cg_test.P90). +# +# Built CPU-only, without -DPSB_HAVE_CUDA and without -lpsb_cuda: the CUDA +# layer has no complex support upstream (psb_?_cuda_vect_mod declares an +# external cdot/zdot, which are not BLAS symbols, and cuda/Makefile never +# compiles the c/z hdiag modules), so c/z cannot be linked against it. +# Building all four the same way keeps the cross-type comparison honest. +# psb_ext stays in: it provides the CPU storage formats (ELL, HLL, DIA, HDIA). +TYPED=psb_s_comm_cg_test psb_d_comm_cg_test psb_c_comm_cg_test psb_z_comm_cg_test +TYPEDOBJS=$(addsuffix .o,$(TYPED)) +CPU_PSBLAS_LIB= -L$(LIBDIR) -lpsb_util -lpsb_linsolve -lpsb_prec -lpsb_ext -lpsb_base +CPU_LDLIBS=$(PSBLDLIBS) + +# GPU variants of the same drivers, s/d only for the reason above. Same +# sources, compiled with -DPSB_HAVE_CUDA and linked against libpsb_cuda, into +# differently named objects and executables so the two builds coexist. These +# drivers declare no module, so no module-file separation is needed. +TYPED_GPU=psb_s_comm_cg_test_gpu psb_d_comm_cg_test_gpu +TYPED_GPUOBJS=$(addsuffix .o,$(TYPED_GPU)) + EXEDIR=./runs EXE=psb_comm_cg_test -all: runsd $(EXE) +all: runsd $(EXE) $(TYPED) + +gpu: runsd $(TYPED_GPU) + +both: all gpu runsd: (if test ! -d runs ; then mkdir runs; fi) @@ -26,12 +50,34 @@ runsd: psb_comm_cg_test.o: $(PROGSRC) $(FC) $(FCOPT) $(FINCLUDES) $(FDEFINES) $(FCUDEFINES) -c $(PROGSRC) -o $@ +$(TYPEDOBJS): %.o: %.F90 + $(FC) $(FCOPT) $(FINCLUDES) $(FDEFINES) -c $< -o $@ + +$(TYPED_GPUOBJS): %_gpu.o: %.F90 + $(FC) $(FCOPT) $(FINCLUDES) $(FDEFINES) $(FCUDEFINES) -c $< -o $@ + $(EXE): $(TOBJS) $(FLINK) $(LOPT) $(TOBJS) -o $(EXE) $(PSBLAS_LIB) $(LDLIBS) /bin/mv $(EXE) $(EXEDIR) +$(TYPED): %: %.o + $(FLINK) $(LOPT) $< -o $@ $(CPU_PSBLAS_LIB) $(CPU_LDLIBS) + /bin/mv $@ $(EXEDIR) + +$(TYPED_GPU): %: %.o + $(FLINK) $(LOPT) $< -o $@ $(PSBLAS_LIB) $(LDLIBS) + /bin/mv $@ $(EXEDIR) + run: all mpirun -np $(NP) $(EXEDIR)/$(EXE) $(IDIM) +run-typed: all + for t in $(TYPED); do \ + echo "=== $$t"; \ + mpirun -np $(NP) $(EXEDIR)/$$t $(IDIM) || exit 1; \ + done + clean: - /bin/rm -f $(TOBJS) *$(.mod) $(EXEDIR)/$(EXE) + /bin/rm -f $(TOBJS) $(TYPEDOBJS) $(TYPED_GPUOBJS) *$(.mod) \ + $(EXEDIR)/$(EXE) $(addprefix $(EXEDIR)/,$(TYPED)) \ + $(addprefix $(EXEDIR)/,$(TYPED_GPU)) diff --git a/test/comm/spmv/Makefile b/test/comm/spmv/Makefile index 080cbfcf4..9db65cf87 100644 --- a/test/comm/spmv/Makefile +++ b/test/comm/spmv/Makefile @@ -13,9 +13,38 @@ FINCLUDES=$(FMFLAG)$(MODDIR) $(FMFLAG). TOBJS=psb_spmv_test.o +# Per-type drivers generated from the template (X_spmv_test.p90). +# +# Built CPU-only, without -DPSB_HAVE_CUDA and without -lpsb_cuda: the CUDA +# layer has no complex support upstream (psb_?_cuda_vect_mod declares an +# external cdot/zdot, which are not BLAS symbols, and cuda/Makefile never +# compiles the c/z hdiag modules), so c/z cannot be linked against it. +# Building all four the same way keeps the cross-type comparison honest. +# psb_ext stays in: it provides the CPU storage formats (ELL, HLL, DIA, HDIA). +TYPED=psb_s_spmv_test psb_d_spmv_test psb_c_spmv_test psb_z_spmv_test +TYPEDOBJS=$(addsuffix .o,$(TYPED)) +CPU_PSBLAS_LIB= -L$(LIBDIR) -lpsb_util -lpsb_linsolve -lpsb_prec -lpsb_ext -lpsb_base +CPU_LDLIBS=$(PSBLDLIBS) + +# GPU variants of the same drivers, s/d only for the reason above. +# Same sources, compiled with -DPSB_HAVE_CUDA and linked against libpsb_cuda, +# into differently named objects and executables so the two builds coexist. +# The source declares a module (psb_?spmv_overlap_test), so the GPU build sends +# its .mod files to GPUMODDIR: without that the two variants of the same type +# would overwrite each other's module file. +TYPED_GPU=psb_s_spmv_test_gpu psb_d_spmv_test_gpu +TYPED_GPUOBJS=$(addsuffix .o,$(TYPED_GPU)) +GPUMODDIR=./gpumod + +NP ?= 4 + EXEDIR=./runs -all: runsd psb_spmv_kernel +all: runsd psb_spmv_kernel $(TYPED) + +gpu: runsd $(TYPED_GPU) + +both: all gpu runsd: (if test ! -d runs ; then mkdir runs; fi) @@ -23,12 +52,36 @@ runsd: psb_spmv_test.o: psb_spmv_test.f90 $(FC) $(FCOPT) -cpp $(FINCLUDES) $(FDEFINES) $(FCUDEFINES) -c $< -o $@ +$(TYPEDOBJS): %.o: %.f90 + $(FC) $(FCOPT) -cpp $(FINCLUDES) $(FDEFINES) -c $< -o $@ + +$(TYPED_GPUOBJS): %_gpu.o: %.f90 + (if test ! -d $(GPUMODDIR) ; then mkdir $(GPUMODDIR); fi) + $(FC) $(FCOPT) -cpp -J$(GPUMODDIR) $(FINCLUDES) $(FDEFINES) $(FCUDEFINES) -c $< -o $@ + psb_spmv_kernel: $(TOBJS) $(FLINK) $(LOPT) $(TOBJS) -o psb_spmv_kernel $(PSBLAS_LIB) $(LDLIBS) /bin/mv psb_spmv_kernel $(EXEDIR) +$(TYPED): %: %.o + $(FLINK) $(LOPT) $< -o $@ $(CPU_PSBLAS_LIB) $(CPU_LDLIBS) + /bin/mv $@ $(EXEDIR) + +$(TYPED_GPU): %: %.o + $(FLINK) $(LOPT) $< -o $@ $(PSBLAS_LIB) $(LDLIBS) + /bin/mv $@ $(EXEDIR) + +run-typed: all + for t in $(TYPED); do \ + echo "=== $$t"; \ + mpirun -np $(NP) $(EXEDIR)/$$t || exit 1; \ + done + clean: - /bin/rm -f $(TOBJS) $(TOBJS_API) *$(.mod) $(EXEDIR)/spmv_overlap $(EXEDIR)/psb_spmv_kernel + /bin/rm -f $(TOBJS) $(TYPEDOBJS) $(TYPED_GPUOBJS) $(TOBJS_API) *$(.mod) \ + $(EXEDIR)/spmv_overlap $(EXEDIR)/psb_spmv_kernel \ + $(addprefix $(EXEDIR)/,$(TYPED)) $(addprefix $(EXEDIR)/,$(TYPED_GPU)) + /bin/rm -rf $(GPUMODDIR) lib: (cd ../../; make library) diff --git a/test/comm/swapdata/Makefile b/test/comm/swapdata/Makefile index 988410d28..bf555697e 100644 --- a/test/comm/swapdata/Makefile +++ b/test/comm/swapdata/Makefile @@ -13,9 +13,15 @@ FINCLUDES=$(FMFLAG)$(MODDIR) $(FMFLAG). TOBJS=psb_comm_test.o +# Per-type drivers generated from the template (X_comm_test.P90). +TYPED=psb_s_comm_test psb_d_comm_test psb_c_comm_test psb_z_comm_test +TYPEDOBJS=$(addsuffix .o,$(TYPED)) + +NP ?= 4 + EXEDIR=./runs -all: runsd psb_comm_test +all: runsd psb_comm_test $(TYPED) runsd: (if test ! -d runs ; then mkdir runs; fi) @@ -24,8 +30,19 @@ psb_comm_test: $(TOBJS) $(FLINK) $(LOPT) $(TOBJS) -o psb_comm_test $(PSBLAS_LIB) $(LDLIBS) /bin/mv psb_comm_test $(EXEDIR) +$(TYPED): %: %.o + $(FLINK) $(LOPT) $< -o $@ $(PSBLAS_LIB) $(LDLIBS) + /bin/mv $@ $(EXEDIR) + +run: all + for t in $(TYPED); do \ + echo "=== $$t"; \ + mpirun -np $(NP) $(EXEDIR)/$$t || exit 1; \ + done + clean: - /bin/rm -f $(TOBJS) $(TOBJS_API) *$(.mod) $(EXEDIR)/psb_comm_test + /bin/rm -f $(TOBJS) $(TYPEDOBJS) $(TOBJS_API) *$(.mod) \ + $(EXEDIR)/psb_comm_test $(addprefix $(EXEDIR)/,$(TYPED)) lib: (cd ../../; make library)