VASP 6.5.1¶
The following instructions are for building VASP 6.5.1 with HDF5 support enabled using the NVIDIA HPC toolkit 22.11 compilers on the DCS cluster in the dcs-2024 partition.
Allocate resources on a compute node¶
The following command will give you one GPU and 20 cores from the dcs-2024 partition for four hours.
salloc -N 1 -n 20 -t 240 -p dcs-2024 --gres=gpu:1
# wait for allocation
ssh <allocatedNode>
ALL of the following instructions are run on the allocated compute node
Environment Setup¶
module use /opt/nvidia/hpc_sdk/modulefiles/nvhpc/
module load 22.11
module load cmake
# avoid hang during mpi init caused by old openmpi version provided with nvhpc
export OMPI_MCA_ess_singleton_isolated=1
# silence warning about there being multiple ports
export OMPI_MCA_btl_openib_warn_default_gid_prefix=0
FFTW 3.3.10¶
wget https://www.fftw.org/fftw-3.3.10.tar.gz
tar xf fftw-3.3.10.tar.gz
cd fftw-3.3.10
./configure --enable-openmp --prefix=/absolute/path/to/fftw-3.3.10/install/dir
make install -j20
HDF5 1.14.6¶
The following instructions follow the process detailed in hdf5-1.14.6/release_docs/INSTALL_CMake.txt.
setup¶
mkdir hdf5-1.14.6-dev
cd hdf5-1.14.6-dev
# scp the hdf5-1.14.6.zip to this directory
unzip hdf5-1.14.6.zip
#download the compression libs and plugins
wget https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz
wget https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.2.2.tar.gz
wget https://github.com/MathisRosenhauer/libaec/releases/download/v1.1.3/libaec-1.1.3.tar.gz
wget https://github.com/HDFGroup/hdf5_plugins/releases/download/hdf5-1.14.6/hdf5_plugins-1.14.tar.gz
#copy the install scripts
cp hdf5-1.14.6/config/cmake/scripts/HDF5config.cmake .
cp hdf5-1.14.6/config/cmake/scripts/CTestScript.cmake .
cp hdf5-1.14.6/config/cmake/scripts/HDF5options.cmake .
edit the install options¶
$ diff hdf5-1.14.6/config/cmake/scripts/HDF5options.cmake HDF5options.cmake
41c41
< #set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=ON")
---
> set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=ON")
43c43
< set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=OFF")
---
> #set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_FORTRAN:BOOL=OFF")
run the install script¶
The ctest command takes about an hour.
ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -VV -O hdf5.log
mkdir hdf5-1.14.6-install
cd hdf5-1.14.6-install
# accept the license then enter 'n' to skip creation of subdir
~/barn/systemSoftware/vasp651/hdf5-1.14.6-dev/HDF5-1.14.6-Linux.sh
VASP¶
setup¶
cp arch/makefile.include.nvhpc_omp_acc ./makefile.include
make the following edits to makefile.include to set the gpu architecture flag and path to the FFTW install dir. Be sure to set the correct paths for FFTW_ROOT and HDF5_ROOT
$ diff arch/makefile.include.nvhpc_omp_acc makefile.include
20,22c20,22
< CC = mpicc -acc -gpu=cc60,cc70,cc80,cuda11.8 -mp
< FC = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.8 -mp
< FCL = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.8 -mp -c++libs
---
> CC = mpicc -acc -gpu=cc70,cuda11.0 -mp
> FC = mpif90 -acc -gpu=cc70,cuda11.0 -mp
> FCL = mpif90 -acc -gpu=cc70,cuda11.0 -mp -c++libs
91c91
< FFTW_ROOT ?= /path/to/your/fftw/installation
---
> FFTW_ROOT ?= /path/to/fftw-3.3.10-install
101,104c101,104
< #CPP_OPTIONS+= -DVASP_HDF5
< #HDF5_ROOT ?= /path/to/your/hdf5/installation
< #LLIBS += -L$(HDF5_ROOT)/lib -lhdf5_fortran
< #INCS += -I$(HDF5_ROOT)/include
---
> CPP_OPTIONS+= -DVASP_HDF5
> HDF5_ROOT ?= /path/to/hdf5-1.14.6-install/HDF_Group/HDF5/1.14.6
> LLIBS += -L$(HDF5_ROOT)/lib -lhdf5_fortran
> INCS += -I$(HDF5_ROOT)/include -I$(HDF5_ROOT)/mod/static
build¶
make DEPS=1 -j20 all
test¶
These tests use multiple gpus and cores and will run for more than 20 minutes. Please don't run them on the front end nodes.
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/gpfs/u/barn/CCNI/CCNIsmth/systemSoftware/vasp651/hdf5-1.14.6-install/HDF_Group/HDF5/1.14.6/lib
make test
VASP 6.4.1¶
The following instructions are for building VASP 6.4.1 using the NVIDIA HPC toolkit 22.11 compilers on the DCS cluster (currently only on dcs101).
Environment Setup¶
module use /opt/nvidia/hpc_sdk/modulefiles/nvhpc/
module load 22.11
module load cmake
# avoid hang during mpi init caused by old openmpi version provided with nvhpc
export OMPI_MCA_ess_singleton_isolated=1
# silence warning about there being multiple ports
export OMPI_MCA_btl_openib_warn_default_gid_prefix=0
FFTW 3.3.5¶
download¶
this likely won't work on cci - you'll have to scp the tarball to blp0[1|2]
wget http://www.fftw.org/fftw-3.3.5.tar.gz
build and install¶
tar xf fftw-3.3.5.tar.gz
cd fftw-3.3.5
./configure --enable-openmp --prefix=/absolute/path/to/fftw-3.3.5-install
make install
VASP¶
setup¶
cp arch/makefile.include.nvhpc_omp_acc ./makefile.include
make the following edits to makefile.include to set the gpu architecture flag and path to the FFTW install dir
$ diff arch/makefile.include.nvhpc_omp_acc makefile.include
20,21c20,21
< FC = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.0 -mp
< FCL = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.0 -mp -c++libs
---
> FC = mpif90 -acc -gpu=cc70,cuda11.0 -mp
> FCL = mpif90 -acc -gpu=cc70,cuda11.0 -mp -c++libs
92c92
< FFTW_ROOT ?= /path/to/your/fftw/installation
---
> FFTW_ROOT ?= /absolute/path/to/fftw-3.3.5-install
build¶
make DEPS=1 -j4 all
test¶
These tests use multiple gpus and cores and will run for more than 20 minutes. Please don't run them on the front end nodes.
cd testsuite
make test
VASP 6.2.1¶
The following instructions are for building VASP 6.2.1 using the NVIDIA HPC toolkit 21.3 compilers on the DCS cluster.
Environment Setup¶
module use /opt/nvidia/hpc_sdk/modulefiles/nvhpc/
module load 21.3
module load cmake
export CUDA_ROOT=/opt/nvidia/hpc_sdk/Linux_ppc64le/21.3/cuda/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/nvidia/hpc_sdk/Linux_ppc64le/21.3/./compilers/extras/qd/lib/
# avoid hang during mpi init caused by old openmpi version provided with nvhpc
export OMPI_MCA_ess_singleton_isolated=1
# silence warning about there being multiple ports
export OMPI_MCA_btl_openib_warn_default_gid_prefix=0
FFTW 3.3.5¶
download¶
this likely won't work on cci - you'll have to scp the tarball to blp0[1|2]
wget http://www.fftw.org/fftw-3.3.5.tar.gz
build and install¶
tar xf fftw-3.3.5.tar.gz
cd fftw-3.3.5
./configure --prefix=/absolute/path/to/fftw-3.3.5-install
make install
VASP¶
setup¶
cp arch/makefile.include.linux_nv_acc ./makefile.include
make the following edits to makefile.include to set the gpu architecture flag and path to the FFTW install dir
$ diff arch/makefile.include.linux_nv_acc makefile.include
17,18c17,18
< FC = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.0
< FCL = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.0 -c++libs
---
> FC = mpif90 -acc -gpu=cc70,cuda11.2
> FCL = mpif90 -acc -gpu=cc70,cuda11.2 -c++libs
52c52
< FFTW ?= /opt/gnu/fftw-3.3.6-pl2-GNU-5.4.0
---
> FFTW ?= /absolute/path/to/fftw-3.3.5-install
build¶
make all
test¶
These tests use multiple gpus and cores. Please don't run them all on the front end nodes.
cd testsuite
make test
VASP 5.4.4¶
The following instructions are for building VASP 5.4.4 using the GNU compilers and Spectrum MPI on AiMOS.
The following instructions have not been tested on AiMOS RedHat8.
Environment Setup¶
Run the following commands to setup your environment:
module use /gpfs/u/software/dcs-rhel8-spack-install/v0162gccSpectrum/lmod/linux-rhel8-ppc64le/Core/
module load spectrum-mpi/10.4-2ycgnlq
module load cuda
module load netlib-scalapack netlib-lapack fftw
export OMPI_CXX=g++
export OMPI_CC=gcc
export OMPI_FC=gfortran
Create the makefile¶
In the VASP source code directory, create a file named
makefile.include with the following contents
# Precompiler options
CPP_OPTIONS= -DHOST=\"LinuxGNU\" \
-DMPI -DMPI_BLOCK=8000 \
-Duse_collective \
-DscaLAPACK \
-DCACHE_SIZE=4000 \
-Davoidalloc \
-Duse_bse_te \
-Dtbdyn \
-Duse_shmem
CPP = gcc -E -P -C -w $*$(FUFFIX) >$*$(SUFFIX) $(CPP_OPTIONS)
FC = mpif90
FCL = mpif90
FREE = -ffree-form -ffree-line-length-none
FFLAGS = -w
OFLAG = -O2
OFLAG_IN = $(OFLAG)
DEBUG = -O0
LIBDIR = /opt/gfortran/libs/
BLAS = -L$(OPENBLAS_ROOT) -lopenblas
LAPACK = -L$(NETLIB_LAPACK_ROOT) -ltmglib -llapack
BLACS =
SCALAPACK = -L$(NETLIB_SCALAPACK_ROOT) -lscalapack $(BLACS)
LLIBS = $(SCALAPACK) $(LAPACK) $(BLAS)
FFTW ?= $(FFTW_ROOT)
LLIBS += -L$(FFTW)/lib -lfftw3
INCS = -I$(FFTW)/include
OBJECTS = fftmpiw.o fftmpi_map.o fftw3d.o fft3dlib.o
OBJECTS_O1 += fftw3d.o fftmpi.o fftmpiw.o
OBJECTS_O2 += fft3dlib.o
# For what used to be vasp.5.lib
CPP_LIB = $(CPP)
FC_LIB = $(FC)
CC_LIB = gcc
CFLAGS_LIB = -O
FFLAGS_LIB = -O1
FREE_LIB = $(FREE)
OBJECTS_LIB= linpack_double.o getshmem.o
# For the parser library
CXX_PARS = g++
LIBS += parser
LLIBS += -Lparser -lparser -lstdc++
# Normally no need to change this
SRCDIR = ../../src
BINDIR = ../../bin
#================================================
# GPU Stuff
CPP_GPU = -DCUDA_GPU -DRPROMU_CPROJ_OVERLAP -DCUFFT_MIN=28 -UscaLAPACK # -DUSE_PINNED_MEMORY
OBJECTS_GPU= fftmpiw.o fftmpi_map.o fft3dlib.o fftw3d_gpu.o fftmpiw_gpu.o
CC = gcc
CXX = g++
CFLAGS = -fPIC -DADD_ -openmp -DMAGMA_WITH_MKL -DMAGMA_SETAFFINITY -DGPUSHMEM=300 -DHAVE_CUBLAS
CUDA_ROOT ?= $(CUDA_PATH)
NVCC := $(CUDA_ROOT)/bin/nvcc
CUDA_LIB := -L$(CUDA_ROOT)/lib64 -lnvToolsExt -lcudart -lcuda -lcufft -lcublas
GENCODE_ARCH := -gencode=arch=compute_60,code=\"sm_70,compute_60\"
MPI_INC = $(SPECTRUM_MPI_ROOT)/include
Build¶
cd vasp.5.4.4.pl2
make all
make gpu