VASP 6.4.1¶

The following instructions are for building VASP 6.4.1 using the NVIDIA HPC toolkit 22.11 compilers on the DCS cluster (currently only on dcs101).

Environment Setup¶

module use /opt/nvidia/hpc_sdk/modulefiles/nvhpc/
module load 22.11
module load cmake

# avoid hang during mpi init caused by old openmpi version provided with nvhpc
export OMPI_MCA_ess_singleton_isolated=1
# silence warning about there being multiple ports
export OMPI_MCA_btl_openib_warn_default_gid_prefix=0

FFTW 3.3.5¶

download¶

this likely won't work on cci - you'll have to scp the tarball to blp0[1|2]

wget http://www.fftw.org/fftw-3.3.5.tar.gz

build and install¶

tar xf fftw-3.3.5.tar.gz
cd fftw-3.3.5
./configure --enable-openmp --prefix=/absolute/path/to/fftw-3.3.5-install
make install

VASP¶

setup¶

cp arch/makefile.include.nvhpc_omp_acc ./makefile.include

make the following edits to makefile.include to set the gpu architecture flag and path to the FFTW install dir

$ diff arch/makefile.include.nvhpc_omp_acc makefile.include
20,21c20,21
< FC          = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.0 -mp
< FCL         = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.0 -mp -c++libs
---
> FC          = mpif90 -acc -gpu=cc70,cuda11.0 -mp
> FCL         = mpif90 -acc -gpu=cc70,cuda11.0 -mp -c++libs
92c92
< FFTW_ROOT  ?= /path/to/your/fftw/installation
---
> FFTW_ROOT  ?= /absolute/path/to/fftw-3.3.5-install

build¶

make DEPS=1 -j4 all

test¶

These tests use multiple gpus and cores and will run for more than 20 minutes. Please don't run them on the front end nodes.

cd testsuite
make test

VASP 6.2.1¶

The following instructions are for building VASP 6.2.1 using the NVIDIA HPC toolkit 21.3 compilers on the DCS cluster.

Environment Setup¶

module use /opt/nvidia/hpc_sdk/modulefiles/nvhpc/
module load 21.3
module load cmake

export CUDA_ROOT=/opt/nvidia/hpc_sdk/Linux_ppc64le/21.3/cuda/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/nvidia/hpc_sdk/Linux_ppc64le/21.3/./compilers/extras/qd/lib/

# avoid hang during mpi init caused by old openmpi version provided with nvhpc
export OMPI_MCA_ess_singleton_isolated=1
# silence warning about there being multiple ports
export OMPI_MCA_btl_openib_warn_default_gid_prefix=0

FFTW 3.3.5¶

download¶

this likely won't work on cci - you'll have to scp the tarball to blp0[1|2]

wget http://www.fftw.org/fftw-3.3.5.tar.gz

build and install¶

tar xf fftw-3.3.5.tar.gz
cd fftw-3.3.5
./configure --prefix=/absolute/path/to/fftw-3.3.5-install
make install

VASP¶

setup¶

cp arch/makefile.include.linux_nv_acc ./makefile.include

make the following edits to makefile.include to set the gpu architecture flag and path to the FFTW install dir

$ diff arch/makefile.include.linux_nv_acc makefile.include
17,18c17,18
< FC = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.0
< FCL = mpif90 -acc -gpu=cc60,cc70,cc80,cuda11.0 -c++libs
---
> FC = mpif90 -acc -gpu=cc70,cuda11.2
> FCL = mpif90 -acc -gpu=cc70,cuda11.2 -c++libs
52c52
< FFTW ?= /opt/gnu/fftw-3.3.6-pl2-GNU-5.4.0
---
> FFTW ?= /absolute/path/to/fftw-3.3.5-install

build¶

make all

test¶

These tests use multiple gpus and cores. Please don't run them all on the front end nodes.

cd testsuite
make test

VASP 5.4.4¶

The following instructions are for building VASP 5.4.4 using the GNU compilers and Spectrum MPI on AiMOS.

The following instructions have not been tested on AiMOS RedHat8.

Environment Setup¶

Run the following commands to setup your environment:

module use /gpfs/u/software/dcs-rhel8-spack-install/v0162gccSpectrum/lmod/linux-rhel8-ppc64le/Core/
module load spectrum-mpi/10.4-2ycgnlq
module load cuda
module load netlib-scalapack netlib-lapack fftw

export OMPI_CXX=g++
export OMPI_CC=gcc
export OMPI_FC=gfortran

Create the makefile¶

In the VASP source code directory, create a file named makefile.include with the following contents

# Precompiler options
CPP_OPTIONS= -DHOST=\"LinuxGNU\" \
             -DMPI -DMPI_BLOCK=8000 \
             -Duse_collective \
             -DscaLAPACK \
             -DCACHE_SIZE=4000 \
             -Davoidalloc \
             -Duse_bse_te \
             -Dtbdyn \
             -Duse_shmem

CPP        = gcc -E -P -C -w $*$(FUFFIX) >$*$(SUFFIX) $(CPP_OPTIONS)

FC         = mpif90
FCL        = mpif90

FREE       = -ffree-form -ffree-line-length-none

FFLAGS     = -w
OFLAG      = -O2
OFLAG_IN   = $(OFLAG)
DEBUG      = -O0

LIBDIR     = /opt/gfortran/libs/
BLAS       = -L$(OPENBLAS_ROOT) -lopenblas
LAPACK     = -L$(NETLIB_LAPACK_ROOT) -ltmglib -llapack
BLACS      =
SCALAPACK  = -L$(NETLIB_SCALAPACK_ROOT) -lscalapack $(BLACS)

LLIBS      = $(SCALAPACK) $(LAPACK) $(BLAS)

FFTW       ?= $(FFTW_ROOT)
LLIBS      += -L$(FFTW)/lib -lfftw3
INCS       = -I$(FFTW)/include

OBJECTS    = fftmpiw.o fftmpi_map.o  fftw3d.o  fft3dlib.o

OBJECTS_O1 += fftw3d.o fftmpi.o fftmpiw.o
OBJECTS_O2 += fft3dlib.o

# For what used to be vasp.5.lib
CPP_LIB    = $(CPP)
FC_LIB     = $(FC)
CC_LIB     = gcc
CFLAGS_LIB = -O
FFLAGS_LIB = -O1
FREE_LIB   = $(FREE)

OBJECTS_LIB= linpack_double.o getshmem.o

# For the parser library
CXX_PARS   = g++

LIBS       += parser
LLIBS      += -Lparser -lparser -lstdc++

# Normally no need to change this
SRCDIR     = ../../src
BINDIR     = ../../bin

#================================================
# GPU Stuff

CPP_GPU    = -DCUDA_GPU -DRPROMU_CPROJ_OVERLAP -DCUFFT_MIN=28 -UscaLAPACK # -DUSE_PINNED_MEMORY

OBJECTS_GPU= fftmpiw.o fftmpi_map.o fft3dlib.o fftw3d_gpu.o fftmpiw_gpu.o

CC         = gcc
CXX        = g++
CFLAGS     = -fPIC -DADD_ -openmp -DMAGMA_WITH_MKL -DMAGMA_SETAFFINITY -DGPUSHMEM=300 -DHAVE_CUBLAS

CUDA_ROOT  ?= $(CUDA_PATH)
NVCC       := $(CUDA_ROOT)/bin/nvcc
CUDA_LIB   := -L$(CUDA_ROOT)/lib64 -lnvToolsExt -lcudart -lcuda -lcufft -lcublas

GENCODE_ARCH    := -gencode=arch=compute_60,code=\"sm_70,compute_60\"

MPI_INC    = $(SPECTRUM_MPI_ROOT)/include

Build¶

cd vasp.5.4.4.pl2
make all
make gpu