gardar.nhpc.hi.is

Information about the machine http://nhpc.hi.is/content/nhpc-system

Instructions assume bash, installation under ${HOME}/global/apps.

Make sure the following modules are loaded:

$ module list

Currently Loaded Modules:
  1) intel/13.1   3) openmpi/1.6.5   5) gold/2.2.0.5
  2) mkl/11.0.0   4) python/2.7.5

Setup the root directory:

export APPHOME=${HOME}/global/apps

mkdir -p ${APPHOME}
cd ${APPHOME}

export GPAW_PLATFORM=`python -c "from distutils import util, sysconfig; print(util.get_platform()+'-'+sysconfig.get_python_version())"`

Download software:

svn co -r 3906 https://svn.fysik.dtu.dk/projects/ase/trunk ase.3906
svn co -r 12224 https://svn.fysik.dtu.dk/projects/gpaw/trunk gpaw.12224
wget https://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-0.9.11271.tar.gz
wget https://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-0.9.9672.tar.gz
tar zxf gpaw-setups-0.9.11271.tar.gz
tar zxf gpaw-setups-0.9.9672.tar.gz
wget "http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.1.tar.gz" -O libxc-2.2.1.tar.gz
tar zxf libxc-2.2.1.tar.gz

and create the modules:

mkdir -p $APPHOME/modulefiles/{ase,gpaw,gpaw-setups,libxc}

cat <<EOF > modulefiles/gpaw/0.11.0.12224
set modulefile [lrange [split [module-info name] {/}] 0 0]
set release    [lrange [split [module-info name] {/}] 1 1]
set apphome    $APPHOME/gpaw.12224
set appname    "gpaw"

prepend-path PATH \$apphome/build/bin.$GPAW_PLATFORM
prepend-path PATH \$apphome/tools
prepend-path PYTHONPATH \$apphome
prepend-path PYTHONPATH \$apphome/build/lib.$GPAW_PLATFORM
EOF

cat <<EOF > modulefiles/ase/3.9.0.3906
set modulefile [lrange [split [module-info name] {/}] 0 0]
set release    [lrange [split [module-info name] {/}] 1 1]
set apphome    $APPHOME/ase.3906
set appname    "ase"

prepend-path PATH \$apphome/tools
prepend-path PYTHONPATH \$apphome
EOF

cat <<EOF > modulefiles/gpaw-setups/0.9.9672
set modulefile [lrange [split [module-info name] {/}] 0 0]
set release    [lrange [split [module-info name] {/}] 1 1]
set apphome    $APPHOME/gpaw-setups-0.9.9672
set appname    "gpaw-setups"

prepend-path GPAW_SETUP_PATH \$apphome
EOF

cat <<EOF > modulefiles/gpaw-setups/0.9.11271
set modulefile [lrange [split [module-info name] {/}] 0 0]
set release    [lrange [split [module-info name] {/}] 1 1]
set apphome    $APPHOME/gpaw-setups-0.9.11271
set appname    "gpaw-setups"

prepend-path GPAW_SETUP_PATH \$apphome
EOF

cat <<EOF > modulefiles/libxc/2.2.1-1
set modulefile [lrange [split [module-info name] {/}] 0 0]
set release    [lrange [split [module-info name] {/}] 1 1]
set apphome    $APPHOME/libxc-2.2.1-1
set appname    "libxc"

prepend-path    LD_LIBRARY_PATH    \$apphome/lib
prepend-path    PATH               \$apphome/bin
prepend-path    C_INCLUDE_PATH     \$apphome/include
prepend-path    PKG_CONFIG_PATH    \$apphome/lib/pkgconfig
EOF

Build libxc:

cd $APPHOME/libxc-2.2.1
./configure --prefix $APPHOME/libxc-2.2.1-1 --enable-shared
make
make install

and GPAW:

cd $APPHOME
wget https://svn.fysik.dtu.dk/projects/gpaw/trunk/doc/platforms/Linux/customize_gardar.py
cd $APPHOME/gpaw.12224
python setup.py --remove-default-flags --customize=../customize_gardar.py build_ext 2>&1 | tee build_ext.log

The customize_gardar.py looks like:

scalapack = True

mklpath ='/global/apps/intel/2013.1/mkl'
omppath ='/global/apps/openmpi/1.6.5/intel/13.1'
lxcpath ='/home/pcje/global/apps/libxc-2.2.1-1'

compiler = 'icc'

libraries = ['xc', 'mpi', 'mkl_scalapack_lp64', 'mkl_lapack95_lp64', 'mkl_intel_lp64', 'mkl_sequential', 'mkl_mc', 'mkl_core', 'mkl_def', 'mkl_intel_thread', 'iomp5']
library_dirs += [f'{omppath}/lib']
library_dirs += [f'{mklpath}/lib/intel64']
library_dirs += [f'{lxcpath}/lib']
include_dirs += [f'{omppath}/include']
include_dirs += [f'{mklpath}/include']
include_dirs += [f'{lxcpath}/include']

extra_link_args += [f'{mklpath}/lib/intel64/libmkl_blacs_openmpi_lp64.a', f'{mklpath}/lib/intel64/libmkl_blas95_lp64.a']

extra_compile_args += ['-O3', '-std=c99', '-fPIC', '-Wall']

define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]

mpicompiler = 'mpicc'
mpilinker = mpicompiler

GPAW tests gpaw-test can be submitted like this:

qsub -l nodes=1:ppn=8 run.sh

where run.sh looks like this:

#!/bin/sh

. /opt/lmod/lmod/init/sh
module use --append ~/global/apps/modulefiles
module load ase/3.9.0.3906
module load gpaw-setups/0.9.11271
module load libxc/2.2.1-1
module load gpaw/0.11.0.12224
export OMP_NUM_THREADS=1

mpiexec `which gpaw-python` `which gpaw-test`