ranger.tacc.utexas.edu

Here you find information about the the system http://services.tacc.utexas.edu/index.php/ranger-user-guide.

The installation of user’s packages on ranger EL4, 64-bit described below uses modules, and assumes csh shell:

  • packages are installed under ~/CAMd:

    mkdir ~/CAMd
    cd ~/CAMd
    
  • module files are located under ~/CAMd/modulefiles

  • download the customize_ranger_EL4.py file:

    wget https://svn.fysik.dtu.dk/projects/gpaw/trunk/doc/platforms/Linux/customize_ranger_EL4.py
    
    scalapack = True
    
    extra_compile_args = ['-O3', '-std=c99', '-fpic']
    
    compiler = 'gcc'
    mpicompiler = '/opt/apps/gcc4_4/openmpi/1.3b/bin/mpicc'
    mpilinker = mpicompiler
    
    mkl_lib_path = '/opt/apps/intel/mkl/10.0.1.014/lib/em64t/'
    ompi_lib_path = '/opt/apps/gcc4_4/openmpi/1.3b/lib'
    
    libraries = []
    
    # use static linking to avoid
    # "cannot allocate memory for thread-local data: ABORT"
    extra_link_args = [
    mkl_lib_path+'libmkl_intel_lp64.a',
    mkl_lib_path+'libmkl_sequential.a',
    mkl_lib_path+'libmkl_core.a',
    mkl_lib_path+'libmkl_blacs_openmpi_lp64.a',
    mkl_lib_path+'libmkl_scalapack_lp64.a',
    mkl_lib_path+'libmkl_blacs_openmpi_lp64.a',
    mkl_lib_path+'libmkl_intel_lp64.a',
    mkl_lib_path+'libmkl_sequential.a',
    mkl_lib_path+'libmkl_core.a',
    mkl_lib_path+'libmkl_intel_lp64.a',
    mkl_lib_path+'libmkl_sequential.a',
    mkl_lib_path+'libmkl_core.a',
    ]
    
    extra_link_args += ['-Wl,-rpath='+ompi_lib_path]
    
    define_macros += [('GPAW_NO_UNDERSCORE_CBLACS', '1')]
    define_macros += [('GPAW_NO_UNDERSCORE_CSCALAPACK', '1')]
    
  • download packages with download_ranger.sh, buy running sh download_ranger.sh:

    setenv APPS "/share/home/01067/tg803307/CAMd"
    setenv MODULEFILES "${APPS}/modulefiles"
    
    # download packages
    set nose_version=0.11.3
    wget http://python-nose.googlecode.com/files/nose-${nose_version}.tar.gz
    set numpy_version=1.5.0
    wget http://downloads.sourceforge.net/numpy/numpy-${numpy_version}.tar.gz
    set ase_version=3.4.1.1765
    wget https://wiki.fysik.dtu.dk/ase-files/python-ase-${ase_version}.tar.gz
    set gpaw_version=0.7.2.6974
    wget https://wiki.fysik.dtu.dk/gpaw-files/gpaw-${gpaw_version}.tar.gz
    set gpaw_setups_version=0.6.6300
    wget http://wiki.fysik.dtu.dk/gpaw-files/gpaw-setups-${gpaw_setups_version}.tar.gz
    
  • start with the following modules loaded:

    login3% module list
    Currently Loaded Modules:
      1) TACC-paths           9) srb-client/3.4.1  17) globus/4.0.8
      2) Linux                10) tg-policy/0.2    18) GLOBUS-4.0
      3) cluster-paths        11) tgproxy/0.9.1    19) TERAGRID-DEV
      4) pgi/7.2-5            12) tgresid/2.3.4    20) CTSSV4
      5) mvapich/1.0.1        13) tgusage/3.0      21) gzip/1.3.12
      6) binutils-amd/070220  14) uberftp/2.4      22) tar/1.22
      7) TERAGRID-paths       15) tginfo/1.0.1     23) cluster
      8) gx-map/0.5.3.3       16) TERAGRID-BASIC   24) TACC
    
  • unload/load the modules:

    module switch pgi/7.2-5 gcc/4.4.5
    module swap mvapich openmpi/1.3b
    module load python/2.5.2
    module load mkl/10.0
    
  • install packages, deploy modules and test with install_ranger_EL4.sh, buy running sh install_ranger_EL4.sh:

    setenv APPS "/share/home/01067/tg803307/CAMd"
    setenv MODULEFILES "${APPS}/modulefiles"
    
    # build packages
    
    set nose_version=0.11.3
    tar zxf nose-${nose_version}.tar.gz
    cd nose-${nose_version}
    python setup.py install --root=${APPS}/nose-${nose_version}-1
    cd ..
    
    set numpy_version=1.5.0
    tar zxf numpy-${numpy_version}.tar.gz
    cd numpy-${numpy_version}
    python setup.py install --root=${APPS}/numpy-${numpy_version}-1
    cd ..
    
    set ase_version=3.4.1.1765
    tar zxf python-ase-${ase_version}.tar.gz
    
    set gpaw_version=0.7.2.6974
    tar zxf gpaw-${gpaw_version}.tar.gz
    
    set gpaw_setups_version=0.6.6300
    tar zxf gpaw-setups-${gpaw_setups_version}.tar.gz
    
    mkdir -p ${MODULEFILES}/nose
    cat <<EOF > ${MODULEFILES}/nose/${nose_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prereq python/2.5.2
    prepend-path    PATH                \$apps_path/nose-${nose_version}-1/opt/apps/python/python-2.5.2/bin/
    prepend-path    PYTHONPATH          \$apps_path/nose-${nose_version}-1/opt/apps/python/python-2.5.2/lib/python2.5/site-packages/
    unset apps_path
    EOF
    
    mkdir -p ${MODULEFILES}/numpy
    cat <<EOF > ${MODULEFILES}/numpy/${numpy_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prereq python/2.5.2
    prereq nose/0.11.3-1
    prepend-path    PATH                \$apps_path/numpy-${numpy_version}-1/opt/apps/python/python-2.5.2/bin
    prepend-path    PYTHONPATH          \$apps_path/numpy-${numpy_version}-1/opt/apps/python/python-2.5.2/lib/python2.5/site-packages/
    unset apps_path
    EOF
    
    mkdir -p ${MODULEFILES}/campos-ase3
    cat <<EOF > ${MODULEFILES}/campos-ase3/${ase_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prereq python/2.5.2
    prereq nose/0.11.3-1
    prereq numpy/1.5.0-1
    prepend-path    PATH                \$apps_path/python-ase-${ase_version}/tools
    prepend-path    PYTHONPATH          \$apps_path/python-ase-${ase_version}/
    unset apps_path
    EOF
    
    mkdir -p ${MODULEFILES}/campos-gpaw-setups
    cat <<EOF > ${MODULEFILES}/campos-gpaw-setups/${gpaw_setups_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prepend-path    GPAW_SETUP_PATH     \$apps_path/gpaw-setups-${gpaw_setups_version}
    unset apps_path
    EOF
    
    mkdir -p ${MODULEFILES}/campos-gpaw
    cat <<EOF > ${MODULEFILES}/campos-gpaw/${gpaw_version}-1
    #%Module1.0
    set apps_path ${APPS}
    prereq python/2.5.2
    prereq nose/0.11.3-1
    prereq numpy/1.5.0-1
    prereq campos-ase3
    prereq campos-gpaw-setups
    prereq gcc/4.4.5
    prereq openmpi/1.3b
    prereq mkl/10.0
    prepend-path    PATH                \$apps_path/gpaw-${gpaw_version}/tools
    prepend-path    PATH                \$apps_path/gpaw-${gpaw_version}/build/bin.linux-x86_64-2.5/
    prepend-path    PYTHONPATH          \$apps_path/gpaw-${gpaw_version}/
    prepend-path    PYTHONPATH          \$apps_path/gpaw-${gpaw_version}/build/lib.linux-x86_64-2.5/
    setenv OMP_NUM_THREADS 1
    unset apps_path
    EOF
    
    module avail
    
    module use --append ${MODULEFILES}
    module load python/2.5.2
    module load nose/0.11.3-1
    module load numpy/1.5.0-1
    module load campos-ase3
    # test numpy
    python -c "import numpy; numpy.test()"
    # test ase
    mkdir -p testase
    cd testase
    testase.py --no-display >& testase.log
    cd ..
    # build gpaw
    cd gpaw-${gpaw_version}
    python setup.py build_ext --customize=../customize_ranger_EL4.py --remove-default-flags >& build_ext.log
    cd ..
    module load campos-gpaw-setups
    module load gcc/4.4.5
    module load openmpi/1.3b
    module load mkl/10.0
    module load campos-gpaw
    mkdir -p testgpaw
    cd testgpaw
    mpiexec -np 4 gpaw-python `which gpaw-test` >& tee testgpaw.log
    

    Note that every time you wish to install a new version of a package, and deploy new module file, better keep the old module file.

  • submit the test job:

    qsub submit.sh
    

    using the following submit.sh:

    #!/bin/bash
    
    #$ -V   # Inherit the submission environment
    #$ -cwd         # Start job in submission directory
    ##$ -N myMPI    # Job Name
    ##$ -j y        # Combine stderr and stdout
    ##$ -o $JOB_NAME.o$JOB_ID       # Name of the output file (eg. myMPI.oJobID)
    #$ -pe 16way 32         # Requests 16 tasks/node, 32 cores total
    #$ -q development       # OR Queue name "normal"
    #$ -l h_rt=00:40:00     # Run time (hh:mm:ss) - 40 mins
    ##$ -M  # Use email notification address
    ##$ -m be       # Email at Begin and End of job
    #set -x         # Echo commands, use "set echo" with csh
    
    module use --append /share/home/01067/tg803307/CAMd/modulefiles
    module load python/2.5.2
    module load nose/0.11.3-1
    module load numpy/1.5.0-1
    module load campos-ase3
    module load campos-gpaw-setups
    module unload pgi
    module load gcc/4.4.5
    module unload mvapich
    module load openmpi/1.3b
    module load mkl/10.0
    module load campos-gpaw
    
    # wget http://svn.fysik.dtu.dk/projects/gpaw/trunk/doc/devel/256H2O/b256H2O.py
    
    ibrun `which gpaw-python` b256H2O.py
    
  • to enable the installation permanently add the following to ~/.bashrc:

    module use --append /share/home/01067/tg803307/CAMd/modulefiles
    module load python/2.5.2
    module load nose/0.11.3-1
    module load numpy/1.5.0-1
    module load campos-ase3
    module load campos-gpaw-setups
    module unload pgi
    module load gcc/4.4.5
    module unload mvapich
    module load openmpi/1.3b
    module load mkl/10.0
    module load campos-gpaw