Running OLAM at HCC

OLAM compilation on Tusker

pgi/11 compilation with mpi and openmp enabled
  1. Load modules:
        module load compiler/pgi/11 openmpi/1.6 szip/2.1 zlib/1.2 NCL/6.1dist
        
  2. Edit the include.mk file.
include.mk
#-----------------  LINUX Intel Fortran ifort/gcc ---------------
F_COMP=mpif90
# If the compiler supports (and the user wants to use)
# the module IEEE_ARITHMETIC, uncomment below
IEEE_ARITHMETIC=yes
# If using MPI libraries:
OLAM_MPI=yes
# If parallel hdf5 is supported, uncomment the next line
OLAM_PARALLEL_HDF5=yes
# If you use the ED2 model, uncomment the next line
#USE_ED2=yes
MPI_PATH=/util/opt/openmpi/1.6/pgi/11
PAR_INCS=-I$(MPI_PATH)/include:$(MPI_PATH)/lib
PAR_LIBS=-L$(MPI_PATH)/lib -lmpi
# OPTIMIZED:
F_OPTS=-O3 -traceback -mp
#F_OPTS=-xHost -O3 -fno-alias -ip -openmp -traceback
#F_OPTS=-g -O3 -xHost -traceback
# DEBUG:
#F_OPTS=-g -fp-model precise -check bounds -traceback \
#        -debug extended -check uninit -ftrapuv
# FORTRAN FLAGS FOR BIG FILES WHICH WOULD HAVE EXCESSIVE COMPILATION TIME
#SLOW_FFLAGS=-O1 -g -no-ip -traceback
C_COMP=mpicc
#C_COMP=mpicc
C_OPTS=-DUNDERSCORE -DLITTLE
NCARG_DIR=/util/src/ncl_ncarg/ncl_ncarg-6.1.2/lib
LIBNCARG=-L$(NCARG_DIR) -lncarg -lncarg_gks -lncarg_c \
          -L/usr/lib64 -lX11 -ldl -lpthread -lgfortran -lcairo
HDF5_LIBS=-L/util/opt/hdf5/1.8.13/openmpi/1.6/pgi/11/lib -lhdf5_fortran -lhdf5 -lz -lm
HDF5_INCS=-I/util/opt/hdf5/1.8.13/openmpi/1.6/pgi/11/include
NETCDF_LIBS=-L/util/opt/netcdf/4.2/pgi/11/lib -lnetcdf
NETCDF_INCS=-I/util/opt/netcdf/4.2/pgi/11/include

LOADER=$(F_COMP)
LOADER_OPTS=-mp
#LOADER_OPTS=-static-intel $(F_OPTS)
# For Apple OSX: the stack size needs to be increased at link time
# LOADER_OPTS=-static-intel $(F_OPTS) -Wl,-stack_size -Wl,0x10000000
# to allow ifort compiler to link with pg-compiled ncar graphics:
# LIBS=-z muldefs -L/opt/pgi/linux86-64/5.2/lib -lpgftnrtl -lpgc
## IMPORTANT:  Need to specify this flag in ED2
#USE_HDF5=1
  1. Command: make clean

  2. Command: make -j 8

intel/12 compilation with mpi and openmp enabled
  1. Load modules:
        module load compiler/intel/12 openmpi/1.6 szip/2.1 zlib/1.2
        
  2. Edit the include.mk file.

    include.mk
    #-----------------  LINUX Intel Fortran ifort/gcc ---------------
    F_COMP=mpif90
    # If the compiler supports (and the user wants to use)
    # the module IEEE_ARITHMETIC, uncomment below
    IEEE_ARITHMETIC=yes
    # If using MPI libraries:
    OLAM_MPI=yes
    # If parallel hdf5 is supported, uncomment the next line
    OLAM_PARALLEL_HDF5=yes
    # If you use the ED2 model, uncomment the next line
    #USE_ED2=yes
    MPI_PATH=/util/opt/openmpi/1.6/intel/12
    PAR_INCS=-I$(MPI_PATH)/include:$(MPI_PATH)/lib
    PAR_LIBS=-L$(MPI_PATH)/lib -lmpi
    # OPTIMIZED:
    F_OPTS=-O3 -traceback -openmp
    #F_OPTS=-xHost -O3 -fno-alias -ip -openmp -traceback
    #F_OPTS=-g -O3 -xHost -traceback
    # DEBUG:
    #F_OPTS=-g -fp-model precise -check bounds -traceback \
    #        -debug extended -check uninit -ftrapuv
    # FORTRAN FLAGS FOR BIG FILES WHICH WOULD HAVE EXCESSIVE COMPILATION TIME
    #SLOW_FFLAGS=-O1 -g -no-ip -traceback
    C_COMP=mpicc
    #C_COMP=mpicc
    C_OPTS=-DUNDERSCORE -DLITTLE
    NCARG_DIR=/util/src/ncl_ncarg/ncl_ncarg-6.1.2/lib
    LIBNCARG=-L$(NCARG_DIR) -lncarg -lncarg_gks -lncarg_c \
              -L/usr/lib64 -lX11 -ldl -lpthread -lgfortran -lcairo
    HDF5_LIBS=-L/util/opt/hdf5/1.8.13/openmpi/1.6/intel/12/lib -lhdf5_fortran -lhdf5 -lz -lm
    HDF5_INCS=-I/util/opt/hdf5/1.8.13/openmpi/1.6/intel/12/include
    NETCDF_LIBS=-L/util/opt/netcdf/4.2/intel/12/lib -lnetcdf
    NETCDF_INCS=-I/util/opt/netcdf/4.2/intel/12/include
    LOADER=$(F_COMP)
    LOADER_OPTS=-openmp
    #LOADER_OPTS=-static-intel $(F_OPTS)
    # For Apple OSX: the stack size needs to be increased at link time
    # LOADER_OPTS=-static-intel $(F_OPTS) -Wl,-stack_size -Wl,0x10000000
    # to allow ifort compiler to link with pg-compiled ncar graphics:
    # LIBS=-z muldefs -L/opt/pgi/linux86-64/5.2/lib -lpgftnrtl -lpgc
    ## IMPORTANT:  Need to specify this flag in ED2
    #USE_HDF5=1

  3. Command: make clean

  4. Command: make -j 8

 

OLAM compilation on Crane

Intel/15 compiler with OpenMPI/1.10
  1. Load modules:
        module load compiler/intel/15 openmpi/1.10 NCL/6.1 netcdf/4.4 phdf5/1.8 szip/2.1 zlib/1.2
        
  2. Edit the include.mk file:

    include.mk
    #-----------------  LINUX Intel Fortran ifort/gcc ---------------
    F_COMP=/util/opt/hdf5/1.8/openmpi/1.10/intel/15/bin/h5pfc
    
    # If the compiler supports (and the user wants to use)
    # the module IEEE_ARITHMETIC, uncomment below
    IEEE_ARITHMETIC=yes
    
    # If using MPI libraries:
    OLAM_MPI=yes
    
    # If parallel hdf5 is supported, uncomment the next line
    OLAM_PARALLEL_HDF5=yes
    
    # If you use the ED2 model, uncomment the next line
    #USE_ED2=yes
    
    #MPI_PATH=/usr/local/mpich
    PAR_INCS=-I/util/opt/openmpi/1.10/intel/15/include
    PAR_LIBS=-L/util/opt/openmpi/1.10/intel/15/lib
    
    # OPTIMIZED:
    F_OPTS=-xHost -O3 -fno-alias -ip -openmp -traceback
    #F_OPTS=-g -O3 -xHost -traceback
    
    # DEBUG:
    #F_OPTS=-g -fp-model precise -check bounds -traceback \
    #        -debug extended -check uninit -ftrapuv
    
    # EXTRA OPTIONS FOR FIXED-SOURCE CODE
    FIXED_SRC_FLAGS=-fixed -132
    
    # FORTRAN FLAGS FOR BIG FILES WHICH WOULD HAVE EXCESSIVE COMPILATION TIME
    SLOW_FFLAGS=-O1 -g -no-ip -traceback
    
    #C_COMP=icc
    C_COMP=mpicc
    C_OPTS=-O3 -DUNDERSCORE -DLITTLE
    
    NCARG_DIR=/util/opt/NCL/6.1/lib
    LIBNCARG=-L$(NCARG_DIR) -lncarg -lncarg_gks -lncarg_c \
    -L/usr/lib64 -lX11 -ldl -lpng -lpthread -lgfortran -lcairo
    
    HDF5_LIBS=-L/util/opt/hdf5/1.8/openmpi/1.10/intel/15/lib
    HDF5_INCS=-I/util/opt/hdf5/1.8/openmpi/1.10/intel/15/include
    
    NETCDF_LIBS=-L/util/opt/netcdf/4.4/intel/15/lib -lnetcdf
    NETCDF_INCS=-I/util/opt/netcdf/4.4/intel/15/include
    
    LOADER=$(F_COMP)
    LOADER_OPTS=-static-intel $(F_OPTS)
    
    # For Apple OSX: the stack size needs to be increased at link time
    # LOADER_OPTS=-static-intel $(F_OPTS) -Wl,-stack_size -Wl,0x10000000
    
    # to allow ifort compiler to link with pg-compiled ncar graphics:
    # LIBS=-z muldefs -L/opt/pgi/linux86-64/5.2/lib -lpgftnrtl -lpgc
    
    ## IMPORTANT:  Need to specify this flag in ED2
    USE_HDF5=1

  3. Command: make clean

  4. Command: make -j 8

 

Sample SLURM submit scripts

PGI compiler:
Sample submit script for PGI compiler
#!/bin/sh
#SBATCH --ntasks=8                                          # 8 cores
#SBATCH --mem-per-cpu=1024                                  # Minimum memory required per CPU (in megabytes)
#SBATCH --time=03:15:00                                     # Run time in hh:mm:ss
#SBATCH --error=/work/[groupname]/[username]/job.%J.err
#SBATCH --output=/work/[groupname]/[username]/job.%J.out
 
module load compiler/pgi/11 openmpi/1.6 szip/2.1 zlib/1.2
 
mpirun /path/to/olam-4.2c-mpi
Intel compiler:
Sample submit script for Intel compiler
#!/bin/sh
#SBATCH --ntasks=8                                          # 8 cores
#SBATCH --mem-per-cpu=1024                                  # Minimum memory required per CPU (in megabytes)
#SBATCH --time=03:15:00                                     # Run time in hh:mm:ss
#SBATCH --error=/work/[groupname]/[username]/job.%J.err
#SBATCH --output=/work/[groupname]/[username]/job.%J.out
 
module load compiler/intel/12 openmpi/1.6 szip/2.1 zlib/1.2
 
mpirun /path/to/olam-4.2c-mpi