icepack.setup

#!/bin/csh -f

set ICE_SANDBOX = `pwd`
set ICE_VERSION = unknown
if (-e columnphysics/version.txt) then
  set ICE_VERSION = `head -1 columnphysics/version.txt | sed -e 's/ /_/g'`
endif
set ICE_SCRIPTS = "${ICE_SANDBOX}/configuration/scripts"
set initargv = ( $argv[*] ) 

set helpheader = 0
set dash = "-"
set spval = "UnDeFiNeD"
set machcomp = ${spval}
set machine = ${spval}
set envnames = intel
set case = ${spval}
set test = ${spval}
set grid = col
set pesx = 1x1
set sets = ""
set tdir = ${spval}
set bdir = ${spval}
set testid = ${spval}
set testsuite = ${spval}
set queue = ${spval}
set acct = ${spval}
set baseCom = ${spval}  # Baseline compare
set baseGen = ${spval}  # Baseline generate
set bfbcomp = ${spval}  # BFB compare
set report  = 0       # test results reporting
set sdate = `date -u "+%y%m%d"`
set stime = `date -u "+%H%M%S"`
set docase = 0
set dotest = 0
set dosuite = 0
set coverage = 0       # code coverage measurement and reporting
set coverageflag = false

if ($#argv < 1) then
  set helpheader = 1
endif

set argv = ( $initargv[*] )
# check for -h
while (1)
  if ($#argv < 1) break;
  if ("$argv[1]" =~ "-h" || "$argv[1]" =~ "--help") then
    set helpheader = 2
    if ("$argv[1]" == "-h") then
      set helpheader = 1
    endif
  endif
  shift argv
end

#------------------------------------------------------------
# Help output

if ( $helpheader > 0) then
cat << EOF1

NAME   
  icepack.setup

SYNOPSIS
    -h || --help
    
    --version

    --setvers versno

    --docintfc

    --case CASE -m MACH 
        [-e ENV][-p MxN][-g GRID][-s SET1,SET2][--acct ACCT]

    --test TEST -m MACH --testid ID 
        [-e ENV][-p MxN][-g GRID][-s SET1,SET2][--acct ACCT]
        [--diff TESTNAME][--bdir DIR][--bgen DIR][--bcmp DIR]
        [--tdir PATH]

    --suite SUITE[,SUITE2] -m MACH --testid ID 
        [-e ENV1,ENV2][--acct ACCT][--bdir DIR][--bgen DIR]
        [--bcmp DIR][--tdir PATH][--report || --coverage]

DESCRIPTION
    --help, -h : help
    --version  : generates icepack version number
    --setvers  : updates the icepack version number in the sandbox
    --docintfc : update the public interface documentation
    --case, -c : case, case directory/name (not with --test or --suite)
    --mach, -m : machine, machine name (required)
    --env,  -e : compilation environment name(s), comma separated (default = "intel")
    --pes,  -p : tasks x threads or "m"x"n" (default is 1x1)
    --queue    : queue name for batch submission
    --acct     : account number for the batch submission
    --grid, -g : grid, grid (default = col)
    --set,  -s : case option setting(s), comma separated (default = " ")

    For testing

    --test     : test, test name (not with --case or --suite)
    --suite    : test suite, pre-defined set or sets of tests, comma separated (not with --case or --test)
    --tdir     : directory name where tests will be located
    --bdir     : baseline directory for regression testing, default ICE_MACHINE_BASELINE
    --bgen     : directory name where output from current tests are copied
    --bcmp     : directory name where output from current tests are compared
    --testid   : test ID, user-defined id for testing (REQUIRED with --test or --suite)
    --diff     : generate comparison against another case
    --report   : automatically post results when tests are complete
    --coverage : generate and report test coverage metrics when tests are complete,
                 requires GNU envname (--env gnu)

EXAMPLES
    icepack.setup -c caseB -m gordon -e cray -s diag1,debug
    icepack.setup --case ~/caseA --mach cheyenne --env intel --set diag24
    icepack.setup --test restart -m onyx -e gnu -s debug -testid myid
    icepack.setup --suite base_suite -m conrad --env intel,cray --testid myv1 --bgen myv1
    icepack.setup --suite base_suite,travis_suite,quick_suite -m cheyenne --testid myv1 --bgen myv1
    icepack.setup --suite quick_suite -m cheyenne --testid myv2 --bgen myv2 --bcmp myv1

SEE ALSO
    icepack.setup --help or
    User Documentation at https://cice-consortium.github.io/Icepack/

EOF1

if ($helpheader > 1) then
cat << EOF1

      Available --mach and --env combinations are in configuration/scripts/machines and include:
EOF1
      set soptions1 = `ls -1 configuration/scripts/machines | grep Macros | sed 's/Macros.//g' `
      set soptions = `echo $soptions1 | fmt -1 | sort `
      foreach sopt ($soptions)
        echo "             $sopt"
      end
cat << EOF1

      Available --set options are in configuration/scripts/options and include:
EOF1
      set soptions1 = `ls -1 configuration/scripts/options | grep set_ | sed 's/set_nml.//g' | sed 's/set_env.//g' | sed 's/set_files.//g' `
      set soptions = `echo $soptions1 | fmt -1 | sort -u `
      foreach sopt ($soptions)
        echo "             $sopt"
      end

cat << EOF1

      Available tests are in configuration/scripts/tests and include:
EOF1
      set soptions1 = `ls -1 configuration/scripts/tests | grep test_ | grep script | sed 's/test_//g' | sed 's/.script//g' `
      set soptions = `echo $soptions1 | fmt -1 | sort `
      foreach sopt ($soptions)
        echo "             $sopt"
      end

cat << EOF1

      Available sets of predefined suites are in configurations/scripts/tests and include:
EOF1
      set soptions1 = `ls -1 configuration/scripts/tests | grep "\.ts" | sed 's/\.ts//g' `
      set soptions = `echo $soptions1 | fmt -1 | sort `
      foreach sopt ($soptions)
        echo "             $sopt"
      end

endif
exit -1
endif

#------------------------------------------------------------
# Read in command line arguments

echo " "
echo "${0}:"

set argv = ( $initargv[*] )
# check for --version
while (1)
  if ($#argv < 1) break;
  if ("$argv[1]" =~ "--version" ) then
    echo "${0}: This is ${ICE_VERSION}"
    exit -1
  endif
  shift argv
end

set argv = ( $initargv[*] )
# check for --setvers
while (1)
  if ($#argv < 1) break;
  if ("$argv[1]" =~ "--setvers" ) then
    shift argv
    if ( $#argv < 1 ) then
      echo "${0}: ERROR in --setvers argument"
      exit -1
    endif
    set versno = $argv[1]
    ${ICE_SCRIPTS}/set_version_number.csh $versno
    if ($status != 0) then
       echo "${0} ERROR in set_version_number.csh"
       exit -1
    endif
    echo "Setting ICEPACK version to ${versno} and exiting"
    exit -1
  endif
  shift argv
end

set argv = ( $initargv[*] )
# check for --docintfc
while (1)
  if ($#argv < 1) break;
  if ("$argv[1]" =~ "--docintfc" ) then
    echo "${0}: Running docintfc"
    echo " "
    cd ${ICE_SANDBOX}/doc
    ./generate_interfaces.sh
    cd ${ICE_SANDBOX}
    echo " "
    echo "${0}: docintfc complete, exiting"
    echo "${0}: remember to add and commit any updated documentation files"
    echo " "
    exit -1
  endif
  shift argv
end

set argv = ( $initargv[*] )
# read in all options
while (1)
  if ( $#argv < 1 ) break;
  set option = $argv[1];

# arguments without settings
  if ("$option" == "--report") then
    set report = 1
    shift argv

  else if ("$option" == "--coverage") then
    set coverage = 1
    set coverageflag = true
    shift argv

# arguments with settings
  else
    shift argv
    if ( $#argv < 1 ) then
      echo "${0}: ERROR in $option, unsupported or missing an argument"
      exit -1
    endif
    if ("$argv[1]" =~ "$dash*" ) then
      echo "${0}: ERROR in $option, possibly missing an argument"
      exit -1
    endif

    if ("$option" == "--case" || "$option" == "-c") then
      set case = $argv[1]
      set docase = 1
    else if ("$option" =~ --mach* || "$option" == "-m") then
      set machine = $argv[1]
    else if ("$option" =~ --env* || "$option" == "-e") then
      set envnames = $argv[1]
    else if ("$option" == "--test") then
      set test = $argv[1]
      set dotest = 1
    else if ("$option" == "--tdir") then
      set tdir = $argv[1]
    else if ("$option" == "--grid" || "$option" == "-g") then
      set grid = $argv[1]``
    else if ("$option" == "--queue") then
      set queue = $argv[1]
    else if ("$option" == "--pes" || "$option" == "-p") then
      set pesx = $argv[1]
    else if ("$option" == "--acct") then
      set acct = $argv[1]
    else if ("$option" =~ --set*  || "$option" == "-s") then
      set sets = $argv[1]
    else if ("$option" == "--bdir") then
      set bdir = $argv[1]
    else if ("$option" == "--bcmp") then
      set baseCom = $argv[1]
    else if ("$option" == "--bgen") then
      set baseGen = $argv[1]
    else if ("$option" == "--diff") then
      set bfbcomp = $argv[1]
    else if ("$option" == "--suite") then
      set testsuite = $argv[1]
      set dosuite = 1
    else if ("$option" == "--testid") then
      set testid = $argv[1]
    else
      echo "${0}: ERROR unknown option $option, use -h for help"
      exit -1
    endif

  shift argv

  endif
end

if (${machine} == ${spval}) then
  echo "${0}: ERROR in arguments, --mach required"
  exit -1
endif

@ dosum = ${docase} + ${dotest} + ${dosuite}
if (${dosum} == 0) then
  echo "${0}: ERROR in arguments, --case, --test, or --suite required"
  exit -1
endif

if (${dosum} > 1) then
  echo "${0}: ERROR in arguments, cannot use more than one of --case, --test, and --suite"
  exit -1
endif

if ($coverage == 1 && $report == 1) then
  echo "${0}: ERROR in arguments, not recommmended to set both --coverage and --report"
  exit -1
endif

if ($coverage == 1 && "$envnames" != "gnu") then
  echo "${0}: ERROR in arguments, must use --env gnu with --coverage"
  exit -1
endif

if ($coverage == 1 && `where curl` == "" && `where wget` == "") then
  echo "${0}: ERROR 'curl' or 'wget' is required for --coverage"
  exit -1
endif

if (${dosuite} == 0) then
  if ($report == 1) then
    echo "${0}: ERROR in arguments, must use --suite with --report"
    exit -1
  endif
  if ($coverage == 1) then
    echo "${0}: ERROR in arguments, must use --suite with --coverage"
    exit -1
  endif
  if ("$envnames" =~ "*,*") then
    echo "${0}: ERROR in arguments, cannot set multiple envnames without --suite"
    exit -1
  else
    set envname = ${envnames}
    set machcomp = ${machine}_${envname}
  endif
else
  if ($coverage == 1) then
    if ("$envnames" =~ "*,*") then
      echo "${0}: ERROR in arguments, cannot set multiple envnames with --coverage"
      exit -1
    else
      set envname = ${envnames}
      set machcomp = ${machine}_${envname}
    endif
  endif
endif

if ((${dosuite} == 1 || ${dotest} == 1) && ${testid} == ${spval}) then
  echo "${0}: ERROR in arguments.  --testid must be passed if using --suite or --test"
  exit -1
endif

#---------------------------------------------------------------------
# Setup tsfile and test suite support stuff

set tsdir  = "."
set tsfile = "caselist.${sdate}-${stime}"
if ( ${dosuite} == 1 ) then
  set tsdir  = "testsuite.${testid}"
  set tsfile = "testsuite.${testid}.${sdate}-${stime}.list"
endif
if ( ${tdir} != ${spval} ) then
  set tsdir = ${tdir}
endif
if (-e ${tsfile}) then
  echo "${0}: ERROR in tsfile, this should never happen"
  exit -1
endif

if ( ${dosuite} == 0 ) then
  set teststring = "${test} ${grid} ${pesx} ${sets}"
  if ( $bfbcomp != ${spval} ) then
    if ( ${sets} == "" ) then
      set teststring = "${teststring} none ${bfbcomp}"
    else
      set teststring = "${teststring} ${bfbcomp}"
    endif
  endif
  echo ${teststring} >! ${tsfile}
  set sets = ""

else
  set tarrays = `echo ${testsuite} | sed 's/,/ /g' | fmt -1 | sort -u`
  set testsuitecnt = 0
  foreach tarray ( ${tarrays} )
    @ testsuitecnt = ${testsuitecnt} + 1
    if (-e ${tarray}) then
      cat ${tarray} >> $tsfile
    else if (-e ${tarray}.ts) then
      cat ${tarray}.ts >> $tsfile
    else
      if (-e ${ICE_SCRIPTS}/tests/${tarray}.ts) then
        cat ${ICE_SCRIPTS}/tests/${tarray}.ts >> $tsfile
      else
        echo "${0}: ERROR, cannot find testsuite file ${tarray}, also checked ${ICE_SCRIPTS}/tests/${tarray}.ts"
        exit -1
      endif
    endif
  end

  if (-e ${tsdir}) then
    echo "${0}: ERROR, ${tsdir} already exists"
    exit -1
  endif

  mkdir -p ${tsdir}
  if ($status != 0) then
    echo "${0}: ERROR, mkdir ${tsdir} aborted"
    exit -1
  endif
  cp -f ${ICE_SCRIPTS}/tests/report_results.csh ${tsdir}
  cp -f ${ICE_SCRIPTS}/tests/timeseries.csh ${tsdir}
  cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir}

cat >! ${tsdir}/suite.run << EOF0
#!/bin/csh -f

set nonomatch && rm -f icepexe.* && unset nonomatch

EOF0

  set remote = `git remote -v | grep -i origin | grep -i push | sed "s|.*\(https.*\)\s.*|\1|g"`
  set branch = `git status | grep -i "on branch" | sed 's|^.*ranch\s*\(\S*\)$|\1|g'`
  set hash     = `git log | grep -i commit | head -1 | cut -d " " -f 2-`
  set hashuser = `git log | grep -i author | head -1 | cut -d : -f 2-`
  set hashdate = `git log | grep -i date   | head -1 | cut -d : -f 2-`
  set cdate = `date -u "+%Y-%m-%d"`
  set ctime = `date -u "+%H:%M:%S"`
  set vers = ${ICE_VERSION}
  set shhash = `echo ${hash} | cut -c 1-10`

cat >! ${tsdir}/results.csh << EOF0
#!/bin/csh -f
rm -f results.log
echo "#------- " >> results.log
echo "#repo = ${remote}" >> results.log
echo "#bran = ${branch}" >> results.log
echo "#hash = ${hash}" >> results.log
echo "#hshs = ${shhash}" >> results.log
echo "#hshu = ${hashuser}" >> results.log
echo "#hshd = ${hashdate}" >> results.log
echo "#suit = ${testsuite}" >> results.log
echo "#date = ${cdate}" >> results.log
echo "#time = ${ctime}" >> results.log
echo "#mach = ${machine}" >> results.log
echo "#user = ${user}" >> results.log
echo "#vers = ${vers}" >> results.log
echo "#------- " >> results.log
EOF0

cat >! ${tsdir}/report_codecov.csh << EOF0
#!/bin/csh -f

source ${ICE_SCRIPTS}/machines/env.${machcomp}

set rn0 = "${sdate}-${stime}:${shhash}:${testsuitecnt}:${testsuite}"
set rn1 = \`echo \${rn0} | sed -e 's/ //g'\`
set report_name = \`echo \${rn1} | sed -e 's/_suite//g'\`

#for codecov
set use_curl = 1
# define CODECOV_TOKEN env variable
if !(\$?CODECOV_TOKEN) then
  if (-e ~/.codecov_icepack_token) then
    source ~/.codecov_icepack_token
  endif
  if !(\$?CODECOV_TOKEN) then
    echo "\${0}: ERROR, CODECOV_TOKEN env variable not defined"
    exit 2
  endif
endif

#for lcov
set lcovalist = ""

EOF0

  chmod +x ${tsdir}/suite.run
  chmod +x ${tsdir}/results.csh
  chmod +x ${tsdir}/report_codecov.csh
  cp -p -f ${tsdir}/report_codecov.csh ${tsdir}/report_lcov.csh

endif

#-------------------------------------------------------------------
# Loop over cases/tests

set nenvnames = "`echo $envnames | sed -e 's/,/ /g'`"

# check that machines and envnames are valid before starting big loop
set doabort = false
foreach envname ( $nenvnames )
  set machcomp = ${machine}_${envname}
  foreach file (env.${machcomp} Macros.${machcomp})
    if !(-e ${ICE_SCRIPTS}/machines/$file) then
      echo "${0}: ERROR, ${ICE_SCRIPTS}/machines/$file not found"
      set doabort = true
    endif
  end
end
if (${doabort} == true) then
  exit -1
endif

set sets_base = "${sets}"
set bfbcomp_base = "$bfbcomp"
foreach envname ( $nenvnames )
  set machcomp = ${machine}_${envname}
  foreach line ( "`cat $tsfile`" )
    # Check if line is a comment line
    if ( $line:q =~ '#'* || $line:q =~ '$'* || $line:q =~ '!'* ) then
      echo "skipping line: $line"
      continue
    endif

    # Obtain the test name, sets, grid, and PE information from .ts file
    set test = `echo $line | cut -d' ' -f1`
    set grid = `echo $line | cut -d' ' -f2`
    set pesx = `echo $line | cut -d' ' -f3`
    set sets_tmp = `echo $line | cut -d' ' -f4`
    set bfbcomp_tmp = `echo $line | cut -d' ' -f5`

    # Append sets from .ts file to the $sets variable
    set sets = "$sets_base,$sets_tmp"

    # Create a new bfbcomp_base variable to store bfbcomp passed to icepack.setup
    # Use bfbcomp_base or bfbcomp_tmp
    if ( $bfbcomp_tmp == "" ) then
      set bfbcomp = "$bfbcomp_base"
    else
      set bfbcomp = "$bfbcomp_tmp"
    endif

    set fbfbcomp = ${spval}
    if ($bfbcomp != ${spval}) then
      set fbfbcomp = ${machcomp}_${bfbcomp}.${testid}
    endif

    set testname_noid = ${spval}
    # create case for test cases
    if (${docase} == 0) then
      set soptions = ""
      # Create sorted array and remove duplicates and "none"
      set setsarray = `echo ${sets} | sed 's/,/ /g' | fmt -1 | sort -u`
      if ("${setsarray}" != "") then
        foreach field (${setsarray})
          if (${field} != "none") then
            set soptions = ${soptions}"_"${field}
          endif
        end
      endif
      # soptions starts with _
      set testname_noid = "${machcomp}_${test}_${grid}_${pesx}${soptions}"
      set testname_base = "${machcomp}_${test}_${grid}_${pesx}${soptions}.${testid}"
      if (${dosuite} == 1) then
        set testname = "${tsdir}/${testname_base}"
      else
        set testname = "${testname_base}"
      endif
      set case = ${testname}
    endif

    if (-d ${case}) then
      echo "${0}: ERROR, case ${case} already exists"
#      exit -1
      continue
    endif

    if (${case} =~ */*) then
      set casename = $case:t
    else
      set casename = $case
    endif

    # unset env variables that might not exist in env machine file
    # to avoid any carry over during multi envname suites
    unsetenv ICE_MACHINE_MAXTHREADS
    unsetenv ICE_MACHINE_MAXPES
    unsetenv ICE_MACHINE_QUIETMODE
    unsetenv ICE_MACHINE_CPPDEFS
    unsetenv ICE_MACHINE_QSTAT
    unsetenv ICE_MACHINE_MACHINFO
    unsetenv ICE_MACHINE_ENVINFO

    #------------------------------------------------------------
    # Setup case directory, copy files to case directory

    mkdir -p ${case}
    echo "`date`${0} $initargv[*]" >> ${case}/README.case

    cd ${case}
    set casedir = `pwd`
    set casescr = "${casedir}/casescripts"
    if !( -d ${casescr}) mkdir ${casescr}

    # from basic script dir to case
    foreach file (icepack.build icepack.settings Makefile icepack_in makdep.c setup_run_dirs.csh)
      if !(-e ${ICE_SCRIPTS}/$file) then
        echo "${0}: ERROR, ${ICE_SCRIPTS}/$file not found"
        exit -1
      endif
      cp -f -p ${ICE_SCRIPTS}/$file ${casedir}/
    end

    # from machines dir to case
    foreach file (env.${machcomp} Macros.${machcomp})
      if !(-e ${ICE_SCRIPTS}/machines/$file) then
        echo "${0}: ERROR, ${ICE_SCRIPTS}/machines/$file not found"
        exit -1
      endif
      cp -f -p ${ICE_SCRIPTS}/machines/$file ${casedir}/
    end

    # from basic script dir to casescr
    foreach file (parse_namelist.sh parse_settings.sh parse_namelist_from_settings.sh icepack_decomp.csh icepack.run.setup.csh icepack.test.setup.csh)
      if !(-e ${ICE_SCRIPTS}/$file) then
        echo "${0}: ERROR, ${ICE_SCRIPTS}/$file not found"
        exit -1
      endif
      cp -f -p ${ICE_SCRIPTS}/$file ${casescr}/
    end

    # from test dir to casescr
    foreach file (icepack.results.csh icepack.codecov.csh icepack.lcov.csh)
      if !(-e ${ICE_SCRIPTS}/tests/$file) then
        echo "${0}: ERROR, ${ICE_SCRIPTS}/$file not found"
        exit -1
      endif
      cp -f -p ${ICE_SCRIPTS}/tests/$file ${casescr}/
    end

    cd ${casedir}
    source ./env.${machcomp} -nomodules || exit 2

    set quietmode = false
    if ($?ICE_MACHINE_QUIETMODE) then
      set quietmode = ${ICE_MACHINE_QUIETMODE}
    endif

    if (${acct} == ${spval}) then
      if (-e ~/.cice_proj) then
        set acct = `head -1 ~/.cice_proj`
      else
        set acct = ${ICE_MACHINE_ACCT}
      endif
    endif

    if (${queue} == ${spval}) then
      if (-e ~/.cice_queue) then
        set queue = `head -1 ~/.cice_queue`
      else
        set queue = ${ICE_MACHINE_QUEUE}
      endif
    endif

    set rundir = ${ICE_MACHINE_WKDIR}/${casename}

    #------------------------------------------------------------
    # Compute a default blocksize

    if ($pesx != 1x1) then
      echo "${0}: ERROR in arguments, -p must be 1x1 if set manually"
      exit -1
    endif

    set chck = `echo ${pesx} | sed  's/^[0-9][0-9]*x[0-9][0-9]*$/OK/'`
    if (${chck} == OK) then
      set task = `echo ${pesx} | sed s/x.\*//`
      set thrd = `echo ${pesx} | sed s/.\*x//`
    else
      echo "${0}: ERROR in -p argument, ${pesx}, must be mxn"
      exit -1
    endif

    setenv ICE_DECOMP_GRID  ${grid}
    setenv ICE_DECOMP_NTASK ${task}
    setenv ICE_DECOMP_NTHRD ${thrd}

    source ${casescr}/icepack_decomp.csh
    if ($status != 0) then
      echo "${0}: ERROR, icepack_decomp.csh aborted"
      exit -1
    endif

    echo "ICE_SANDBOX  = ${ICE_SANDBOX}"
    echo "ICE_CASENAME = ${casename}"
    echo "ICE_CASEDIR  = ${casedir}"
    echo "ICE_MACHINE  = ${machine}"
    echo "ICE_ENVNAME  = ${envname}"
    echo "ICE_RUNDIR   = ${rundir}"

    #------------------------------------------------------------
    # Copy in and update icepack.settings and icepack_in files

    set fimods = ${casescr}/icepack_in.mods
    set fsmods = ${casescr}/icepack.settings.mods

    cp icepack_in ${casescr}/icepack_in.base
    cp icepack.settings ${casescr}/icepack.settings.base
    if (-e ${fimods}) rm ${fimods}
    if (-e ${fsmods}) rm ${fsmods}

cat >! ${fimods} << EOF1
# icepack.setup settings

nprocs = ${task}
data_dir = '${ICE_MACHINE_INPUTDATA}/Icepack_data/forcing'
EOF1

    # If this is a baseline-compare test, modify ICE_RUNDIR
    if ($bdir != ${spval}) then
      setenv basedir_tmp ${bdir}
    else
      setenv basedir_tmp ${ICE_MACHINE_BASELINE}
    endif
    if ("$baseGen" =~ "default") then
      set d1 = `echo ${cdate} | cut -c 3- | sed 's/-//g'`
      set t1 = `echo ${ctime} | sed 's/://g'`
      set baseGen = icepack.${shhash}.${d1}-${t1}
    endif
    if ("$baseCom" =~ "default") then
      set baseCom = `ls -t $basedir_tmp | head -1`
    endif


cat >! ${fsmods} << EOF1
# icepack.setup settings

setenv ICE_SANDBOX  ${ICE_SANDBOX}
setenv ICE_SCRIPTS  ${ICE_SCRIPTS}
setenv ICE_CASENAME ${casename}
setenv ICE_CASEDIR  ${casedir}
setenv ICE_MACHINE  ${machine}
setenv ICE_ENVNAME  ${envname}
setenv ICE_MACHCOMP ${machcomp}
setenv ICE_RUNDIR   ${rundir}
setenv ICE_GRID     ${grid}
setenv ICE_NXGLOB   ${ICE_DECOMP_NXGLOB}
setenv ICE_NTASKS   ${task}
setenv ICE_NTHRDS   ${thrd}
setenv ICE_BASELINE ${basedir_tmp}
setenv ICE_BASEGEN  ${baseGen}
setenv ICE_BASECOM  ${baseCom}
setenv ICE_SPVAL    ${spval}
setenv ICE_QUIETMODE  ${quietmode}
setenv ICE_TEST     ${test}
setenv ICE_TESTNAME ${testname_noid}
setenv ICE_BFBCOMP  ${fbfbcomp}
setenv ICE_ACCOUNT  ${acct}
setenv ICE_QUEUE    ${queue}
setenv ICE_COVERAGE ${coverageflag}
EOF1

    if (${sets} != "") then
      set setsx = `echo ${sets} | sed 's/,/ /g'`
      set setsxorig = "$setsx"
      set setsx = ""
      foreach name ($setsxorig)
        if (-e ${ICE_SCRIPTS}/options/set_files.${name}) then
          echo "adding options files from set_files.${name}"
          echo "`date`${0} adding options files from set_files.${name}" >> ${casedir}/README.case
          set setsnew = `cat ${ICE_SCRIPTS}/options/set_files.${name}`
          foreach nset ($setsnew)
            if ($nset !~ "#*") then
              set setsx = "$setsx $nset"
            endif
          end
        else
          set setsx = "$setsx $name"
        endif
      end

    else
      set setsx = ""
    endif

    if (${docase} == 0) then
      # from test options to casescr in case any test time changes are applied
      if (-e ${ICE_SCRIPTS}/tests/test_${test}.files) then
        cp -f -p ${ICE_SCRIPTS}/tests/test_${test}.files ${casescr}
        foreach file (`cat ${casescr}/test_${test}.files`)
          if (-e ${ICE_SCRIPTS}/options/$file) then
            cp -f -p ${ICE_SCRIPTS}/options/$file ${casescr}
          else
            echo "${0}: ERROR, could not find $file from test_${test}.files"
            exit -1
          endif
        end
      endif
    endif

    foreach name (${grid} ${test} ${setsx})
      set found = 0
      if (-e ${ICE_SCRIPTS}/options/set_nml.${name}) then

cat >> ${fimods} << EOF2

# set_nml.${name}

EOF2
        cat ${ICE_SCRIPTS}/options/set_nml.${name} >> ${fimods}

cat >> ${fimods} << EOF2

EOF2
        echo "adding namelist mods set_nml.${name}"
        echo "`date`${0} adding namelist modes set_nml.${name}" >> ${casedir}/README.case
        set found = 1
      endif
      if (-e ${ICE_SCRIPTS}/options/set_env.${name}) then

cat >> ${fsmods} << EOF2

# set_env.${name}

EOF2

        cat ${ICE_SCRIPTS}/options/set_env.${name} >> ${fsmods}

cat >> ${fsmods} << EOF2

EOF2
        echo "adding env mods set_env.${name}"
        echo "`date`${0} adding namelist modes set_env.${name}" >> ${casedir}/README.case
        set found = 1
      endif
      if (${found} == 0) then
        if (${name} == ${test}) then
          #echo "${ICE_SCRIPTS}/options/set_[nml,env].${name} not found"
          echo "`date`${0} ${ICE_SCRIPTS}/options/set_[nml,env].${name} not found" >> ${casedir}/README.case
        else
          echo "${0}: ERROR, ${ICE_SCRIPTS}/options/set_[nml,env].${name} not found"
          echo "`date`${0}: ERROR, ${ICE_SCRIPTS}/options/set_[nml,env].${name} not found" >> ${casedir}/README.case
          exit -1
        endif
      endif
    end

    ${casescr}/parse_settings.sh icepack.settings ${fsmods}
    ${casescr}/parse_namelist.sh icepack_in ${fimods}
    source ./icepack.settings
    source ./env.${machcomp} -nomodules || exit 2
    ${casescr}/parse_namelist_from_settings.sh icepack_in icepack.settings

    #------------------------------------------------------------
    # Generate run script

    source ./icepack.settings
    source ./env.${machcomp} -nomodules || exit 2

    ${casescr}/icepack.run.setup.csh
    if ($status != 0) then
      echo "${0}: ERROR, icepack.run.setup.csh aborted"
      exit -1
    endif

    #------------------------------------------------------------
    # Generate test script

    if (${docase} == 0) then
      # Print information to stdout
      echo "Creating scripts for $test test"

      # Generate test script
      ${casescr}/icepack.test.setup.csh 
      if ($status != 0) then
        echo "${0}: ERROR, icepack.test.setup.csh aborted"
        exit -1
      endif

    endif

    #------------------------------------------------------------
    # Generate testsuite stuff

    if ( ${dosuite} == 1 ) then
      cd ${ICE_SANDBOX}
      # Write build and run commands to suite.run

cat >> ${tsdir}/results.csh << EOF
cat ${testname_base}/test_output >> results.log
EOF

cat >> ${tsdir}/report_codecov.csh << EOF
mkdir ${testname_base}/codecov_output
cp ${rundir}/compile/*.{gcno,gcda} ${testname_base}/codecov_output/

EOF

cat >> ${tsdir}/report_lcov.csh << EOF
lcov --gcov-tool gcov -c -d ${rundir}/compile -o ${testname_base}/lcov.info
if (-s ${testname_base}/lcov.info) then
  set lcovalist = "\${lcovalist} -a ${testname_base}/lcov.info"
endif

EOF

cat >> ${tsdir}/suite.run << EOF

echo "-------test--------------"
echo "${testname_base}"
cd ${testname_base}
source ./icepack.settings
set md5chk = \`where md5sum\`
if ("\${md5chk}" != "") then
  set md5file = icepack.settings.md5
  grep -v " ICE_CASENAME " icepack.settings | grep -v " ICE_CASEDIR " | grep -v " ICE_RUNDIR " | grep -v " ICE_TESTNAME " | grep -v " ICE_TEST " >! \${md5file}
  set icepsetmd5 = \` md5sum \${md5file} | cut -d " " -f 1 \`
  rm \${md5file}
  set icepexe = "../icepexe.\${icepsetmd5}"
  ./icepack.build --exe \${icepexe}
  set bldstat = \${status}
  if !(-e \${icepexe}) cp -p \${ICE_RUNDIR}/icepack \${icepexe}
else
  ./icepack.build
  set bldstat = \${status}
endif
if (\$bldstat == 0) then
  ./icepack.submit | tee -a ../suite.jobs
endif
cd ..
EOF
    
      # Reset case for the next test in suite
      set case = ${spval}

      echo "" 
      echo "---" 
      echo "" 
    endif

  # This is the foreach end for the testsuite
  end
# This is the foreach end for the envnames
end

#-----------------------------------------------------
# more testsuite stuff

if ( ${dosuite} == 1 ) then

  # Delete reused icepexe files at the end to save space
  cat >> ${tsdir}/suite.run << EOF0

set nonomatch && rm -f icepexe.* && unset nonomatch

EOF0

   # Add code to post processing scripts

  if ($?ICE_MACHINE_QSTAT) then
    cat >! ${tsdir}/poll_queue.env << EOF0
setenv ICE_MACHINE_QSTAT ${ICE_MACHINE_QSTAT}
EOF0
  endif

  cat ${casescr}/icepack.results.csh >> ${tsdir}/results.csh

  cat ${casescr}/icepack.codecov.csh >> ${tsdir}/report_codecov.csh

  cat ${casescr}/icepack.lcov.csh >> ${tsdir}/report_lcov.csh

  # build and submit tests
  cd ${tsdir}
  ./suite.run | tee suite.log
  if ($report == 1) then
    echo "Reporting results"
    ./poll_queue.csh
    ./results.csh
    ./report_results.csh
  endif
  if ($coverage == 1) then
    echo "Generating coverage reports"
    ./poll_queue.csh
    ./report_lcov.csh
#    ./report_codecov.csh
  endif
  cd ${ICE_SANDBOX}

endif

echo " "
echo "${0} done"
echo " "
exit 0