Merge pull request #226 from ESCOMP/andre-standalone-dev

Support a standalone checkout from git

This PR allows a standalone git-based checkout to work. Starting with
this tag, we'll be using git rather than svn for CLM/CTSM development.

The biggest changes here are pulling in manage_externals, which serves
the role of svn externals. After cloning the repository, you can get all
of the necessary externals by running:

./manage_externals/checkout_externals

See the file README_EXTERNALS.rst at the top level of the repository for
more details on using this tool.

Other than that, this PR mostly involves changes to allow you to run
from the new directory structure of a standalone checkout: Now all of
the CLM/CTSM directories appear at the top level of the checkout, rather
than nested under components/clm.

.CLMTrunkChecklist

@@ -42,6 +42,8 @@ This should show no diffs
 
 (8) Make the trunk tag
 
+(9) Send an email to clm-dev with the contents of the latest ChangeLog
+entry (until we have automated this for the git repo)
 
 NOTES:
 

.config_files.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+
+<?xml-stylesheet type="text/xsl" ?>
+
+<entry_id>
+
+  <!-- This is the same as the default entry in
+       cime/config/cesm/config_files.xml except for the value for clm:
+       In a standalone clm checkout, COMP_ROOT_DIR_LND is $SRCROOT
+       rather than $SRCROOT/components/clm.
+
+       However, because of the way overrides are handled, we need to
+       re-specify the full information here rather than just overriding
+       the value for clm.
+  -->
+  <entry id="COMP_ROOT_DIR_LND">
+    <type>char</type>
+    <default_value>unset</default_value>
+    <values>
+      <value component="clm"      >$SRCROOT</value>
+      <value component="dlnd"      >$CIMEROOT/src/components/data_comps/dlnd</value>
+      <value component="slnd"      >$CIMEROOT/src/components/stub_comps/slnd</value>
+      <value component="xlnd"      >$CIMEROOT/src/components/xcpl_comps/xlnd</value>
+    </values>
+    <group>case_comps</group>
+    <file>env_case.xml</file>
+    <desc>Root directory of the case land model component  </desc>
+    <schema>$CIMEROOT/config/xml_schemas/config_compsets.xsd</schema>
+  </entry>
+
+</entry_id>

.gitignore

@@ -1,3 +1,11 @@
+# directories checked out by manage_externals, and other files created
+# by manage_externals
+manage_externals.log
+src/fates/
+tools/PTCLM/
+cime/
+components/
+
 # ignore svn directories
 **/.svn/**
 .svn/
@@ -25,7 +33,4 @@ CMakeFiles/
 core.*
 *.gz
 *.log !run.log
-
-
-
-
+*.pyc

Externals.cfg

@@ -0,0 +1,38 @@
+[clm]
+local_path = .
+protocol = externals_only
+externals = Externals_CLM.cfg
+required = True
+
+[cism]
+local_path = components/cism
+protocol = git
+repo_url = https://github.com/ESCOMP/cism-wrapper
+tag = cism2_1_46
+externals = Externals_CISM.cfg
+required = True
+
+[rtm]
+local_path = components/rtm
+protocol = git
+repo_url = https://github.com/ESCOMP/rtm
+tag = rtm1_0_65
+required = True
+
+[mosart]
+local_path = components/mosart
+protocol = git
+repo_url = https://github.com/ESCOMP/mosart
+tag = mosart1_0_30
+required = True
+
+[cime]
+local_path = cime
+protocol = git
+repo_url = https://github.com/CESM-Development/cime
+tag = clm4518/n03/cime5.4.0-alpha.03
+required = True
+
+[externals_description]
+schema_version = 1.0.0
+

Externals_CLM.cfg

@@ -0,0 +1,17 @@
+[fates]
+local_path = src/fates
+protocol = git
+repo_url = https://github.com/NCAR/fates-release
+tag = fates_s1.4.1_a3.0.0_rev2
+required = True
+
+[PTCLM]
+local_path = tools/PTCLM
+protocol = git
+repo_url = https://github.com/ESCOMP/ptclm
+tag = PTCLM2_171216c
+required = True
+
+[externals_description]
+schema_version = 1.0.0
+

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2005-2017, University Corporation for Atmospheric Research (UCAR)
+Copyright (c) 2005-2018, University Corporation for Atmospheric Research (UCAR)
 All rights reserved.
 
 Developed by: 

README.rst

@@ -1,10 +1,12 @@
-===
-CLM
-===
+====
+CTSM
+====
 
-The Community Land Model, CLM, is part of the Community Earth System Model.
+The Community Terrestrial Systems Model.
 
-See the CESM web site for documentation and information:
+This includes the Community Land Model (CLM5 and CLM4.5) of the Community Earth System Model.
 
-http://www.cesm.ucar.edu
+For documentation, quick start, diagnostics, model output and
+references, see
 
+http://www.cesm.ucar.edu/models/cesm2.0/land/

README_EXTERNALS.rst

@@ -0,0 +1,124 @@
+Obtaining the full model code and associated scripting infrastructure
+=====================================================================
+
+CTSM is released via github. You will need some familiarity with git in order
+to modify the code and commit these changes. However, to simply checkout and run the
+code, no git knowledge is required other than what is documented in the following steps.
+
+To obtain the CTSM code you need to do the following:
+
+#. Clone the repository. ::
+
+      git clone https://github.com/escomp/ctsm.git my_ctsm_sandbox
+
+   This will create a directory ``my_ctsm_sandbox/`` in your current working directory.
+
+#. Run the script **manage_externals/checkout_externals**. ::
+
+      ./manage_externals/checkout_externals
+
+   The **checkout_externals** script is a package manager that will
+   populate the ctsm directory with the relevant versions of each of the
+   components along with the CIME infrastructure code.
+
+At this point you have a working version of CTSM.
+
+To see full details of how to set up a case, compile and run, see the CIME documentation at http://esmci.github.io/cime/ .
+
+More details on checkout_externals
+----------------------------------
+
+The file **Externals.cfg** in your top-level CTSM directory tells
+**checkout_externals** which tag/branch of each component should be
+brought in to generate your sandbox. (This file serves the same purpose
+as SVN_EXTERNAL_DIRECTORIES when CLM was in a subversion repository.)
+
+NOTE: Just like svn externals, checkout_externals will always attempt
+to make the working copy exactly match the externals description. If
+you manually modify an external without updating Externals.cfg, e.g. switch
+to a different tag, then rerunning checkout_externals will switch you
+back to the external described in Externals.cfg. See below
+documentation `Customizing your CTSM sandbox`_ for more details.
+
+**You need to rerun checkout_externals whenever Externals.cfg has
+changed** (unless you have already manually updated the relevant
+external(s) to have the correct branch/tag checked out). Common times
+when this is needed are:
+
+* After checking out a new CTSM branch/tag
+
+* After merging some other CTSM branch/tag into your currently
+  checked-out branch
+
+**checkout_externals** must be run from the root of the source
+tree. For example, if you cloned CTSM with::
+
+  git clone https://github.com/escomp/ctsm.git my_ctsm_sandbox
+
+then you must run **checkout_externals** from
+``/path/to/my_ctsm_sandbox``.
+
+To see more details of **checkout_externals**, issue ::
+
+  ./manage_externals/checkout_externals --help
+
+Customizing your CTSM sandbox
+=============================
+
+There are several use cases to consider when you want to customize or modify your CTSM sandbox.
+
+Switching to a different CTSM branch or tag
+-------------------------------------------
+
+If you have already checked out a branch or tag and **HAVE NOT MADE ANY
+MODIFICATIONS** it is simple to change your sandbox. Say that you
+checked out ctsm1.0.0 but really wanted to have ctsm1.1.0;
+you would simply do the following::
+
+  git checkout ctsm1.1.0
+  ./manage_externals/checkout_externals
+
+You should **not** use this method if you have made any source code
+changes, or if you have any ongoing CTSM cases that were created from
+this sandbox. In these cases, it is often easiest to do a second **git
+clone**.
+
+Pointing to a different version of a component
+----------------------------------------------
+
+Each entry in **Externals.cfg** has the following form (we use CIME as an
+example below)::
+
+  [cime]
+  local_path = cime
+  protocol = git
+  repo_url = https://github.com/CESM-Development/cime
+  tag = cime5.4.0-alpha.20
+  required = True
+
+Each entry specifies either a tag or a branch. To point to a new tag:
+
+#. Modify the relevant entry/entries in **Externals.cfg** (e.g., changing
+   ``cime5.4.0-alpha.20`` to ``cime5.4.0-alpha.21`` above)
+
+#. Checkout the new component(s)::
+
+     ./manage_externals/checkout_externals
+
+Keep in mind that changing individual components from a tag may result
+in an invalid model (won't compile, won't run, not scientifically
+meaningful) and is unsupported.
+
+Committing your change to Externals.cfg
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+After making this change, it's a good idea to commit the change in your
+local CTSM git repository. First create a branch in your local
+repository, then commit it. (Unlike with subversion, branches are stored
+locally unless you explicitly push them up to github. Feel free to
+create whatever local branches you'd like.) For example::
+
+  git checkout -b my_ctsm_branch
+  git add Externals.cfg
+  git commit -m "Update CIME to cime5.4.0-alpha.20"
+

bld/CLMBuildNamelist.pm

@@ -59,6 +59,7 @@ SYNOPSIS
 
      Create the namelist for CLM
 REQUIRED OPTIONS
+     -cimeroot "directory"    Path to cime directory
      -config "filepath"       Read the given CLM configuration cache file.
                               Default: "config_cache.xml".
      -d "directory"           Directory where output namelist file will be written
@@ -245,7 +246,8 @@ sub process_commandline {
   # the array!
   $nl_flags->{'cmdline'} = "@ARGV";
 
-  my %opts = ( config                => "config_cache.xml",
+  my %opts = ( cimeroot              => undef,
+               config                => "config_cache.xml",
                csmdata               => undef,
                clm_usr_name          => undef,
                co2_type              => undef,
@@ -282,6 +284,7 @@ sub process_commandline {
              );
 
   GetOptions(
+             "cimeroot=s"                => \$opts{'cimeroot'},
              "clm_demand=s"              => \$opts{'clm_demand'},
              "co2_ppmv=f"                => \$opts{'co2_ppmv'},
              "co2_type=s"                => \$opts{'co2_type'},
@@ -345,9 +348,9 @@ sub check_for_perl_utils {
   my $cfgdir = shift;
   my $opts_ref = shift;
 
-  # Determine CESM root directory and perl5lib root directory
-  my $cesmroot = abs_path( "$cfgdir/../../../");
-  my $perl5lib_dir = "$cesmroot/cime/utils/perl5lib";
+  # Determine CIME root directory and perl5lib root directory
+  my $cimeroot = $opts_ref->{'cimeroot'};
+  my $perl5lib_dir = "$cimeroot/utils/perl5lib";
 
   #-----------------------------------------------------------------------------
   # Add $perl5lib_dir to the list of paths that Perl searches for modules
@@ -4397,7 +4400,6 @@ sub main {
   my $cfg = read_configure_definition($cfgdir, \%opts);
 
   my $physv      = config_files::clm_phys_vers->new( $cfg->get('phys') );
-  my $cesmroot   = abs_path( "$cfgdir/../../../");
   my $definition = read_namelist_definition($cfgdir, \%opts, \%nl_flags, $physv);
   my $defaults   = read_namelist_defaults($cfgdir, \%opts, \%nl_flags, $cfg, $physv);