build.exec

#!/bin/bash
# 
# Copyright (c) 2017, SingularityWare, LLC. All rights reserved.
#
# Copyright (c) 2015-2017, Gregory M. Kurtzer. All rights reserved.
# 
# Copyright (c) 2016-2017, The Regents of the University of California,
# through Lawrence Berkeley National Laboratory (subject to receipt of any
# required approvals from the U.S. Dept. of Energy).  All rights reserved.
# 
# This software is licensed under a customized 3-clause BSD license.  Please
# consult LICENSE file distributed with the sources of this project regarding
# your rights to use or distribute this software.
# 
# NOTICE.  This Software was developed under funding from the U.S. Department of
# Energy and the U.S. Government consequently retains certain rights. As such,
# the U.S. Government has been granted for itself and others acting on its
# behalf a paid-up, nonexclusive, irrevocable, worldwide license in the Software
# to reproduce, distribute copies to the public, prepare derivative works, and
# perform publicly and display publicly, and to permit other to do so. 
# 
# 


## Basic sanity
if [ -z "$SINGULARITY_libexecdir" ]; then
    echo "Could not identify the Singularity libexecdir."
    exit 1
fi

## Load functions
if [ -f "$SINGULARITY_libexecdir/singularity/functions" ]; then
    . "$SINGULARITY_libexecdir/singularity/functions"
else
    echo "Error loading functions: $SINGULARITY_libexecdir/singularity/functions"
    exit 1
fi

if ! which mksquashfs > /dev/null 2>&1; then
    message ERROR "You must install squashfs-tools to build images\n"
    ABORT 255
fi

build_cleanup() {
    message 1 "Cleaning up...\n";

    if [ -f "${SINGULARITY_CONTENTS:-}" ]; then
        rm -f "${SINGULARITY_CONTENTS:-}"
    fi

    if [ -n "${SINGULARITY_CLEANUP:-}" ]; then
        rm -f "${SINGULARITY_BUILDDEF:-}"
    fi

    if [ -z "${SINGULARITY_NOCLEANUP:-}" -a -z "${SINGULARITY_SANDBOX:-}" ]; then
        if [ -d "${SINGULARITY_ROOTFS:-}" ]; then
            rm -rf "${SINGULARITY_ROOTFS:-}"
        fi
    fi
}

atexit build_cleanup

USERID=`id -ru`

SINGULARITY_CHECKS="no"
SINGULARITY_CHECKTAGS=bootstrap
SINGULARITY_CHECKLEVEL=3
while true; do
    case ${1:-} in
        -h|--help|help)
            exec "$SINGULARITY_libexecdir/singularity/cli/help.exec" "$SINGULARITY_COMMAND"
        ;;
        -s|--sandbox)
            SINGULARITY_SANDBOX=1
            shift
        ;;
        -w|--writable)
            if [ "$USERID" != "0" ]; then
                message ERROR "Writable images must be created as root\n"
                message ERROR "Use --sandbox (does not required sudo) for a writable folder.\n"
                exit 1
            fi
            SINGULARITY_WRITABLE=1
            shift
        ;;
        -f|-F|--force)
            SINGULARITY_FORCE=1
            shift
        ;;
        -T|--notest)
            shift
            SINGULARITY_NOTEST=1
            export SINGULARITY_NOTEST
        ;;
        -s|--section)
            shift
            SINGULARITY_BUILDSECTION="$1"
            export SINGULARITY_BUILDSECTION
            shift
        ;;
        -c|--check|--checks)
            SINGULARITY_CHECKS="yes"
            shift;
        ;;
        -l|--low)                          # levels 3,2,1
            SINGULARITY_CHECKLEVEL=3       # LOW
            shift;
        ;;
        -m|--med|--medium)                 # levels 2,1
            SINGULARITY_CHECKLEVEL=2       # MED
            shift;
        ;;
        -h|--high)
            SINGULARITY_CHECKLEVEL=1      # level 1 only
            shift;                        # HIGH
        ;;
        -t|--tag)
            shift;
            SINGULARITY_CHECKTAGS="${1:-}"
            shift;
        ;;
        -*)
            message ERROR "Unknown option: ${1:-}\n"
            exit 1
        ;;
        *)
            break
        ;;
    esac
done


################################################################################
# Source Usage and Help
################################################################################

if [ -f "$SINGULARITY_libexecdir/singularity/cli/$SINGULARITY_COMMAND.info" ]; then
    . "$SINGULARITY_libexecdir/singularity/cli/$SINGULARITY_COMMAND.info"
else
    message ERROR "Could not find the info file for: $SINGULARITY_COMMAND\n"
    ABORT 255
fi

if [ -z "${2:-}" ]; then
    if [ -n "${USAGE:-}" ]; then
        echo "USAGE: $USAGE"
    else
        echo "To see usage summary type: singularity help $SINGULARITY_COMMAND"
    fi
    exit 0
fi


SINGULARITY_CONTAINER_OUTPUT="${1:-}"
shift

################################################################################
# Are the commands and permissions valid?
################################################################################

SINGULARITY_BUILDDEF="${1:-}"

# not allowed to use --writable and --sandbox
if [ -n "${SINGULARITY_WRITABLE:-}" -a -n "${SINGULARITY_SANDBOX:-}" ]; then
    message ERROR "--writable and --sandbox are conflicting options. There can be only one.\n"
    ABORT 255
fi

# The user is not root
if [ "$USERID" != "0" ]; then

    # They want sandbox
    if [ -n "${SINGULARITY_SANDBOX:-}" ]; then

        # Sandbox with deffile is no go.
        if eval is_deffile "${SINGULARITY_BUILDDEF}"; then 
            message ERROR "You must be the root user to sandbox from a definition file\n"
            exit 1

        # Sandbox with anything else is ok
        else
            message WARNING "Building sandbox as non-root may result in wrong file permissions\n"
        fi
    fi
fi


################################################################################
# Sandbox Directory and Image Creation
################################################################################


# if the container exists, build into it (or delete it if -F was passed)
if [ -e "$SINGULARITY_CONTAINER_OUTPUT" ]; then
    if eval is_image "${SINGULARITY_CONTAINER_OUTPUT}"; then
        if [ -z "${SINGULARITY_FORCE:-}" ]; then 
            message 1 "Building into existing container: $SINGULARITY_CONTAINER_OUTPUT\n"
            SINGULARITY_BTSTRP_2EXISTING=1
        elif [ -n "${SINGULARITY_FORCE:-}" ]; then
            # this may be a directory, so rm -rf 
            rm -rf "$SINGULARITY_CONTAINER_OUTPUT"
        fi
    else
        message ERROR "$SINGULARITY_CONTAINER_OUTPUT is not an image file\n"
        exit 1
    fi
fi

if [ -n "${SINGULARITY_SANDBOX:-}" ]; then
    SINGULARITY_ROOTFS="$SINGULARITY_CONTAINER_OUTPUT"
    SINGULARITY_IMAGE="$SINGULARITY_CONTAINER_OUTPUT"
    if [ "$USERID" == "0" ]; then
        SINGULARITY_ROOTFS_DIRNAME=`dirname "$SINGULARITY_ROOTFS"`
        SINGULARITY_ROOTFS_DIRNAME_OWNER=`stat -c '%u' $SINGULARITY_ROOTFS_DIRNAME`
    fi
    if [ ! -d "${SINGULARITY_ROOTFS:-}" ]; then
        if ! mkdir -p "${SINGULARITY_ROOTFS:-}"; then
            message ERROR "Could not create sandbox!\n";
            exit 1
        fi
    fi
else
    if ! SINGULARITY_ROOTFS=`mktemp -d ${SINGULARITY_TMPDIR:-/tmp}/.singularity-build.XXXXXX`; then
        message ERROR "Failed to create temporary directory\n"
        ABORT 255
    fi
    SINGULARITY_IMAGE="$SINGULARITY_ROOTFS"
fi

export SINGULARITY_IMAGE SINGULARITY_ROOTFS \
       SINGULARITY_CHECKS SINGULARITY_CHECKTAGS SINGULARITY_CHECKLEVEL


################################################################################
# Image Handlers
# Note this code is redundant, needs to be consolidated with handlers/ folder
################################################################################


case $SINGULARITY_BUILDDEF in
    docker://*)
        SINGULARITY_CONTAINER="$SINGULARITY_BUILDDEF"
        if ! SINGULARITY_CONTENTS=`mktemp ${TMPDIR:-/tmp}/.singularity-layers.XXXXXXXX`; then
            message ERROR "Failed to create temporary directory\n"
            ABORT 255
        fi
        export SINGULARITY_CONTAINER SINGULARITY_CONTENTS
        eval_abort "$SINGULARITY_libexecdir/singularity/python/import.py"
        message 1 "Importing: base Singularity environment\n"
        zcat "$SINGULARITY_libexecdir/singularity/bootstrap-scripts/environment.tar" | tar xBf - -C "${SINGULARITY_ROOTFS}" || exit $?

        for i in `cat "$SINGULARITY_CONTENTS"`; do
            message 1 "Importing: $i\n"
            zcat "$i" | (cd "$SINGULARITY_ROOTFS"; tar --exclude=dev/* -xf - ) || exit $?
        done

        SINGULARITY_BUILDDEF="$SINGULARITY_ROOTFS"
        SINGULARITY_QUIET_SANDBOXMESSAGE=1
    ;;

    shub://*)
        SINGULARITY_CONTAINER="$SINGULARITY_BUILDDEF"
        if ! SINGULARITY_CONTENTS=`mktemp ${TMPDIR:-/tmp}/.singularity-layerfile.XXXXXX`; then
            message ERROR "Failed to create temporary directory\n"
            ABORT 255
        fi

        # If not set, don't export PULLFOLDER. PULL in shub/main.py will default to it
        if [ ! -n "${SINGULARITY_PULLFOLDER:-}" ]; then
            export SINGULARITY_CONTENTS SINGULARITY_CONTAINER
        else
            export SINGULARITY_PULLFOLDER SINGULARITY_CONTENTS SINGULARITY_CONTAINER
        fi

        # relying on pull.py for error checking here
        ${SINGULARITY_libexecdir}/singularity/python/pull.py

        # switch $SINGULARITY_CONTAINER from remote to local 
        SINGULARITY_BUILDDEF=`cat $SINGULARITY_CONTENTS`
        SINGULARITY_CLEANUP=1
    ;;

esac


if [ -f "${SINGULARITY_BUILDDEF:-}" ]; then
    if eval is_tar "${SINGULARITY_BUILDDEF}"; then
        message 1 "Building from local tar file: $SINGULARITY_BUILDDEF\n"
        nonroot_build_warning
        if ! eval "zcat_compat ${SINGULARITY_BUILDDEF} 2>/dev/null | tar xf - -C ${SINGULARITY_ROOTFS}" >/dev/null 2>&1; then
            message ERROR "Failed to export contents of ${SINGULARITY_BUILDDEF} to ${SINGULARITY_ROOTFS}\n"
            ABORT 255
        fi

    elif eval is_image "${SINGULARITY_BUILDDEF}"; then
        message 1 "Building from local image: $SINGULARITY_BUILDDEF\n"
        nonroot_build_warning
        if ! eval "${SINGULARITY_bindir}"/singularity image.export "${SINGULARITY_BUILDDEF}" 2>/dev/null | tar xBf - -C "${SINGULARITY_ROOTFS}" >/dev/null 2>&1; then
            message ERROR "Failed to export contents of ${SINGULARITY_BUILDDEF} to ${SINGULARITY_ROOTFS}\n"
            ABORT 255
        fi

    elif eval is_deffile "${SINGULARITY_BUILDDEF}"; then 
        message 1 "Using container recipe deffile: $SINGULARITY_BUILDDEF\n"
        if [ "$USERID" != "0" ]; then
            message ERROR "You must be the root user to build from a definition file\n"
            exit 1
        fi
        if [ -n "${SINGULARITY_BTSTRP_2EXISTING:-}" ]; then # we are bootstrapping to an existing image and need to populate the rootfs
            if ! eval "${SINGULARITY_bindir}"/singularity image.export "${SINGULARITY_CONTAINER_OUTPUT}" | tar xBf - -C "${SINGULARITY_ROOTFS}"; then
                message ERROR "Failed to export contents of ${SINGULARITY_BUILDDEF} to ${SINGULARITY_ROOTFS}\n"
                ABORT 255
            fi
        fi
        export SINGULARITY_BUILDDEF
        eval_abort "$SINGULARITY_libexecdir/singularity/bin/builddef"

    else
        message ERROR "Unsupported file type: $SINGULARITY_BUILDDEF\n"
        exit 1
    fi

elif [ -d "$SINGULARITY_BUILDDEF" ]; then
    nonroot_build_warning
    if [ -z ${SINGULARITY_QUIET_SANDBOXMESSAGE:-} ]; then
        message 1 "Building image from sandbox: $SINGULARITY_BUILDDEF\n"
        SINGULARITY_NOCLEANUP=1 # don't delete when building directly from sandbox
    fi
    SINGULARITY_ROOTFS="$SINGULARITY_BUILDDEF"

else
    message ERROR "Unknown container build definition format: $SINGULARITY_BUILDDEF\n"
    ABORT 255
fi


if [ "$USERID" != "0" ]; then
    # This is required as some files in a container are not readable by owner and
    # thus fail to build when not root.
    chmod u+rw -R "$SINGULARITY_ROOTFS"
fi

if [ -z "${SINGULARITY_SANDBOX:-}" ]; then
    if [ -n "${SINGULARITY_WRITABLE:-}" ]; then

        if [ -z "${SINGULARITY_BTSTRP_2EXISTING:-}" ]; then
            CONTAINER_SIZE=`du -sm ${SINGULARITY_ROOTFS} | cut -f 1`
            # using the let builtin instead of bc which is not guaranteed 
            let "IMAGE_SIZE=(${CONTAINER_SIZE:-768} * 125)/100"
            if [ "$IMAGE_SIZE" -lt 10 ]; then
                IMAGE_SIZE=10
            fi
            message 1 "Creating empty Singularity writable container ${CONTAINER_SIZE}MB\n"
            eval_abort ${SINGULARITY_bindir}/singularity image.create --size ${IMAGE_SIZE} ${SINGULARITY_CONTAINER_OUTPUT}
        fi

        message 1 "Building Singularity image...\n"
        tar -cf - -C "$SINGULARITY_ROOTFS" . | eval_abort ${SINGULARITY_bindir}/singularity image.import ${SINGULARITY_CONTAINER_OUTPUT}
    else
        message 1 "Building Singularity image...\n"
        if [ "$USERID" != "0" ]; then
            OPTS="-all-root"
        else
            OPTS=""
        fi
        if ! mksquashfs "$SINGULARITY_ROOTFS/" "$SINGULARITY_CONTAINER_OUTPUT" -noappend $OPTS > /dev/null; then
            message ERROR "Failed squashing image, left template directory at: $SINGULARITY_ROOTFS\n"
            exit 1
        fi
    fi

    if ! ${SINGULARITY_libexecdir}/singularity/bin/prepheader "$SINGULARITY_CONTAINER_OUTPUT" > /dev/null; then
        message ERROR "Failed to prepend singularity header\n"
	exit 1
    fi

    chmod a+x "$SINGULARITY_CONTAINER_OUTPUT"
fi

message 1 "Singularity container built: $SINGULARITY_CONTAINER_OUTPUT\n"