sed tips and tricks -

I’m creating a Puppet Starter Kit with some standard manifests included and a complete set of documentation. All documentation should be written in Markdown and will be served by Markdoc. But I want to generate all Markdown files from the Puppet manifests, so I only need to document the manifest file. Generating the Markdown is not that difficult, except that I kept ending up with empty lines at the top of the manifest code and I wanted to get rid of those. Of course this should be done with sed, because the whole generation process is written in bash. When playing around with sed I found _~ sed `/./,$!d' filename _~ which, I think, is genius in it’s simplicity. After you find something, do not remove. Life in UNIX and Linux is nice!

The complete (quick and dirty) script is:

#!/bin/bash
# vi: set sw=4 ts=4 ai:

process()
{
    infile="${1}"
    outfile="${2}"
    readme="$(dirname ${infile})/README.md"

    echo "# Puppet manifest documentation"  >> "${outfile}"
    echo "----------"           >> "${outfile}"

    if [[ -f ${readme} ]]
    then
        cp -p ${readme} ${outfile}
    else
        if $(grep -q '^#-- DOC START --#' "${infile}")
        then
            sed -n  -e '/^#-- DOC START --#/,/#-- DOC END --#/p' "${infile}"    | \
                sed -e '/^#-- DOC START --#/d'                                  \
                    -e '/^#-- DOC END --#/d'                                    \
                    -e 's/^#//'                                                 \
                    -e 's/^ //' > "${outfile}"
        else
            echo "### No infile documentation available" >> "${outfile}"
        fi
        echo "----------"               >> "${outfile}"
        echo "# Puppet manifest"        >> "${outfile}"
        echo "### File: \`${file}\`"    >> "${outfile}"
        echo "~~~~~~~~~~ {.puppet}"     >> "${outfile}"

        if $(grep -q '^#-- DOC START --#' "${infile}")
        then
            sed -e '1,/^#-- DOC END --#/d' "${infile}"      \
                -e '/^# `\$[I]d/d'                          \
                -e '/./,$!d' >> "${outfile}"
        else
            cat "${f}" >> "${outfile}"
        fi
    fi
    echo "~~~~~~~~~~" >> "${outfile}"

    echo "<br />" >> "${outfile}"
    if $(grep -q '`\$Id' "${infile}")
    then
        grep '`\$[I]d' "${infile}" | sed -e 's/^# *//' -e 's/\(`\$[I]d\)/Version: \1/' >> "${outfile}"
        echo "----------" >> "${outfile}"
    fi
}

TOPDIR="$(puppet config print confdir)/doc"
if [[ ! -d ${TOPDIR} ]]
then
    echo "The top documentation directory cannot be found (${TOPDIR})"
    exit 1
fi

cd ${TOPDIR} ||
{   echo "Cannot change to the documentation directory"
    exit 1
}

if [[ -d todo ]]
then
    echo "# Todo list for Puppet" > todo/index.md
    for f in $(ls todo/*.md | grep -v 'index.md$')
    do
        file=$(basename ${f})
        md=$(basename ${f} .md)
        echo "* [${md}](${md})" >> todo/index.md
    done
fi

find puppet -type f | grep -v '/\.svn/' | xargs rm -f
envs="production develop"

echo "# Puppet manifests" > puppet/index.md
mkdir -p puppet

# Quick and dirty file selection
for f in $(ls   ../manifests/*.pp           \
                ../manifests/*/*.pp         \
                ../manifests/*/*/*.pp       \
                ../manifests/*/*/*/*.pp     \
                ../manifests/*/*/*/*/*.pp   \
                ../manifests/*/*/*/*/*/*.pp \
                    2>/dev/null)
do
    file=$(basename ${f})
    md=$(basename ${f} .pp)

    if [[ "${md}" = "init" ]]
    then
        md=$(basename $(dirname $(dirname ${f})))
    fi
    echo "* [${md}]($md)" >> puppet/${env}/index.md

    process "${f}" "puppet/${md}.md"
done

echo "# Puppet modules" >> puppet/index.md
for env in ${envs}
do
    mkdir -p puppet/${env}
    echo "* [${env}](${env})" >> puppet/index.md
    echo "# Puppet environment \`${env}\`" > puppet/${env}/index.md

    for f in $(ls ../modules/${env}/*/manifests/*.pp 2>/dev/null)
    do
        file=$(basename ${f})
        md=$(basename ${f} .pp)

        if [[ "${md}" = "init" ]]
        then
            md=$(basename $(dirname $(dirname ${f})))
        fi
        echo "* [${md}]($md)" >> puppet/${env}/index.md

        process "${f}" "puppet/${env}/${md}.md"
    done
done


if [[ x"${1:-}" = x"-a" ]]
then
    rm -rf resources
    mkdir resources
    cd resources

    echo "# All known resources for Puppet version $(puppet -V)" > index.md
    echo "| Resource name | Description |" >> index.md
    echo "|---------------|-------------|" >> index.md
    puppet describe --list | awk 'NR > 1 { print }' | sort | while read res min desc
    do
        echo "| [\`${res}\`](${res}) | ${desc} |" >> index.md
        puppet describe "${res}" > "${res}.md"
    done
    cd ..
fi

markdoc build

sysadm linux code

See also