From: Teemu Murtola Date: Fri, 5 Jun 2015 10:47:50 +0000 (+0300) Subject: Improve Jenkins docs build X-Git-Url: http://biod.pnpi.spb.ru/gitweb/?a=commitdiff_plain;h=27b65cdc838b8ddf739f27d973ac60fdeeb142a8;p=alexxy%2Fgromacs.git Improve Jenkins docs build - Use GMX_COMPACT_DOXYGEN=ON for per-patch set builds to save disk space and some running time. - Build also the man and install-guide targets, which were previously missing. - Make the build fail if it actually fails, and unstable if it just produces warnings. - Make the end of the console output contain some useful information on what exactly failed. This gets put into unsuccessful-reason.log, which can be used by Jenkins to show as the reason of the build failure when it reports back to Gerrit. - Add options to the script to make it more convenient to test. Fix some issues found in man page generation and in handling of GMX_COMPACT_DOXYGEN. Change-Id: Ice7622d7760c3dbd74f7b5b25700482d0a08e06b --- diff --git a/admin/build-docs.sh b/admin/build-docs.sh index 6791d77660..8426b844fb 100755 --- a/admin/build-docs.sh +++ b/admin/build-docs.sh @@ -40,6 +40,9 @@ # to Jenkins tries to be as static as possible to avoid the need to rebase # changes because of job configuration changes. # +# Command line options to use an existing build directory are provided for more +# easily testing the script outside Jenkins. +# # Interface from Jenkins to this script is: # * the location of this script file, # * the first argument is the build type ("gerrit" or "nightly"), @@ -52,22 +55,31 @@ # * stdout (any instance of "FAILED" marks the build unstable), # * HTML documentation produced at build/docs/html/, and # * log files under build/docs/logs/: +# - fail-reason.log contains a short description of what failed/was unstable # - all .log files from this directory are published as Jenkins artifacts # - all .log files under doxygen/ subdir are scanned for Doxygen warnings # - all .log files under sphinx/ subdir are scanned for Sphinx warnings +# Parse arguments function usage() { - echo "usage: build-docs.sh gerrit|nightly [-D...=...]*" - echo "All additional options (-D etc.) are passed to CMake" + echo "usage: build-docs.sh gerrit|nightly" + echo " [--use-build-dir=] [--skip-cmake]" + echo " [--build-cmd=] [-D...=...]*" + echo "All other additional options (-D etc.) are passed to CMake" } +declare -a cmake_args + +build_dir=build +build_cmd=make +do_cmake=1 GERRIT_MODE= NIGHTLY_MODE= case "$1" in gerrit) GERRIT_MODE=1 ;; - nightly) + nightly) NIGHTLY_MODE=1 ;; *) @@ -76,6 +88,39 @@ case "$1" in ;; esac shift +for arg in "$@" ; do + if [[ "$arg" == --use-build-dir=* ]] ; then + build_dir=${arg#--use-build-dir=} + elif [[ "$arg" == --build-cmd=* ]] ; then + build_cmd=${arg#--build-cmd=} + elif [[ "$arg" == --skip-cmake ]] ; then + do_cmake= + else + cmake_args+=("$arg") + fi +done + +log_output_dir=docs/logs +unsuccessful_log=$log_output_dir/unsuccessful-reason.log + +# Utilities for failure reporting +FAILED= +function report_failure() { + echo "$1" + echo "$1" >> $unsuccessful_log + FAILED=1 +} +function report_unstable() { + echo "FAILED: $1" + echo "$1" >> $unsuccessful_log +} +function exit_if_failed() { + if [[ $FAILED ]] ; then + echo "Documentation build FAILED:" + cat $unsuccessful_log + exit 1 + fi +} set -x @@ -87,52 +132,96 @@ if [ ! -f "admin/build-docs.sh" ] ; then fi # Some of the documentation targets can only be built out of source. -rm -rf build -mkdir build -cd build +mkdir -p $build_dir +cd $build_dir +# Copy the output logs to a static hierarchy to allow changing the exact file +# names and adding new logs without changing the Jenkins job configuration. +mkdir -p $log_output_dir +rm -f $unsuccessful_log + +cmake_args+=("-DGMX_BUILD_HELP=ON" "-DGMX_BUILD_MANUAL=ON") +if [[ $GERRIT_MODE ]] ; then + cmake_args+=("-DGMX_COMPACT_DOXYGEN=ON") +fi # Make a configuration that is fast, just for building docs. -cmake .. "$@" \ - -DGMX_BUILD_HELP=yes -DGMX_BUILD_MANUAL=yes \ - -DCMAKE_BUILD_TYPE=Debug -DGMX_OPENMP=off -DGMX_SIMD=None -DGMX_GPU=no +cmake_args+=("-DCMAKE_BUILD_TYPE=Debug" "-DGMX_OPENMP=OFF" "-DGMX_SIMD=None" "-DGMX_GPU=OFF") +if [[ $do_cmake ]] ; then + cmake $srcdir "${cmake_args[@]}" || report_failure "CMake configuration failed" + exit_if_failed +else + echo "Skipped cmake; args ${cmake_args[@]}" +fi # Need to make gmx to export rst help. -make gmx -j 2 || echo "FAILED to make gmx" +$build_cmd gmx -j 2 || report_failure "Building gmx failed" -# webpage target makes all the documentation components. -make webpage || echo "FAILED to make webpage" +# Do various parts individually to report the errors. +$build_cmd manual || report_failure "PDF manual failed to build" +cp docs/manual/gromacs.log $log_output_dir/ +grep "LaTeX Warning: Reference .* on page .* undefined" docs/manual/gromacs.log && report_unstable "undefined references in manual" -# Ideally, we would also make the other components individually, to check -# that the targets still work, but most of them duplicate the build - -grep "LaTeX Warning: Reference .* on page .* undefined" docs/manual/gromacs.log && echo "FAILED - undefined references in manual" +$build_cmd doxygen-all || report_failure "Doxygen documentation failed to build" # run check-source if [[ $GERRIT_MODE ]] ; then - make check-source || echo "FAILED: check-source found errors" + $build_cmd check-source || report_failure "check-source failed to run" fi +mkdir $log_output_dir/doxygen +cp docs/doxygen/*.log $log_output_dir/doxygen/ +for target in {check-source,doxygen-xml,doxygen-full,doxygen-lib,doxygen-user} ; do + if [ -s $log_output_dir/doxygen/$target.log ] ; then + report_unstable "$target produced warnings" + fi +done +exit_if_failed + +# Generate Sphinx input. +$build_cmd sphinx-input || report_failure "Generating Sphinx input failed" +$build_cmd sphinx-programs || report_failure "Running gmx help -export rst failed" +exit_if_failed + +# Run various Sphinx commands +$build_cmd webpage-sphinx || report_failure "Sphinx: HTML generation failed" +$build_cmd man || report_failure "Sphinx: man page generation failed" +$build_cmd install-guide || report_failure "Sphinx: INSTALL generation failed" +mkdir ${log_output_dir}/sphinx +cp docs/sphinx-*.log ${log_output_dir}/sphinx/ +for log in {html,man,install} ; do + if [ -s $log_output_dir/sphinx/sphinx-$log ] ; then + case $log in + html) + format="HTML" + ;; + man) + format="man page" + ;; + install) + format="INSTALL" + ;; + esac + report_unstable "Sphinx: $format generation produced warnings" + fi +done +exit_if_failed + +# webpage target makes some final work for the documentation. +$build_cmd webpage || report_failure "webpage target failed to build" -# Copy the output logs to a static hierarchy to allow changing the exact file -# names and adding new logs without changing the Jenkins job configuration. -mkdir docs/logs -cp docs/manual/manual.log docs/logs/ -mkdir docs/logs/doxygen -cp docs/doxygen/*.log docs/logs/doxygen/ -mkdir docs/logs/sphinx -cp docs/sphinx-*.log docs/logs/sphinx/ - -cd .. +cd $srcdir -if [ -f build/docs/html/index.html ] ; then - linkchecker build/docs/html/index.html -f docs/linkcheckerrc \ +if [ -f $build_dir/docs/html/index.html ] ; then + linkchecker $build_dir/docs/html/index.html -f docs/linkcheckerrc \ --ignore-url html-full --ignore-url html-user --ignore-url html-lib \ --ignore-url .tar.gz --ignore-url _sources # add this to previous line once content stabilizes # || echo "FAILED linkchecker" else - echo "No build/docs/html/index.html was made; can't check links!" + echo "No docs/html/index.html in $build_dir was made; can't check links!" fi -# Exit with zero code to avoid failing builds because of whatever was the last -# command executed. +cd $build_dir + +[[ -s $unsuccessful_log ]] && cat $unsuccessful_log +[[ $FAILED ]] && exit 1 exit 0 diff --git a/docs/doxygen/CMakeLists.txt b/docs/doxygen/CMakeLists.txt index ff9dc058a8..90f89c1e36 100644 --- a/docs/doxygen/CMakeLists.txt +++ b/docs/doxygen/CMakeLists.txt @@ -111,7 +111,7 @@ if (DOXYGEN_FOUND) ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile-common ${CMAKE_CURRENT_BINARY_DIR}/RunDoxygen.cmake) if (GMX_COMPACT_DOXYGEN) - list(APPEND doxygen_deps ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile-compact) + list(APPEND doxygen_deps ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile-compact) endif() gmx_add_custom_output_target(doxygen-source-timestamp OUTPUT STAMP @@ -210,7 +210,6 @@ if (DOXYGEN_FOUND) ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/check-source.py -S ${CMAKE_SOURCE_DIR} -B ${CMAKE_BINARY_DIR} -l ${CMAKE_CURRENT_BINARY_DIR}/check-source.log - --exitcode --ignore ${CMAKE_CURRENT_SOURCE_DIR}/suppressions.txt --ignore-cycles ${CMAKE_CURRENT_SOURCE_DIR}/cycle-suppressions.txt) add_custom_target(check-source COMMAND ${check_source_command} diff --git a/src/gromacs/commandline/cmdlinehelpmodule.cpp b/src/gromacs/commandline/cmdlinehelpmodule.cpp index 1183ce68f2..2a2959cbf8 100644 --- a/src/gromacs/commandline/cmdlinehelpmodule.cpp +++ b/src/gromacs/commandline/cmdlinehelpmodule.cpp @@ -583,7 +583,7 @@ void HelpExportReStructuredText::exportModuleHelp( displayName.c_str(), tag.c_str(), module.shortDescription())); manPagesFile_->writeLine( - formatString(" ('programs/%s', '%s', \"%s\", '', 1),", + formatString(" ('onlinehelp/%s', '%s', \"%s\", '', 1),", tag.c_str(), tag.c_str(), module.shortDescription())); } @@ -593,7 +593,7 @@ void HelpExportReStructuredText::finishModuleExport() indexFile_.reset(); // TODO: Generalize. manPagesFile_->writeLine( - formatString(" ('programs/%s', '%s', '%s', '', 1)", + formatString(" ('onlinehelp/%s', '%s', '%s', '', 1)", binaryName_.c_str(), binaryName_.c_str(), RootHelpText::title)); manPagesFile_->writeLine("]"); diff --git a/src/gromacs/commandline/tests/refdata/CommandLineHelpModuleTest_ExportsHelp.xml b/src/gromacs/commandline/tests/refdata/CommandLineHelpModuleTest_ExportsHelp.xml index f92487f6eb..b2612d7fb7 100644 --- a/src/gromacs/commandline/tests/refdata/CommandLineHelpModuleTest_ExportsHelp.xml +++ b/src/gromacs/commandline/tests/refdata/CommandLineHelpModuleTest_ExportsHelp.xml @@ -9,10 +9,10 @@ ]]>