From deceba34d995656a51658877b1737c7c891038e2 Mon Sep 17 00:00:00 2001 From: "M. Eric Irrgang" Date: Tue, 29 Sep 2020 17:27:03 +0300 Subject: [PATCH] Specify OMP_NUM_THREADS for Pyhon CI scripts. * Explicitly propagate OMP_NUM_THREADS through mpiexec. * Apply self-consistent CI memory parameters. * Only use 1 OMP thread for gmxapi Python tests. We want to use two tMPI ranks, and no extra threads. tMPI threads are set separately. Refs #3704 --- admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh | 2 ++ admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh | 2 ++ admin/ci-scripts/build-and-test-sample_restraint-2020.sh | 1 + admin/ci-scripts/build-and-test-sample_restraint-2021.sh | 1 + admin/gitlab-ci/python-gmxapi.gitlab-ci.yml | 9 +++++++-- .../gitlab-ci/sample_restraint-regression.gitlab-ci.yml | 3 ++- admin/gitlab-ci/sample_restraint.gitlab-ci.yml | 6 +++++- 7 files changed, 20 insertions(+), 4 deletions(-) diff --git a/admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh b/admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh index f2a4c2a141..a48cd4ca19 100644 --- a/admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh +++ b/admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh @@ -48,6 +48,7 @@ PROGRAM=(`which python` -m mpi4py -m pytest \ if [ -x `which mpiexec` ]; then PYTHONDONTWRITEBYTECODE=1 \ mpiexec --allow-run-as-root \ + -x OMP_NUM_THREADS=1 \ --mca opal_warn_on_missing_libcuda 0 \ --mca orte_base_help_aggregate 0 \ -n 1 ${PROGRAM[@]} --junitxml=$PLUGIN_MPI_TEST_XML : \ @@ -68,6 +69,7 @@ PROGRAM=(`which python` -m mpi4py -m pytest \ if [ -x `which mpiexec` ]; then PYTHONDONTWRITEBYTECODE=1 \ mpiexec --allow-run-as-root \ + -x OMP_NUM_THREADS=1 \ --mca opal_warn_on_missing_libcuda 0 \ --mca orte_base_help_aggregate 0 \ -n 1 ${PROGRAM[@]} --junitxml=$PLUGIN_MPI_TEST_XML : \ diff --git a/admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh b/admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh index b0b623cc52..e7512e13e8 100644 --- a/admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh +++ b/admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh @@ -52,6 +52,7 @@ PROGRAM=(`which python` -m mpi4py -m pytest \ if [ -x `which mpiexec` ]; then PYTHONDONTWRITEBYTECODE=1 \ mpiexec --allow-run-as-root \ + -x OMP_NUM_THREADS=1 \ --mca opal_warn_on_missing_libcuda 0 \ --mca orte_base_help_aggregate 0 \ -n 1 ${PROGRAM[@]} --junitxml=$PLUGIN_MPI_TEST_XML : \ @@ -72,6 +73,7 @@ PROGRAM=(`which python` -m mpi4py -m pytest \ if [ -x `which mpiexec` ]; then PYTHONDONTWRITEBYTECODE=1 \ mpiexec --allow-run-as-root \ + -x OMP_NUM_THREADS=1 \ --mca opal_warn_on_missing_libcuda 0 \ --mca orte_base_help_aggregate 0 \ -n 1 ${PROGRAM[@]} --junitxml=$PLUGIN_MPI_TEST_XML : \ diff --git a/admin/ci-scripts/build-and-test-sample_restraint-2020.sh b/admin/ci-scripts/build-and-test-sample_restraint-2020.sh index f7bf6eaac2..e82ffe71d9 100644 --- a/admin/ci-scripts/build-and-test-sample_restraint-2020.sh +++ b/admin/ci-scripts/build-and-test-sample_restraint-2020.sh @@ -52,6 +52,7 @@ pushd python_packaging/sample_restraint if [ -x `which mpiexec` ]; then PYTHONDONTWRITEBYTECODE=1 \ mpiexec --allow-run-as-root \ + -x OMP_NUM_THREADS=1 \ --mca opal_warn_on_missing_libcuda 0 \ --mca orte_base_help_aggregate 0 \ -n 1 ${PROGRAM[@]} --junitxml=$PLUGIN_MPI_TEST_XML : \ diff --git a/admin/ci-scripts/build-and-test-sample_restraint-2021.sh b/admin/ci-scripts/build-and-test-sample_restraint-2021.sh index ab9b0d2d19..8de02b5270 100644 --- a/admin/ci-scripts/build-and-test-sample_restraint-2021.sh +++ b/admin/ci-scripts/build-and-test-sample_restraint-2021.sh @@ -78,6 +78,7 @@ pushd python_packaging/sample_restraint if [ -x `which mpiexec` ]; then PYTHONDONTWRITEBYTECODE=1 \ mpiexec --allow-run-as-root \ + -x OMP_NUM_THREADS=1 \ --mca opal_warn_on_missing_libcuda 0 \ --mca orte_base_help_aggregate 0 \ -n 1 ${PROGRAM[@]} --junitxml=$PLUGIN_MPI_TEST_XML : \ diff --git a/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml b/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml index 682a951e58..a9801ec389 100644 --- a/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml +++ b/admin/gitlab-ci/python-gmxapi.gitlab-ci.yml @@ -13,6 +13,7 @@ KUBERNETES_CPU_LIMIT: 2 KUBERNETES_CPU_REQUEST: 2 KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi PY_UNIT_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml PY_MPI_UNIT_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml PY_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-JUnitTestResults.xml @@ -20,7 +21,7 @@ EXTRA_INSTALLS: "curl libbz2-dev libffi-dev liblzma-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev llvm python-openssl tk-dev zlib1g-dev" script: - source $INSTALL_DIR/bin/GMXRC - - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR bash admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh + - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR OMP_NUM_THREADS=1 bash admin/ci-scripts/build-and-test-py-gmxapi-0.1.sh # Note that the XML output is concatenated when multiple reports are produced # by a single job. If this presents problems, we may have to split the tests # into multiple jobs. In particular, test names will appear once for MPI and @@ -81,13 +82,17 @@ gmxapi-0.1:clang-8:py-3.8.2: image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 stage: test variables: + KUBERNETES_CPU_LIMIT: 2 + KUBERNETES_CPU_REQUEST: 2 + KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi PY_UNIT_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml PY_MPI_UNIT_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml PY_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-JUnitTestResults.xml PY_MPI_ACCEPTANCE_TEST_XML: $CI_PROJECT_DIR/gmxapi-acceptance-mpi-JUnitTestResults.xml script: - source $INSTALL_DIR/bin/GMXRC - - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR bash admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh + - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR OMP_NUM_THREADS=1 bash admin/ci-scripts/build-and-test-py-gmxapi-0.2.sh artifacts: reports: junit: diff --git a/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml b/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml index 7fd4fc9b2a..f7c7f60efb 100644 --- a/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml +++ b/admin/gitlab-ci/sample_restraint-regression.gitlab-ci.yml @@ -11,12 +11,13 @@ KUBERNETES_CPU_LIMIT: 2 KUBERNETES_CPU_REQUEST: 2 KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi PLUGIN_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml PLUGIN_MPI_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml EXTRA_INSTALLS: "curl libbz2-dev libffi-dev liblzma-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev llvm python-openssl tk-dev zlib1g-dev" script: - source $INSTALL_DIR/bin/GMXRC - - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR bash admin/ci-scripts/build-and-test-sample_restraint-2020.sh + - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR OMP_NUM_THREADS=1 bash admin/ci-scripts/build-and-test-sample_restraint-2020.sh artifacts: reports: junit: diff --git a/admin/gitlab-ci/sample_restraint.gitlab-ci.yml b/admin/gitlab-ci/sample_restraint.gitlab-ci.yml index 1c99c8fb97..3488feed78 100644 --- a/admin/gitlab-ci/sample_restraint.gitlab-ci.yml +++ b/admin/gitlab-ci/sample_restraint.gitlab-ci.yml @@ -6,12 +6,16 @@ image: gromacs/cmake-3.15.7-llvm-8-intelopencl-openmpi:2020 stage: test variables: + KUBERNETES_CPU_LIMIT: 2 + KUBERNETES_CPU_REQUEST: 2 + KUBERNETES_MEMORY_LIMIT: 2Gi + KUBERNETES_MEMORY_REQUEST: 2Gi PLUGIN_TEST_XML: $CI_PROJECT_DIR/py-JUnitTestResults.xml PLUGIN_MPI_TEST_XML: $CI_PROJECT_DIR/py-mpi-JUnitTestResults.xml EXTRA_INSTALLS: "curl libbz2-dev libffi-dev liblzma-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libssl-dev llvm python-openssl tk-dev zlib1g-dev" script: - source $INSTALL_DIR/bin/GMXRC - - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR bash admin/ci-scripts/build-and-test-sample_restraint-2020.sh + - source $VENVPATH/bin/activate && INSTALL_DIR=$PWD/$INSTALL_DIR OMP_NUM_THREADS=1 bash admin/ci-scripts/build-and-test-sample_restraint-2020.sh artifacts: reports: junit: -- 2.22.0