2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
35 /*! \libinternal \file
37 * Extra assertions for unit tests.
39 * This file provides assertion macros that extend/replace Google Test
42 * - floating-point comparison
43 * - comparison against NULL
48 * The implementation is somewhat ugly, and accesses some Google Test
49 * internals. Could be nice to clean it up a bit.
52 * \author Teemu Murtola <teemu.murtola@gmail.com>
54 * \ingroup module_testutils
56 #ifndef GMX_TESTUTILS_TESTASSERTS_H
57 #define GMX_TESTUTILS_TESTASSERTS_H
61 #include <gtest/gtest.h>
63 #include "gromacs/utility/basedefinitions.h"
64 #include "gromacs/utility/exceptions.h"
65 #include "gromacs/utility/real.h"
77 * Called for an expected exception from EXPECT_THROW_GMX().
79 * \param[in] ex Exception that was thrown.
81 void processExpectedException(const std::exception& ex);
83 } // namespace internal
85 //! \libinternal \addtogroup module_testutils
88 /*! \name Assertions for exceptions
90 * These macros replace `(ASSERT|EXPECT)(_NO)?_THROW` from Google Test.
91 * They are used exactly like the Google Test ones, but also print details of
92 * any unexpected exceptions using \Gromacs-specific routines.
93 * This makes it much easier to see at one glance what went wrong.
94 * See Google Test documentation for details on how to use the macros.
100 * Internal implementation macro for exception assertations.
102 * \param statement Statements to execute.
103 * \param expected_exception Exception type that \p statement should throw.
104 * \param fail Function/macro to call on failure.
106 * The implementation is copied and adjusted from
107 * include/gtest/internal/gtest-internal.h in Google Test 1.6.0.
109 #define GMX_TEST_THROW_(statement, expected_exception, fail) \
110 GTEST_AMBIGUOUS_ELSE_BLOCKER_ \
111 if (::testing::AssertionResult gmx_ar = ::testing::AssertionSuccess()) \
113 bool gmx_caught_expected = false; \
116 GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \
118 catch (expected_exception const& ex) \
120 gmx_caught_expected = true; \
121 ::gmx::test::internal::processExpectedException(ex); \
123 catch (std::exception const& ex) \
125 gmx_ar << "Expected: " #statement " throws an exception of type " \
126 << #expected_exception ".\n Actual: it throws a different type.\n" \
127 << "Exception details:\n" \
128 << ::gmx::formatExceptionMessageToString(ex); \
129 goto GTEST_CONCAT_TOKEN_(gmx_label_testthrow_, __LINE__); \
133 gmx_ar << "Expected: " #statement " throws an exception of type " \
134 << #expected_exception ".\n Actual: it throws a different type."; \
135 goto GTEST_CONCAT_TOKEN_(gmx_label_testthrow_, __LINE__); \
137 if (!gmx_caught_expected) \
139 gmx_ar << "Expected: " #statement " throws an exception of type " \
140 << #expected_exception ".\n Actual: it throws nothing."; \
141 goto GTEST_CONCAT_TOKEN_(gmx_label_testthrow_, __LINE__); \
145 GTEST_CONCAT_TOKEN_(gmx_label_testthrow_, __LINE__) : fail(gmx_ar.message())
148 * Internal implementation macro for exception assertations.
150 * \param statement Statements to execute.
151 * \param fail Function/macro to call on failure.
153 * The implementation is copied and adjusted from
154 * include/gtest/internal/gtest-internal.h in Google Test 1.6.0.
156 #define GMX_TEST_NO_THROW_(statement, fail) \
157 GTEST_AMBIGUOUS_ELSE_BLOCKER_ \
158 if (::testing::AssertionResult gmx_ar = ::testing::AssertionSuccess()) \
162 GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \
164 catch (std::exception const& ex) \
166 gmx_ar << "Expected: " #statement " doesn't throw an exception.\n" \
167 << " Actual: it throws.\n" \
168 << "Exception details:\n" \
169 << ::gmx::formatExceptionMessageToString(ex); \
170 goto GTEST_CONCAT_TOKEN_(gmx_label_testnothrow_, __LINE__); \
174 gmx_ar << "Expected: " #statement " doesn't throw an exception.\n" \
175 << " Actual: it throws."; \
176 goto GTEST_CONCAT_TOKEN_(gmx_label_testnothrow_, __LINE__); \
180 GTEST_CONCAT_TOKEN_(gmx_label_testnothrow_, __LINE__) : fail(gmx_ar.message())
184 * Asserts that a statement throws a given exception.
188 #define EXPECT_THROW_GMX(statement, expected_exception) \
189 GMX_TEST_THROW_(statement, expected_exception, GTEST_NONFATAL_FAILURE_)
191 * Asserts that a statement does not throw.
195 #define EXPECT_NO_THROW_GMX(statement) GMX_TEST_NO_THROW_(statement, GTEST_NONFATAL_FAILURE_)
197 * Asserts that a statement throws a given exception.
201 #define ASSERT_THROW_GMX(statement, expected_exception) \
202 GMX_TEST_THROW_(statement, expected_exception, GTEST_FATAL_FAILURE_)
204 * Asserts that a statement does not throw.
208 #define ASSERT_NO_THROW_GMX(statement) GMX_TEST_NO_THROW_(statement, GTEST_FATAL_FAILURE_)
211 * Wrapper around EXPECT_DEATH_IF_SUPPORTED gtest macro for thread safe execution.
213 * Makes sure that tests that are supposed to trigger an assertion are only executed
214 * in a threadsafe environment, and not when when running under e.g. MPI.
218 #define GMX_EXPECT_DEATH_IF_SUPPORTED(expr, msg) \
221 EXPECT_DEATH_IF_SUPPORTED(expr, msg); \
225 * Wrapper around ASSERT_DEATH_IF_SUPPORTED gtest macro for thread safe execution.
227 * Makes sure that tests that are supposed to trigger an assertion are only executed
228 * in a threadsafe environment when running under e.g. MPI.
232 #define GMX_ASSERT_DEATH_IF_SUPPORTED(expr, msg) \
235 ASSERT_DEATH_IF_SUPPORTED(expr, msg); \
240 /*! \libinternal \brief
241 * Computes and represents a floating-point difference value.
243 * Methods in this class do not throw, except for toString(), which may throw
246 * \see FloatingPointTolerance
248 class FloatingPointDifference
251 /*! \brief Initializes a single-precision difference.
253 * \param ref First term in difference
254 * \param value Second term in difference
256 * For absolute and ULP differences the two parameters are equivalent,
257 * since the difference is symmetric. For relative differences
258 * the first term is interpreted as the reference value, from which
259 * we extract the magnitude to compare with.
261 FloatingPointDifference(float ref, float value);
263 /*! \brief Initializes a double-precision difference.
265 * \param ref First term in difference
266 * \param value Second term in difference
268 * For absolute and ULP differences the two parameters are equivalent,
269 * since the difference is symmetric. For relative differences
270 * the first term is interpreted as the reference value, from which
271 * we extract the magnitude to compare with.
273 FloatingPointDifference(double ref, double value);
276 * Whether one or both of the compared values were NaN.
278 * If this returns `true`, other accessors return meaningless values.
281 //! Returns the difference as an absolute number (always non-negative).
282 double asAbsolute() const { return absoluteDifference_; }
284 * Returns the difference as ULPs (always non-negative).
286 * The ULPs are calculated for the type that corresponds to the
287 * constructor used to initialize the difference.
288 * The ULP difference between 0.0 and -0.0 is zero.
290 uint64_t asUlps() const { return ulpDifference_; }
292 * Whether the compared values were of different sign.
294 * 0.0 and -0.0 are treated as positive and negative, respectively.
296 bool signsDiffer() const { return bSignDifference_; }
298 * Whether the difference is between single- or double-precision
301 bool isDouble() const { return bDouble_; }
302 //! Formats the difference as a string for assertion failure messages.
303 std::string toString() const;
305 //! Returns the magnitude of the original second term of the difference.
306 double termMagnitude() const { return termMagnitude_; }
309 //! Save the magnitude of the reference value for relative (i.e., not ULP) tolerance
310 double termMagnitude_;
311 //! Stores the absolute difference, or NaN if one or both values were NaN.
312 double absoluteDifference_;
313 uint64_t ulpDifference_;
314 bool bSignDifference_;
316 * Whether the difference was computed for single or double precision.
318 * This sets the units for `ulpDifference_`.
323 /*! \libinternal \brief
324 * Specifies a floating-point comparison tolerance and checks whether a
325 * difference is within the tolerance.
327 * The related functions section lists methods that can be construct methods
328 * using less parameters than the full constructor, and with more obvious
329 * semantics. These should be preferred over using the constructor directly.
331 * Several types of tolerances are possible:
332 * - _absolute tolerance_: difference between the values must be smaller than
333 * the given tolerance for the check to pass.
334 * Setting the absolute tolerance to zero disables the absolute tolerance
336 * - _relative tolerance_: the absolute difference between the numbers must
337 * be smaller than the tolerance multiplied by the first number. Setting
338 * the relative tolerance to zero disables this check.
339 * - _ULP tolerance_: ULP (units of least precision) difference between the
340 * values must be smaller than the given tolerance for the check to pass.
341 * Setting the ULP tolerance to zero requires exact match.
342 * Setting the ULP tolerance to UINT64_MAX disables the ULP check.
343 * `0.0` and `-0.0` are treated as equal for the ULP check.
344 * - _sign check_: if set, any values that are of different signs fail the
345 * check (note that this also applies to `0.0` and `-0.0`: a value with a
346 * different sign than the zero will fail the check).
348 * Either an absolute, relative, or ULP tolerance must always be specified.
349 * If several of them are specified, then the check passes if either of the
350 * tolerances is satisfied.
352 * Any combination of absolute, relative, and ULP tolerance can be combined with
353 * the sign check. In this case, the sign check must succeed for the check to
354 * pass, even if other tolerances are satisfied.
356 * The tolerances can be specified separately for single and double precision
357 * comparison. Different initialization functions have different semantics on
358 * how the provided tolerance values are interpreted; check their
361 * Methods in this class do not throw, except for toString(), which may throw
365 * The factory methods that take ULP difference could be better formulated as
366 * methods that take the acceptable number of incorrect bits and/or the number
369 * \see FloatingPointDifference
371 class FloatingPointTolerance
375 * Creates a tolerance with the specified values.
377 * \param[in] singleAbsoluteTolerance
378 * Allowed absolute difference in a single-precision number.
379 * \param[in] doubleAbsoluteTolerance
380 * Allowed absolute difference in a double-precision number.
381 * \param[in] singleRelativeTolerance
382 * Allowed relative difference in a single-precision number.
383 * \param[in] doubleRelativeTolerance
384 * Allowed relative difference in a double-precision number.
385 * \param[in] singleUlpTolerance
386 * Allowed ULP difference in a single-precision number.
387 * \param[in] doubleUlpTolerance
388 * Allowed ULP difference in a double-precision number.
389 * \param[in] bSignMustMatch
390 * Whether sign mismatch fails the comparison.
392 FloatingPointTolerance(float singleAbsoluteTolerance,
393 double doubleAbsoluteTolerance,
394 float singleRelativeTolerance,
395 double doubleRelativeTolerance,
396 uint64_t singleUlpTolerance,
397 uint64_t doubleUlpTolerance,
398 bool bSignMustMatch) :
399 singleAbsoluteTolerance_(singleAbsoluteTolerance),
400 doubleAbsoluteTolerance_(doubleAbsoluteTolerance),
401 singleRelativeTolerance_(singleRelativeTolerance),
402 doubleRelativeTolerance_(doubleRelativeTolerance),
403 singleUlpTolerance_(singleUlpTolerance),
404 doubleUlpTolerance_(doubleUlpTolerance),
405 bSignMustMatch_(bSignMustMatch)
410 * Checks whether a difference is within the specified tolerance.
412 * NaNs are always treated outside the tolerance.
414 bool isWithin(const FloatingPointDifference& difference) const;
416 //! Formats the tolerance as a string for assertion failure messages.
417 std::string toString(const FloatingPointDifference& difference) const;
420 float singleAbsoluteTolerance_;
421 double doubleAbsoluteTolerance_;
422 float singleRelativeTolerance_;
423 double doubleRelativeTolerance_;
424 uint64_t singleUlpTolerance_;
425 uint64_t doubleUlpTolerance_;
426 bool bSignMustMatch_;
430 * Creates a tolerance that only allows a specified ULP difference.
432 * The tolerance uses the given ULP value for both precisions, i.e., double
433 * precision will have much stricter tolerance.
435 * \related FloatingPointTolerance
437 static inline FloatingPointTolerance ulpTolerance(uint64_t ulpDiff)
439 return FloatingPointTolerance(0.0, 0.0, 0.0, 0.0, ulpDiff, ulpDiff, false);
443 * Creates a tolerance that allows a difference in two compared values that is
444 * relative to the given magnitude.
446 * \param[in] magnitude Magnitude of the numbers the computation operates in.
447 * \param[in] tolerance Relative tolerance permitted (e.g. 1e-4).
449 * In addition to setting an relative tolerance for both
450 * precisions, this sets the absolute tolerance such that values close to zero
451 * (in general, smaller than \p magnitude) do not fail the check if they
452 * differ by less than \p tolerance evaluated at \p magnitude. This accounts
453 * for potential loss of precision for small values, and should be used when
454 * accuracy of values much less than \p magnitude do not matter for
457 * \related FloatingPointTolerance
459 FloatingPointTolerance relativeToleranceAsFloatingPoint(double magnitude, double tolerance);
462 * Creates a tolerance that allows a precision-dependent difference in two
463 * compared values that is relative to the given magnitude.
465 * \param[in] magnitude Magnitude of the numbers the computation
467 * \param[in] singleTolerance Relative tolerance permitted (e.g. 1e-4)
468 * in single precision.
469 * \param[in] doubleTolerance Relative tolerance permitted (e.g. 1e-4)
470 * in double precision.
472 * In addition to setting an relative tolerance for both
473 * precisions, this sets the absolute tolerance such that values close to zero
474 * (in general, smaller than \p magnitude) do not fail the check if they
475 * differ by less than \p tolerance evaluated at \p magnitude. This accounts
476 * for potential loss of precision for small values, and should be used when
477 * accuracy of values much less than \p magnitude do not matter for
480 * \related FloatingPointTolerance
482 FloatingPointTolerance relativeToleranceAsPrecisionDependentFloatingPoint(double magnitude,
483 float singleTolerance,
484 double doubleTolerance);
487 * Creates a tolerance that allows a precision-dependent relative difference in
488 * a complex computation.
490 * \param[in] magnitude Magnitude of the numbers the computation operates in.
491 * \param[in] singleUlpDiff Expected accuracy of single-precision
492 * computation (in ULPs).
493 * \param[in] doubleUlpDiff Expected accuracy of double-precision
494 * computation (in ULPs).
496 * This works as relativeToleranceAsUlp(), but allows setting the ULP
497 * difference separately for the different precisions. This supports
498 * cases where the double-precision calculation can acceptably has a higher ULP
499 * difference, but relaxing the single-precision tolerance would lead to an
500 * unnecessarily loose test.
502 * \related FloatingPointTolerance
504 static inline FloatingPointTolerance relativeToleranceAsPrecisionDependentUlp(double magnitude,
505 uint64_t singleUlpDiff,
506 uint64_t doubleUlpDiff)
508 return FloatingPointTolerance(float(magnitude) * singleUlpDiff * GMX_FLOAT_EPS,
509 magnitude * doubleUlpDiff * GMX_DOUBLE_EPS, 0.0, 0.0,
510 singleUlpDiff, doubleUlpDiff, false);
514 * Creates a tolerance that allows a specified absolute difference.
516 * \related FloatingPointTolerance
518 static inline FloatingPointTolerance absoluteTolerance(double tolerance)
520 return FloatingPointTolerance(float(tolerance), tolerance, 0.0, 0.0, UINT64_MAX, UINT64_MAX, false);
524 * Creates a tolerance that allows a relative difference in a complex
527 * \param[in] magnitude Magnitude of the numbers the computation operates in.
528 * \param[in] ulpDiff Expected accuracy of the computation (in ULPs).
530 * In addition to setting the ULP tolerance as ulpTolerance(), this sets the
531 * absolute tolerance such that values close to zero (in general, smaller than
532 * \p magnitude) do not fail the check if they differ by less than \p ulpDiff
533 * evaluated at \p magnitude. This accounts for potential loss of precision
534 * for small values, and should be used when accuracy of values much less than
535 * \p magnitude do not matter for correctness.
537 * \related FloatingPointTolerance
539 static inline FloatingPointTolerance relativeToleranceAsUlp(double magnitude, uint64_t ulpDiff)
541 return relativeToleranceAsPrecisionDependentUlp(magnitude, ulpDiff, ulpDiff);
546 //! Default tolerance in ULPs for two floating-point values to compare equal.
547 constexpr uint64_t g_defaultUlpTolerance = 4;
548 } // namespace detail
551 * Returns the default tolerance for comparing `real` numbers.
553 * \related FloatingPointTolerance
555 static inline FloatingPointTolerance defaultRealTolerance()
557 return relativeToleranceAsUlp(1.0, detail::g_defaultUlpTolerance);
562 * Returns the default tolerance for comparing single-precision numbers when
563 * compared by \Gromacs built in either precision mode.
565 * This permits a checker compiled with any \Gromacs precision to compare
566 * equal or not in the same way.
568 * \related FloatingPointTolerance
570 static inline FloatingPointTolerance defaultFloatTolerance()
572 return relativeToleranceAsPrecisionDependentUlp(
573 1.0, detail::g_defaultUlpTolerance,
574 static_cast<uint64_t>(detail::g_defaultUlpTolerance * (GMX_FLOAT_EPS / GMX_DOUBLE_EPS)));
577 /*! \name Assertions for floating-point comparison
579 * These routines extend `(EXPECT|ASSERT)_(FLOAT|DOUBLE)_EQ` and
580 * `(EXPECT|ASSERT)_NEAR` from Google Test to provide more flexible assertions
581 * for floating-point values.
583 * See gmx::test::FloatingPointTolerance for the possible ways to specify the
584 * tolerance, and gmx::test::FloatingPointDifference for some additional
585 * details of the difference calculation.
591 * Assertion predicate formatter for comparing two floating-point values.
593 template<typename FloatType>
594 static inline ::testing::AssertionResult assertEqualWithinTolerance(const char* expr1,
596 const char* /*exprTolerance*/,
599 const FloatingPointTolerance& tolerance)
601 FloatingPointDifference diff(value1, value2);
602 if (tolerance.isWithin(diff))
604 return ::testing::AssertionSuccess();
606 return ::testing::AssertionFailure() << " Value of: " << expr2 << std::endl
607 << " Actual: " << value2 << std::endl
608 << " Expected: " << expr1 << std::endl
609 << " Which is: " << value1 << std::endl
610 << "Difference: " << diff.toString() << std::endl
611 << " Tolerance: " << tolerance.toString(diff);
616 * Asserts that two single-precision values are within the given tolerance.
620 #define EXPECT_FLOAT_EQ_TOL(value1, value2, tolerance) \
621 EXPECT_PRED_FORMAT3(::gmx::test::assertEqualWithinTolerance<float>, value1, value2, tolerance)
623 * Asserts that two double-precision values are within the given tolerance.
627 #define EXPECT_DOUBLE_EQ_TOL(value1, value2, tolerance) \
628 EXPECT_PRED_FORMAT3(::gmx::test::assertEqualWithinTolerance<double>, value1, value2, tolerance)
629 /*! \def EXPECT_REAL_EQ_TOL
631 * Asserts that two `real` values are within the given tolerance.
636 * Asserts that two single-precision values are within the given tolerance.
640 #define ASSERT_FLOAT_EQ_TOL(value1, value2, tolerance) \
641 ASSERT_PRED_FORMAT3(::gmx::test::assertEqualWithinTolerance<float>, value1, value2, tolerance)
643 * Asserts that two double-precision values are within the given tolerance.
647 #define ASSERT_DOUBLE_EQ_TOL(value1, value2, tolerance) \
648 ASSERT_PRED_FORMAT3(::gmx::test::assertEqualWithinTolerance<double>, value1, value2, tolerance)
649 /*! \def ASSERT_REAL_EQ_TOL
651 * Asserts that two `real` values are within the given tolerance.
657 # define EXPECT_REAL_EQ_TOL(value1, value2, tolerance) \
658 EXPECT_DOUBLE_EQ_TOL(value1, value2, tolerance)
659 # define ASSERT_REAL_EQ_TOL(value1, value2, tolerance) \
660 ASSERT_DOUBLE_EQ_TOL(value1, value2, tolerance)
662 # define EXPECT_REAL_EQ_TOL(value1, value2, tolerance) \
663 EXPECT_FLOAT_EQ_TOL(value1, value2, tolerance)
664 # define ASSERT_REAL_EQ_TOL(value1, value2, tolerance) \
665 ASSERT_FLOAT_EQ_TOL(value1, value2, tolerance)
668 //! EXPECT_REAL_EQ_TOL with default tolerance
669 #define EXPECT_REAL_EQ(value1, value2) \
670 EXPECT_REAL_EQ_TOL(value1, value2, ::gmx::test::defaultRealTolerance())
671 //! ASSERT_REAL_EQ_TOL with default tolerance
672 #define ASSERT_REAL_EQ(value1, value2) \
673 ASSERT_REAL_EQ_TOL(value1, value2, ::gmx::test::defaultRealTolerance())
678 * Helper method for `(EXPECT|ASSERT)_PLAIN`.
680 static inline ::testing::AssertionResult plainAssertHelper(const char* /*expr*/,
681 const ::testing::AssertionResult& expr)
688 * Assert for predicates that return AssertionResult and produce a full failure
691 * `expr` should evaluate to AssertionResult, and on failure the message from
692 * the result is used as-is, unlike in EXPECT_TRUE().
696 #define EXPECT_PLAIN(expr) EXPECT_PRED_FORMAT1(plainAssertHelper, expr)
698 * Assert for predicates that return AssertionResult and produce a full failure
701 * `expr` should evaluate to AssertionResult, and on failure the message from
702 * the result is used as-is, unlike in ASSERT_TRUE().
706 #define ASSERT_PLAIN(expr) ASSERT_PRED_FORMAT1(plainAssertHelper, expr)