2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2011-2018, The GROMACS development team.
5 * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
6 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 * and including many others, as listed in the AUTHORS file in the
8 * top-level source directory and at http://www.gromacs.org.
10 * GROMACS is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public License
12 * as published by the Free Software Foundation; either version 2.1
13 * of the License, or (at your option) any later version.
15 * GROMACS is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with GROMACS; if not, see
22 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 * If you want to redistribute modifications to GROMACS, please
26 * consider that scientific software is very special. Version
27 * control is crucial - bugs must be traceable. We will be happy to
28 * consider code for inclusion in the official distribution, but
29 * derived work must not be called official GROMACS. Details are found
30 * in the README & COPYING files - if they are missing, get the
31 * official version at http://www.gromacs.org.
33 * To help us fund GROMACS development, we humbly ask that you cite
34 * the research papers on the package. Check out http://www.gromacs.org.
36 /*! \libinternal \file
38 * Functionality for writing tests that can produce their own reference data.
40 * See \ref page_refdata for more details.
42 * \author Teemu Murtola <teemu.murtola@gmail.com>
43 * \author Mark Abraham <mark.j.abraham@gmail.com>
45 * \ingroup module_testutils
47 #ifndef GMX_TESTUTILS_REFDATA_H
48 #define GMX_TESTUTILS_REFDATA_H
54 #include "gromacs/utility/basedefinitions.h"
55 #include "gromacs/utility/classhelpers.h"
60 class IOptionsContainer;
61 class KeyValueTreeObject;
62 class KeyValueTreeValue;
65 template<typename ValueType>
71 class FloatingPointTolerance;
73 /*! \libinternal \brief
74 * Mode of operation for reference data handling.
76 * There should be no need to use this type outside the test utility module.
78 * \ingroup module_testutils
80 enum class ReferenceDataMode : int
83 * Compare to existing reference data.
85 * If reference data does not exist, or if the test results differ from
86 * those in the reference data, the test fails.
90 * Create missing reference data.
92 * If reference data does not exist for a test, that test behaves as if
93 * ReferenceDataMode::UpdateAll had been specified. Tests for which reference data
94 * exists, behave like with ReferenceDataMode::Compare.
98 * Update reference data that does not pass comparison.
100 * Tests utilizing reference data should always pass in this mode unless
101 * there is an I/O error.
105 * Update reference data, overwriting old data.
107 * Tests utilizing reference data should always pass in this mode unless
108 * there is an I/O error.
111 //! Marks the end of the enum
115 /*! \libinternal \brief
116 * Initializes reference data handling.
118 * Adds command-line options to \p options to set the reference data mode.
119 * By default, ReferenceDataMode::Compare is used, but ``--ref-data create`` or
120 * ``--ref-data update`` can be used to change it.
122 * This function is automatically called by initTestUtils().
124 * \ingroup module_testutils
126 void initReferenceData(IOptionsContainer* options);
128 class TestReferenceChecker;
132 class TestReferenceDataImpl;
135 /*! \libinternal \brief
136 * Handles creation of and comparison to test reference data.
138 * See \ref page_refdata for an overview of the functionality.
140 * This class provides functionality to use the same code to generate reference
141 * data and then on later runs compare the results of the code against that
142 * reference. The mode in which the class operates (writing reference data or
143 * comparing against existing data) is set using a command-line option that
144 * is automatically managed when using the testutils module to implement tests.
145 * Tests only need to create an instance of TestReferenceData, obtain a
146 * TestReferenceChecker using the rootChecker() method and use the various
147 * check*() methods in TestReferenceChecker to indicate values to check. If
148 * the test is running in reference data creation mode, it will produce an XML
149 * file with the values recorder. In comparison mode, it will read that same
150 * XML file and produce a Google Test non-fatal assertion for every discrepancy
151 * it detects with the reference data (including missing reference data file or
152 * individual item). Exceptions derived from TestException are thrown for I/O
153 * errors and syntax errors in the reference data.
155 * Simple example (using Google Test):
157 int functionToTest(int param);
163 TEST(MyTest, SimpleTest)
165 TestReferenceData data;
167 TestReferenceChecker checker(data.rootChecker());
168 checker.checkInteger(functionToTest(3), "ValueWith3");
169 checker.checkInteger(functionToTest(5), "ValueWith5");
170 TestReferenceChecker compound(
171 checker.checkCompound("CustomCompound", "Item"));
172 compound.checkInteger(function2ToTest(3), "ValueWith3");
173 compound.checkInteger(function2ToTest(5), "ValueWith5");
174 checker.checkInteger(functionToTest(4), "ValueWith4");
175 checker.checkVector(functionProducingRVec(), "Describe The RVec");
181 * If rootChecker() is never called, no comparison is done (i.e., missing
182 * reference data file is not reported as an error, nor is empty reference data
183 * file created in write mode).
185 * For floating-point comparisons, the reference data should be generated in
186 * double precision (currently, no warning is provided even if this is not the
187 * case, but the double precision tests will then very likely fail).
190 * \ingroup module_testutils
192 class TestReferenceData
196 * Initializes the reference data in the global mode.
199 /*! \brief Initializes the reference data in the global mode while
200 * providing the test name.
202 * Providing a name for the test, rather than using the default
203 * name generated by GoogleTest, is useful when many tests should
204 * share common reference data because e.g. they are
205 * value-parameterized over multiple equivalent implementations or
206 * run modes. The name should include a filename suffix
207 * e.g. `.xml` and refers to a file that will be found in the
208 * `refdata` subdirectory for the test case. */
209 explicit TestReferenceData(std::string testNameOverride);
211 * Initializes the reference data in a specific mode.
213 * This function is only useful for self-testing the reference data
214 * framework. As such, it also puts the framework in a state where it
215 * logs additional internal information for failures to help diagnosing
216 * problems in the framework, and stores the reference data in a
217 * temporary directory instead of the source tree.
218 * The default constructor should be used in tests utilizing this class.
220 explicit TestReferenceData(ReferenceDataMode mode);
222 * Frees reference data structures.
224 * The reference data is written out if necessary automatically when
227 ~TestReferenceData();
230 * Returns a root-level checker object for comparisons.
232 * Each call returns an independent instance.
234 TestReferenceChecker rootChecker();
237 std::shared_ptr<internal::TestReferenceDataImpl> impl_;
239 GMX_DISALLOW_COPY_AND_ASSIGN(TestReferenceData);
242 /*! \libinternal \brief
243 * Handles comparison to test reference data.
245 * Every check*() method takes an id string as th
246 * e last parameter. This id is
247 * used to uniquely identify the value in the reference data, and it makes the
248 * output XML more human-friendly and more robust to errors. The id can be
249 * NULL; in this case, multiple elements with no id are created, and they will
250 * be matched in the same order as in which they are created. The
251 * checkCompound() method can be used to create a set of reference values
252 * grouped together. In this case, all check*() calls using the returned child
253 * TestReferenceChecker object will create the reference data within this
254 * group, and the ids only need to be unique within the compound. Compounds
257 * For usage example, see TestReferenceData.
259 * Copies of this class behave have independent internal state.
262 * \ingroup module_testutils
264 class TestReferenceChecker
268 * Creates a checker that cannot be used for checking.
270 * Attempting to call the check methods generates an assert.
271 * It is possible to check whether the checker is initialized by
273 * This constructor exists to allow declaring checker variables that
274 * will receive their value later without resorting to dynamic
277 TestReferenceChecker();
278 //! Creates a deep copy of the other checker.
279 explicit TestReferenceChecker(const TestReferenceChecker& other);
280 //! Moves the checker.
281 TestReferenceChecker(TestReferenceChecker&& other) noexcept;
282 ~TestReferenceChecker();
284 //! Prevents implicit copying during assignment.
285 TestReferenceChecker& operator=(const TestReferenceChecker&) = delete;
286 //! Assigns a test reference checker.
287 TestReferenceChecker& operator=(TestReferenceChecker&& other) noexcept;
289 //! Returns whether the checker is initialized.
290 bool isValid() const;
291 //! Allows testing whether the checker is initialized directly with if.
292 explicit operator bool() const { return isValid(); }
295 * Sets the tolerance for floating-point comparisons.
297 * All following floating-point comparisons using this checker will use
298 * the new tolerance. Child checkers created with checkCompound()
299 * will inherit the tolerance from their parent checker at the time
300 * checkCompound() is called.
304 void setDefaultTolerance(const FloatingPointTolerance& tolerance);
307 * Checks that all reference values have been compared against.
309 * All values under the compound represented by this checker are
310 * checked, and a non-fatal Google Test assertion is produced if some
311 * values have not been used.
313 * If not called explicitly, the same check will be done for all
314 * reference data values when the test ends.
316 * This method also marks the values used, so that subsequent checks
317 * (including the check at the end of the test) will not produce
318 * another assertion about the same values.
320 void checkUnusedEntries();
322 /*! \brief Disables checking for unused entries
324 * \see checkUnusedEntries()
326 void disableUnusedEntriesCheck();
329 * Checks whether a data item is present.
331 * \param[in] bPresent Whether to check for presence or absence.
332 * \param[in] id Unique identifier of the item to check.
333 * \returns true if bPresent was true and the data item was found.
335 * If \p bPresent is true, checks that a data item with \p id is
336 * present, otherwise checks that the data item is absent.
337 * If the check fails, a non-fatal Google Test assertion is generated.
339 * If reference data is being written, the check always succeeds and the
340 * return value is \p bPresent.
342 * The main use of this method is to assign meaning for missing
343 * reference data. Example use:
345 if (checker.checkPresent(bHaveVelocities, "Velocities"))
347 // <check the velocities>
351 bool checkPresent(bool bPresent, const char* id);
354 * Initializes comparison of a group of related data items.
356 * \param[in] type Informational type for the compound.
357 * \param[in] id Unique identifier for the compound among its
359 * \returns Checker to use for comparison within the compound.
361 * All checks performed with the returned checker only
362 * need to have unique ids within the compound, not globally.
364 * Compound structures can be nested.
366 TestReferenceChecker checkCompound(const char* type, const char* id);
367 //! \copydoc checkCompound(const char *, const char *)
368 TestReferenceChecker checkCompound(const char* type, const std::string& id);
370 //! Check a single boolean value.
371 void checkBoolean(bool value, const char* id);
372 //! Check a single string value.
373 void checkString(const char* value, const char* id);
374 //! Check a single string value.
375 void checkString(const std::string& value, const char* id);
377 * Check a multi-line string value.
379 * This method works as checkString(), but should be used for long
380 * strings that may contain, e.g., newlines. Typically used to check
381 * formatted output, and attempts to make the output XML such that it
382 * is easier to edit by hand to set the desired output formatting.
384 void checkTextBlock(const std::string& value, const char* id);
385 //! Check a single unsigned char value.
386 void checkUChar(unsigned char value, const char* id);
387 //! Check a single integer value.
388 void checkInteger(int value, const char* id);
389 //! Check a single int32 value.
390 void checkInt32(int32_t value, const char* id);
391 //! Check a single uint32 value.
392 void checkUInt32(uint32_t value, const char* id);
393 //! Check a single int64 value.
394 void checkInt64(int64_t value, const char* id);
395 //! Check a single uint64 value.
396 void checkUInt64(uint64_t value, const char* id);
397 //! Check a single single-precision floating point value.
398 void checkFloat(float value, const char* id);
399 //! Check a single double-precision floating point value.
400 void checkDouble(double value, const char* id);
401 //! Check a single floating point value.
402 void checkReal(float value, const char* id);
403 //! Check a single floating point value.
404 void checkReal(double value, const char* id);
405 //! Check a vector of three integer values.
406 void checkVector(const int value[3], const char* id);
407 //! Check a vector of three single-precision floating point values.
408 void checkVector(const float value[3], const char* id);
409 //! Check a vector of three double-precision floating point values.
410 void checkVector(const double value[3], const char* id);
411 //! Check a BasicVector of ints, ie. IVec
412 void checkVector(const BasicVector<int>& value, const char* id);
413 //! Check a BasicVector of floats, ie. RVec
414 void checkVector(const BasicVector<float>& value, const char* id);
415 //! Check a BasicVector of doubles, ie. DVec
416 void checkVector(const BasicVector<double>& value, const char* id);
417 //! Check a single floating-point value from a string.
418 void checkRealFromString(const std::string& value, const char* id);
419 //! Checks a any value that contains a supported simple type.
420 void checkAny(const Any& value, const char* id);
421 //! Checks a key-value tree rooted at a object.
422 void checkKeyValueTreeObject(const KeyValueTreeObject& tree, const char* id);
423 //! Checks a generic key-value tree value.
424 void checkKeyValueTreeValue(const KeyValueTreeValue& value, const char* id);
426 /*! \name Methods to read values from reference data
428 * These methods assume that a value with the given `id` has already
429 * been created in the test with `check*()` methods, and that it has
432 * Currently, these methods do not work correctly if the reference data
433 * file does not exist, so a test using them may fail with exceptions
434 * before the reference data has been generated.
437 //! Reads an unsigned char value.
438 unsigned char readUChar(const char* id);
439 //! Reads an integer value.
440 int readInteger(const char* id);
441 //! Reads a 32-bit integer value.
442 int32_t readInt32(const char* id);
443 //! Reads a 64-bit integer value.
444 int64_t readInt64(const char* id);
445 //! Reads a float value.
446 float readFloat(const char* id);
447 //! Reads a double value.
448 double readDouble(const char* id);
449 //! Reads a string value.
450 std::string readString(const char* id);
453 /*! \name Overloaded versions of simple checker methods
455 * These methods provide overloads under a single name for all the
456 * methods checkBoolean(), checkString(), checkReal() and checkVector().
457 * They are provided mainly to allow template implementations (such as
458 * checkSequence()). Typically callers should use the individually
459 * named versions for greater clarity.
462 //! Check a single boolean value.
463 void checkValue(bool value, const char* id) { checkBoolean(value, id); }
464 //! Check a single string value.
465 void checkValue(const char* value, const char* id) { checkString(value, id); }
466 //! Check a single string value.
467 void checkValue(const std::string& value, const char* id) { checkString(value, id); }
468 //! Check a single signed integer value
469 void checkValue(int value, const char* id) { checkInteger(value, id); }
470 //! Check a single signed integer value of width 64 bits.
471 void checkValue(int64_t value, const char* id) { checkInt64(value, id); }
472 //! Check a single unsigned integer value of width 64 bits.
473 void checkValue(uint64_t value, const char* id) { checkUInt64(value, id); }
474 //! Check a single single-precision floating point value.
475 void checkValue(float value, const char* id) { checkFloat(value, id); }
476 //! Check a single double-precision floating point value.
477 void checkValue(double value, const char* id) { checkDouble(value, id); }
478 //! Check a vector of three integer values.
479 void checkValue(const int value[3], const char* id) { checkVector(value, id); }
480 //! Check a vector of three single-precision floating point values.
481 void checkValue(const float value[3], const char* id) { checkVector(value, id); }
482 //! Check a vector of three double-precision floating point values.
483 void checkValue(const double value[3], const char* id) { checkVector(value, id); }
484 //! Check a BasicVector of integer values, ie. IVec.
485 void checkValue(const BasicVector<int>& value, const char* id) { checkVector(value, id); }
486 //! Check a BasicVector of float values, ie. RVec.
487 void checkValue(const BasicVector<float>& value, const char* id) { checkVector(value, id); }
488 //! Check a BasicVector of double values, ie. DVec.
489 void checkValue(const BasicVector<double>& value, const char* id) { checkVector(value, id); }
490 //! Check a generic key-value tree value.
491 void checkValue(const KeyValueTreeValue& value, const char* id)
493 checkKeyValueTreeValue(value, id);
498 * Generic method to check a sequence of simple values.
500 * \tparam Iterator Input iterator that allows multiple (two) passes.
501 * Value type must be one of those accepted by checkValue(), or
502 * implicitly convertible to one.
503 * \param[in] begin Iterator to the start of the range to check.
504 * \param[in] end Iterator to the end of the range to check.
505 * \param[in] id Unique identifier for the sequence among its
508 template<class Iterator>
509 void checkSequence(Iterator begin, Iterator end, const char* id)
511 typename std::iterator_traits<Iterator>::difference_type length = std::distance(begin, end);
512 TestReferenceChecker compound(checkSequenceCompound(id, length));
513 for (Iterator i = begin; i != end; ++i)
515 compound.checkValue(*i, nullptr);
519 * Generic method to check a sequence of custom values.
521 * \tparam Iterator Input iterator that allows multiple (two) passes.
522 * \tparam ItemChecker Functor to check an individual value. Signature
523 * void(TestReferenceChecker *, const T &), where T is the value
524 * type of \p Iterator.
525 * \param[in] begin Iterator to the start of the range to check.
526 * \param[in] end Iterator to the end of the range to check.
527 * \param[in] id Unique identifier for the sequence among its
529 * \param[in] checkItem Functor to check an individual item.
531 * This method creates a compound checker \c compound within which all
532 * values of the sequence are checked. Calls \c checkItem(&compound, *i)
533 * with that compound for each iterator \c i in the range [begin, end).
534 * \p checkItem should use the various check methods in the passed
535 * checker to check each value.
537 * This method can be used to check a sequence made of compound types.
538 * Typically \p checkItem will create a compound within the passed
539 * checker to check different aspects of the value that was passed
540 * to it. Either NULL or a unique identifier string must be used for
541 * the id value of that compound. */
542 template<class Iterator, class ItemChecker>
543 void checkSequence(Iterator begin, Iterator end, const char* id, ItemChecker checkItem)
545 typename std::iterator_traits<Iterator>::difference_type length = std::distance(begin, end);
546 TestReferenceChecker compound(checkSequenceCompound(id, length));
547 for (Iterator i = begin; i != end; ++i)
549 checkItem(&compound, *i);
553 * Check an array of values.
555 * \tparam T Type of values to check. Should be one of those accepted
556 * by checkValue(), or implicitly convertible to one.
558 * \param[in] length Number of values to check.
559 * \param[in] values Pointer to the first value to check.
560 * \param[in] id Unique identifier for the sequence among its
563 * This is a convenience method that delegates all work to
567 void checkSequenceArray(size_t length, const T* values, const char* id)
569 checkSequence(values, values + length, id);
572 * Convenience method for checking that a sequence is empty.
574 * \param[in] id Unique identifier for the sequence among its
577 * This method provides a convenient solution for a case where there is
578 * implicitly a sequence to be checked, but there is no pointer
579 * available to the values since the sequence is empty.
580 * Since this method does not require the type of the values, it can be
581 * used in such cases easily.
583 void checkEmptySequence(const char* id);
585 * Initializes a compound for a sequence of items.
587 * \param[in] id Unique identifier for the sequence among its
589 * \param[in] length Number of items that will be in the sequence.
590 * \returns Checker to use for comparison within the sequence.
592 * This method can be used to check custom sequences where
593 * checkSequence() is not appropriate.
595 TestReferenceChecker checkSequenceCompound(const char* id, size_t length);
601 * Constructs a checker with a specific internal state.
603 * Is private to only allow users of this class to create instances
604 * using TestReferenceData::rootChecker() or checkCompound()
607 explicit TestReferenceChecker(Impl* impl);
609 std::unique_ptr<Impl> impl_;
612 * Needed to expose the constructor only to TestReferenceData.
614 friend class TestReferenceData;