2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2011-2018, The GROMACS development team.
5 * Copyright (c) 2019,2020,2021, by the GROMACS development team, led by
6 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 * and including many others, as listed in the AUTHORS file in the
8 * top-level source directory and at http://www.gromacs.org.
10 * GROMACS is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public License
12 * as published by the Free Software Foundation; either version 2.1
13 * of the License, or (at your option) any later version.
15 * GROMACS is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with GROMACS; if not, see
22 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 * If you want to redistribute modifications to GROMACS, please
26 * consider that scientific software is very special. Version
27 * control is crucial - bugs must be traceable. We will be happy to
28 * consider code for inclusion in the official distribution, but
29 * derived work must not be called official GROMACS. Details are found
30 * in the README & COPYING files - if they are missing, get the
31 * official version at http://www.gromacs.org.
33 * To help us fund GROMACS development, we humbly ask that you cite
34 * the research papers on the package. Check out http://www.gromacs.org.
36 /*! \libinternal \file
38 * Functionality for writing tests that can produce their own reference data.
40 * See \ref page_refdata for more details.
42 * \author Teemu Murtola <teemu.murtola@gmail.com>
44 * \ingroup module_testutils
46 #ifndef GMX_TESTUTILS_REFDATA_H
47 #define GMX_TESTUTILS_REFDATA_H
53 #include "gromacs/utility/basedefinitions.h"
54 #include "gromacs/utility/classhelpers.h"
59 class IOptionsContainer;
60 class KeyValueTreeObject;
61 class KeyValueTreeValue;
64 template<typename ValueType>
70 class FloatingPointTolerance;
72 /*! \libinternal \brief
73 * Mode of operation for reference data handling.
75 * There should be no need to use this type outside the test utility module.
77 * \ingroup module_testutils
79 enum class ReferenceDataMode : int
82 * Compare to existing reference data.
84 * If reference data does not exist, or if the test results differ from
85 * those in the reference data, the test fails.
89 * Create missing reference data.
91 * If reference data does not exist for a test, that test behaves as if
92 * ReferenceDataMode::UpdateAll had been specified. Tests for which reference data
93 * exists, behave like with ReferenceDataMode::Compare.
97 * Update reference data that does not pass comparison.
99 * Tests utilizing reference data should always pass in this mode unless
100 * there is an I/O error.
104 * Update reference data, overwriting old data.
106 * Tests utilizing reference data should always pass in this mode unless
107 * there is an I/O error.
110 //! Marks the end of the enum
114 /*! \libinternal \brief
115 * Initializes reference data handling.
117 * Adds command-line options to \p options to set the reference data mode.
118 * By default, ReferenceDataMode::Compare is used, but ``--ref-data create`` or
119 * ``--ref-data update`` can be used to change it.
121 * This function is automatically called by initTestUtils().
123 * \ingroup module_testutils
125 void initReferenceData(IOptionsContainer* options);
127 class TestReferenceChecker;
131 class TestReferenceDataImpl;
134 /*! \libinternal \brief
135 * Handles creation of and comparison to test reference data.
137 * See \ref page_refdata for an overview of the functionality.
139 * This class provides functionality to use the same code to generate reference
140 * data and then on later runs compare the results of the code against that
141 * reference. The mode in which the class operates (writing reference data or
142 * comparing against existing data) is set using a command-line option that
143 * is automatically managed when using the testutils module to implement tests.
144 * Tests only need to create an instance of TestReferenceData, obtain a
145 * TestReferenceChecker using the rootChecker() method and use the various
146 * check*() methods in TestReferenceChecker to indicate values to check. If
147 * the test is running in reference data creation mode, it will produce an XML
148 * file with the values recorder. In comparison mode, it will read that same
149 * XML file and produce a Google Test non-fatal assertion for every discrepancy
150 * it detects with the reference data (including missing reference data file or
151 * individual item). Exceptions derived from TestException are thrown for I/O
152 * errors and syntax errors in the reference data.
154 * Simple example (using Google Test):
156 int functionToTest(int param);
162 TEST(MyTest, SimpleTest)
164 TestReferenceData data;
166 TestReferenceChecker checker(data.rootChecker());
167 checker.checkInteger(functionToTest(3), "ValueWith3");
168 checker.checkInteger(functionToTest(5), "ValueWith5");
169 TestReferenceChecker compound(
170 checker.checkCompound("CustomCompound", "Item"));
171 compound.checkInteger(function2ToTest(3), "ValueWith3");
172 compound.checkInteger(function2ToTest(5), "ValueWith5");
173 checker.checkInteger(functionToTest(4), "ValueWith4");
174 checker.checkVector(functionProducingRVec(), "Describe The RVec");
180 * If rootChecker() is never called, no comparison is done (i.e., missing
181 * reference data file is not reported as an error, nor is empty reference data
182 * file created in write mode).
184 * For floating-point comparisons, the reference data should be generated in
185 * double precision (currently, no warning is provided even if this is not the
186 * case, but the double precision tests will then very likely fail).
189 * \ingroup module_testutils
191 class TestReferenceData
195 * Initializes the reference data in the global mode.
199 * Initializes the reference data in a specific mode.
201 * This function is only useful for self-testing the reference data
202 * framework. As such, it also puts the framework in a state where it
203 * logs additional internal information for failures to help diagnosing
204 * problems in the framework, and stores the reference data in a
205 * temporary directory instead of the source tree.
206 * The default constructor should be used in tests utilizing this class.
208 explicit TestReferenceData(ReferenceDataMode mode);
210 * Frees reference data structures.
212 * The reference data is written out if necessary automatically when
215 ~TestReferenceData();
218 * Returns a root-level checker object for comparisons.
220 * Each call returns an independent instance.
222 TestReferenceChecker rootChecker();
225 std::shared_ptr<internal::TestReferenceDataImpl> impl_;
227 GMX_DISALLOW_COPY_AND_ASSIGN(TestReferenceData);
230 /*! \libinternal \brief
231 * Handles comparison to test reference data.
233 * Every check*() method takes an id string as th
234 * e last parameter. This id is
235 * used to uniquely identify the value in the reference data, and it makes the
236 * output XML more human-friendly and more robust to errors. The id can be
237 * NULL; in this case, multiple elements with no id are created, and they will
238 * be matched in the same order as in which they are created. The
239 * checkCompound() method can be used to create a set of reference values
240 * grouped together. In this case, all check*() calls using the returned child
241 * TestReferenceChecker object will create the reference data within this
242 * group, and the ids only need to be unique within the compound. Compounds
245 * For usage example, see TestReferenceData.
247 * Copies of this class behave have independent internal state.
250 * \ingroup module_testutils
252 class TestReferenceChecker
256 * Creates a checker that cannot be used for checking.
258 * Attempting to call the check methods generates an assert.
259 * It is possible to check whether the checker is initialized by
261 * This constructor exists to allow declaring checker variables that
262 * will receive their value later without resorting to dynamic
265 TestReferenceChecker();
266 //! Creates a deep copy of the other checker.
267 explicit TestReferenceChecker(const TestReferenceChecker& other);
268 //! Moves the checker.
269 TestReferenceChecker(TestReferenceChecker&& other) noexcept;
270 ~TestReferenceChecker();
272 //! Prevents implicit copying during assignment.
273 TestReferenceChecker& operator=(const TestReferenceChecker&) = delete;
274 //! Assigns a test reference checker.
275 TestReferenceChecker& operator=(TestReferenceChecker&& other) noexcept;
277 //! Returns whether the checker is initialized.
278 bool isValid() const;
279 //! Allows testing whether the checker is initialized directly with if.
280 explicit operator bool() const { return isValid(); }
283 * Sets the tolerance for floating-point comparisons.
285 * All following floating-point comparisons using this checker will use
286 * the new tolerance. Child checkers created with checkCompound()
287 * will inherit the tolerance from their parent checker at the time
288 * checkCompound() is called.
292 void setDefaultTolerance(const FloatingPointTolerance& tolerance);
295 * Checks that all reference values have been compared against.
297 * All values under the compound represented by this checker are
298 * checked, and a non-fatal Google Test assertion is produced if some
299 * values have not been used.
301 * If not called explicitly, the same check will be done for all
302 * reference data values when the test ends.
304 * This method also marks the values used, so that subsequent checks
305 * (including the check at the end of the test) will not produce
306 * another assertion about the same values.
308 void checkUnusedEntries();
310 /*! \brief Disables checking for unused entries
312 * \see checkUnusedEntries()
314 void disableUnusedEntriesCheck();
317 * Checks whether a data item is present.
319 * \param[in] bPresent Whether to check for presence or absence.
320 * \param[in] id Unique identifier of the item to check.
321 * \returns true if bPresent was true and the data item was found.
323 * If \p bPresent is true, checks that a data item with \p id is
324 * present, otherwise checks that the data item is absent.
325 * If the check fails, a non-fatal Google Test assertion is generated.
327 * If reference data is being written, the check always succeeds and the
328 * return value is \p bPresent.
330 * The main use of this method is to assign meaning for missing
331 * reference data. Example use:
333 if (checker.checkPresent(bHaveVelocities, "Velocities"))
335 // <check the velocities>
339 bool checkPresent(bool bPresent, const char* id);
342 * Initializes comparison of a group of related data items.
344 * \param[in] type Informational type for the compound.
345 * \param[in] id Unique identifier for the compound among its
347 * \returns Checker to use for comparison within the compound.
349 * All checks performed with the returned checker only
350 * need to have unique ids within the compound, not globally.
352 * Compound structures can be nested.
354 TestReferenceChecker checkCompound(const char* type, const char* id);
355 //! \copydoc checkCompound(const char *, const char *)
356 TestReferenceChecker checkCompound(const char* type, const std::string& id);
358 //! Check a single boolean value.
359 void checkBoolean(bool value, const char* id);
360 //! Check a single string value.
361 void checkString(const char* value, const char* id);
362 //! Check a single string value.
363 void checkString(const std::string& value, const char* id);
365 * Check a multi-line string value.
367 * This method works as checkString(), but should be used for long
368 * strings that may contain, e.g., newlines. Typically used to check
369 * formatted output, and attempts to make the output XML such that it
370 * is easier to edit by hand to set the desired output formatting.
372 void checkTextBlock(const std::string& value, const char* id);
373 //! Check a single unsigned char value.
374 void checkUChar(unsigned char value, const char* id);
375 //! Check a single integer value.
376 void checkInteger(int value, const char* id);
377 //! Check a single int32 value.
378 void checkInt32(int32_t value, const char* id);
379 //! Check a single uint32 value.
380 void checkUInt32(uint32_t value, const char* id);
381 //! Check a single int64 value.
382 void checkInt64(int64_t value, const char* id);
383 //! Check a single uint64 value.
384 void checkUInt64(uint64_t value, const char* id);
385 //! Check a single single-precision floating point value.
386 void checkFloat(float value, const char* id);
387 //! Check a single double-precision floating point value.
388 void checkDouble(double value, const char* id);
389 //! Check a single floating point value.
390 void checkReal(float value, const char* id);
391 //! Check a single floating point value.
392 void checkReal(double value, const char* id);
393 //! Check a vector of three integer values.
394 void checkVector(const int value[3], const char* id);
395 //! Check a vector of three single-precision floating point values.
396 void checkVector(const float value[3], const char* id);
397 //! Check a vector of three double-precision floating point values.
398 void checkVector(const double value[3], const char* id);
399 //! Check a BasicVector of ints, ie. IVec
400 void checkVector(const BasicVector<int>& value, const char* id);
401 //! Check a BasicVector of floats, ie. RVec
402 void checkVector(const BasicVector<float>& value, const char* id);
403 //! Check a BasicVector of doubles, ie. DVec
404 void checkVector(const BasicVector<double>& value, const char* id);
405 //! Check a single floating-point value from a string.
406 void checkRealFromString(const std::string& value, const char* id);
407 //! Checks a any value that contains a supported simple type.
408 void checkAny(const Any& value, const char* id);
409 //! Checks a key-value tree rooted at a object.
410 void checkKeyValueTreeObject(const KeyValueTreeObject& tree, const char* id);
411 //! Checks a generic key-value tree value.
412 void checkKeyValueTreeValue(const KeyValueTreeValue& value, const char* id);
414 /*! \name Methods to read values from reference data
416 * These methods assume that a value with the given `id` has already
417 * been created in the test with `check*()` methods, and that it has
420 * Currently, these methods do not work correctly if the reference data
421 * file does not exist, so a test using them may fail with exceptions
422 * before the reference data has been generated.
425 //! Reads an unsigned char value.
426 unsigned char readUChar(const char* id);
427 //! Reads an integer value.
428 int readInteger(const char* id);
429 //! Reads a 32-bit integer value.
430 int32_t readInt32(const char* id);
431 //! Reads a 64-bit integer value.
432 int64_t readInt64(const char* id);
433 //! Reads a float value.
434 float readFloat(const char* id);
435 //! Reads a double value.
436 double readDouble(const char* id);
437 //! Reads a string value.
438 std::string readString(const char* id);
441 /*! \name Overloaded versions of simple checker methods
443 * These methods provide overloads under a single name for all the
444 * methods checkBoolean(), checkString(), checkReal() and checkVector().
445 * They are provided mainly to allow template implementations (such as
446 * checkSequence()). Typically callers should use the individually
447 * named versions for greater clarity.
450 //! Check a single boolean value.
451 void checkValue(bool value, const char* id) { checkBoolean(value, id); }
452 //! Check a single string value.
453 void checkValue(const char* value, const char* id) { checkString(value, id); }
454 //! Check a single string value.
455 void checkValue(const std::string& value, const char* id) { checkString(value, id); }
456 //! Check a single signed integer value
457 void checkValue(int value, const char* id) { checkInteger(value, id); }
458 //! Check a single signed integer value of width 64 bits.
459 void checkValue(int64_t value, const char* id) { checkInt64(value, id); }
460 //! Check a single unsigned integer value of width 64 bits.
461 void checkValue(uint64_t value, const char* id) { checkUInt64(value, id); }
462 //! Check a single single-precision floating point value.
463 void checkValue(float value, const char* id) { checkFloat(value, id); }
464 //! Check a single double-precision floating point value.
465 void checkValue(double value, const char* id) { checkDouble(value, id); }
466 //! Check a vector of three integer values.
467 void checkValue(const int value[3], const char* id) { checkVector(value, id); }
468 //! Check a vector of three single-precision floating point values.
469 void checkValue(const float value[3], const char* id) { checkVector(value, id); }
470 //! Check a vector of three double-precision floating point values.
471 void checkValue(const double value[3], const char* id) { checkVector(value, id); }
472 //! Check a BasicVector of integer values, ie. IVec.
473 void checkValue(const BasicVector<int>& value, const char* id) { checkVector(value, id); }
474 //! Check a BasicVector of float values, ie. RVec.
475 void checkValue(const BasicVector<float>& value, const char* id) { checkVector(value, id); }
476 //! Check a BasicVector of double values, ie. DVec.
477 void checkValue(const BasicVector<double>& value, const char* id) { checkVector(value, id); }
478 //! Check a generic key-value tree value.
479 void checkValue(const KeyValueTreeValue& value, const char* id)
481 checkKeyValueTreeValue(value, id);
486 * Generic method to check a sequence of simple values.
488 * \tparam Iterator Input iterator that allows multiple (two) passes.
489 * Value type must be one of those accepted by checkValue(), or
490 * implicitly convertible to one.
491 * \param[in] begin Iterator to the start of the range to check.
492 * \param[in] end Iterator to the end of the range to check.
493 * \param[in] id Unique identifier for the sequence among its
496 template<class Iterator>
497 void checkSequence(Iterator begin, Iterator end, const char* id)
499 typename std::iterator_traits<Iterator>::difference_type length = std::distance(begin, end);
500 TestReferenceChecker compound(checkSequenceCompound(id, length));
501 for (Iterator i = begin; i != end; ++i)
503 compound.checkValue(*i, nullptr);
507 * Generic method to check a sequence of custom values.
509 * \tparam Iterator Input iterator that allows multiple (two) passes.
510 * \tparam ItemChecker Functor to check an individual value. Signature
511 * void(TestReferenceChecker *, const T &), where T is the value
512 * type of \p Iterator.
513 * \param[in] begin Iterator to the start of the range to check.
514 * \param[in] end Iterator to the end of the range to check.
515 * \param[in] id Unique identifier for the sequence among its
517 * \param[in] checkItem Functor to check an individual item.
519 * This method creates a compound checker \c compound within which all
520 * values of the sequence are checked. Calls \c checkItem(&compound, *i)
521 * with that compound for each iterator \c i in the range [begin, end).
522 * \p checkItem should use the various check methods in the passed
523 * checker to check each value.
525 * This method can be used to check a sequence made of compound types.
526 * Typically \p checkItem will create a compound within the passed
527 * checker to check different aspects of the value that was passed
528 * to it. Either NULL or a unique identifier string must be used for
529 * the id value of that compound. */
530 template<class Iterator, class ItemChecker>
531 void checkSequence(Iterator begin, Iterator end, const char* id, ItemChecker checkItem)
533 typename std::iterator_traits<Iterator>::difference_type length = std::distance(begin, end);
534 TestReferenceChecker compound(checkSequenceCompound(id, length));
535 for (Iterator i = begin; i != end; ++i)
537 checkItem(&compound, *i);
541 * Check an array of values.
543 * \tparam T Type of values to check. Should be one of those accepted
544 * by checkValue(), or implicitly convertible to one.
546 * \param[in] length Number of values to check.
547 * \param[in] values Pointer to the first value to check.
548 * \param[in] id Unique identifier for the sequence among its
551 * This is a convenience method that delegates all work to
555 void checkSequenceArray(size_t length, const T* values, const char* id)
557 checkSequence(values, values + length, id);
560 * Convenience method for checking that a sequence is empty.
562 * \param[in] id Unique identifier for the sequence among its
565 * This method provides a convenient solution for a case where there is
566 * implicitly a sequence to be checked, but there is no pointer
567 * available to the values since the sequence is empty.
568 * Since this method does not require the type of the values, it can be
569 * used in such cases easily.
571 void checkEmptySequence(const char* id);
573 * Initializes a compound for a sequence of items.
575 * \param[in] id Unique identifier for the sequence among its
577 * \param[in] length Number of items that will be in the sequence.
578 * \returns Checker to use for comparison within the sequence.
580 * This method can be used to check custom sequences where
581 * checkSequence() is not appropriate.
583 TestReferenceChecker checkSequenceCompound(const char* id, size_t length);
589 * Constructs a checker with a specific internal state.
591 * Is private to only allow users of this class to create instances
592 * using TestReferenceData::rootChecker() or checkCompound()
595 explicit TestReferenceChecker(Impl* impl);
597 std::unique_ptr<Impl> impl_;
600 * Needed to expose the constructor only to TestReferenceData.
602 friend class TestReferenceData;