2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2011,2012,2013,2014,2015,2016,2017, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
35 /*! \libinternal \file
37 * Functionality for writing tests that can produce their own reference data.
39 * See \ref page_refdata for more details.
41 * \author Teemu Murtola <teemu.murtola@gmail.com>
43 * \ingroup module_testutils
45 #ifndef GMX_TESTUTILS_REFDATA_H
46 #define GMX_TESTUTILS_REFDATA_H
52 #include "gromacs/utility/basedefinitions.h"
53 #include "gromacs/utility/classhelpers.h"
58 class IOptionsContainer;
59 class KeyValueTreeObject;
60 class KeyValueTreeValue;
66 class FloatingPointTolerance;
68 /*! \libinternal \brief
69 * Mode of operation for reference data handling.
71 * There should be no need to use this type outside the test utility module.
73 * \ingroup module_testutils
75 enum ReferenceDataMode
78 * Compare to existing reference data.
80 * If reference data does not exist, or if the test results differ from
81 * those in the reference data, the test fails.
85 * Create missing reference data.
87 * If reference data does not exist for a test, that test behaves as if
88 * ::erefdataUpdateAll had been specified. Tests for which reference data
89 * exists, behave like with ::erefdataCompare.
91 erefdataCreateMissing,
93 * Update reference data that does not pass comparison.
95 * Tests utilizing reference data should always pass in this mode unless
96 * there is an I/O error.
98 erefdataUpdateChanged,
100 * Update reference data, overwriting old data.
102 * Tests utilizing reference data should always pass in this mode unless
103 * there is an I/O error.
108 /*! \libinternal \brief
109 * Initializes reference data handling.
111 * Adds command-line options to \p options to set the reference data mode.
112 * By default, ::erefdataCompare is used, but ``--ref-data create`` or
113 * ``--ref-data update`` can be used to change it.
115 * This function is automatically called by initTestUtils().
117 * \ingroup module_testutils
119 void initReferenceData(IOptionsContainer *options);
121 class TestReferenceChecker;
125 class TestReferenceDataImpl;
128 /*! \libinternal \brief
129 * Handles creation of and comparison to test reference data.
131 * See \ref page_refdata for an overview of the functionality.
133 * This class provides functionality to use the same code to generate reference
134 * data and then on later runs compare the results of the code against that
135 * reference. The mode in which the class operates (writing reference data or
136 * comparing against existing data) is set using a command-line option that
137 * is automatically managed when using the testutils module to implement tests.
138 * Tests only need to create an instance of TestReferenceData, obtain a
139 * TestReferenceChecker using the rootChecker() method and use the various
140 * check*() methods in TestReferenceChecker to indicate values to check. If
141 * the test is running in reference data creation mode, it will produce an XML
142 * file with the values recorder. In comparison mode, it will read that same
143 * XML file and produce a Google Test non-fatal assertion for every discrepancy
144 * it detects with the reference data (including missing reference data file or
145 * individual item). Exceptions derived from TestException are thrown for I/O
146 * errors and syntax errors in the reference data.
148 * Simple example (using Google Test):
150 int functionToTest(int param);
152 TEST(MyTest, SimpleTest)
154 gmx::test::TestReferenceData data;
156 gmx::test::TestReferenceChecker checker(data.rootChecker());
157 checker.checkInteger(functionToTest(3), "ValueWith3");
158 checker.checkInteger(functionToTest(5), "ValueWith5");
159 gmx::test::TestReferenceChecker compound(
160 checker.checkCompound("CustomCompound", "Item"));
161 compound.checkInteger(function2ToTest(3), "ValueWith3");
162 compound.checkInteger(function2ToTest(5), "ValueWith5");
163 checker.checkInteger(functionToTest(4), "ValueWith4");
167 * If rootChecker() is never called, no comparison is done (i.e., missing
168 * reference data file is not reported as an error, nor is empty reference data
169 * file created in write mode).
171 * For floating-point comparisons, the reference data should be generated in
172 * double precision (currently, no warning is provided even if this is not the
173 * case, but the double precision tests will then very likely fail).
176 * \ingroup module_testutils
178 class TestReferenceData
182 * Initializes the reference data in the global mode.
186 * Initializes the reference data in a specific mode.
188 * This function is only useful for self-testing the reference data
189 * framework. As such, it also puts the framework in a state where it
190 * logs additional internal information for failures to help diagnosing
191 * problems in the framework, and stores the reference data in a
192 * temporary directory instead of the source tree.
193 * The default constructor should be used in tests utilizing this class.
195 explicit TestReferenceData(ReferenceDataMode mode);
197 * Frees reference data structures.
199 * The reference data is written out if necessary automatically when
202 ~TestReferenceData();
205 * Returns a root-level checker object for comparisons.
207 * Each call returns an independent instance.
209 TestReferenceChecker rootChecker();
212 std::shared_ptr<internal::TestReferenceDataImpl> impl_;
214 GMX_DISALLOW_COPY_AND_ASSIGN(TestReferenceData);
217 /*! \libinternal \brief
218 * Handles comparison to test reference data.
220 * Every check*() method takes an id string as the last parameter. This id is
221 * used to uniquely identify the value in the reference data, and it makes the
222 * output XML more human-friendly and more robust to errors. The id can be
223 * NULL; in this case, multiple elements with no id are created, and they will
224 * be matched in the same order as in which they are created. The
225 * checkCompound() method can be used to create a set of reference values
226 * grouped together. In this case, all check*() calls using the returned child
227 * TestReferenceChecker object will create the reference data within this
228 * group, and the ids only need to be unique within the compound. Compounds
231 * For usage example, see TestReferenceData.
233 * Copies of this class behave have independent internal state.
236 * \ingroup module_testutils
238 class TestReferenceChecker
242 * Creates a checker that cannot be used for checking.
244 * Attempting to call the check methods generates an assert.
245 * It is possible to check whether the checker is initialized by
247 * This constructor exists to allow declaring checker variables that
248 * will receive their value later without resorting to dynamic
251 TestReferenceChecker();
252 //! Creates a deep copy of the other checker.
253 explicit TestReferenceChecker(const TestReferenceChecker &other);
254 //! Moves the checker.
255 TestReferenceChecker(TestReferenceChecker &&other);
256 ~TestReferenceChecker();
258 //! Prevents implicit copying during assignment.
259 TestReferenceChecker &operator=(const TestReferenceChecker &) = delete;
260 //! Assigns a test reference checker.
261 TestReferenceChecker &operator=(TestReferenceChecker &&other);
263 //! Returns whether the checker is initialized.
264 bool isValid() const;
265 //! Allows testing whether the checker is initialized directly with if.
266 explicit operator bool() const { return isValid(); }
269 * Sets the tolerance for floating-point comparisons.
271 * All following floating-point comparisons using this checker will use
272 * the new tolerance. Child checkers created with checkCompound()
273 * will inherit the tolerance from their parent checker at the time
274 * checkCompound() is called.
278 void setDefaultTolerance(const FloatingPointTolerance &tolerance);
281 * Checks that all reference values have been compared against.
283 * All values under the compound represented by this checker are
284 * checked, and a non-fatal Google Test assertion is produced if some
285 * values have not been used.
287 * If not called explicitly, the same check will be done for all
288 * reference data values when the test ends.
290 * This method also marks the values used, so that subsequent checks
291 * (including the check at the end of the test) will not produce
292 * another assertion about the same values.
294 void checkUnusedEntries();
297 * Checks whether a data item is present.
299 * \param[in] bPresent Whether to check for presence or absence.
300 * \param[in] id Unique identifier of the item to check.
301 * \returns true if bPresent was true and the data item was found.
303 * If \p bPresent is true, checks that a data item with \p id is
304 * present, otherwise checks that the data item is absent.
305 * If the check fails, a non-fatal Google Test assertion is generated.
307 * If reference data is being written, the check always succeeds and the
308 * return value is \p bPresent.
310 * The main use of this method is to assign meaning for missing
311 * reference data. Example use:
313 if (checker.checkPresent(bHaveVelocities, "Velocities"))
315 // <check the velocities>
319 bool checkPresent(bool bPresent, const char *id);
322 * Initializes comparison of a group of related data items.
324 * \param[in] type Informational type for the compound.
325 * \param[in] id Unique identifier for the compound among its
327 * \returns Checker to use for comparison within the compound.
329 * All checks performed with the returned checker only
330 * need to have unique ids within the compound, not globally.
332 * Compound structures can be nested.
334 TestReferenceChecker checkCompound(const char *type, const char *id);
336 //! Check a single boolean value.
337 void checkBoolean(bool value, const char *id);
338 //! Check a single string value.
339 void checkString(const char *value, const char *id);
340 //! Check a single string value.
341 void checkString(const std::string &value, const char *id);
343 * Check a multi-line string value.
345 * This method works as checkString(), but should be used for long
346 * strings that may contain, e.g., newlines. Typically used to check
347 * formatted output, and attempts to make the output XML such that it
348 * is easier to edit by hand to set the desired output formatting.
350 void checkTextBlock(const std::string &value, const char *id);
351 //! Check a single unsigned char value.
352 void checkUChar(unsigned char value, const char *id);
353 //! Check a single integer value.
354 void checkInteger(int value, const char *id);
355 //! Check a single int64 value.
356 void checkInt64(gmx_int64_t value, const char *id);
357 //! Check a single uint64 value.
358 void checkUInt64(gmx_uint64_t value, const char *id);
359 //! Check a single single-precision floating point value.
360 void checkFloat(float value, const char *id);
361 //! Check a single double-precision floating point value.
362 void checkDouble(double value, const char *id);
363 //! Check a single floating point value.
364 void checkReal(float value, const char *id);
365 //! Check a single floating point value.
366 void checkReal(double value, const char *id);
367 //! Check a vector of three integer values.
368 void checkVector(const int value[3], const char *id);
369 //! Check a vector of three single-precision floating point values.
370 void checkVector(const float value[3], const char *id);
371 //! Check a vector of three double-precision floating point values.
372 void checkVector(const double value[3], const char *id);
373 //! Check a single floating-point value from a string.
374 void checkRealFromString(const std::string &value, const char *id);
375 //! Checks a variant value that contains a supported simple type.
376 void checkVariant(const Variant &value, const char *id);
377 //! Checks a key-value tree rooted at a object.
378 void checkKeyValueTreeObject(const KeyValueTreeObject &tree, const char *id);
379 //! Checks a generic key-value tree value.
380 void checkKeyValueTreeValue(const KeyValueTreeValue &value, const char *id);
382 /*! \name Methods to read values from reference data
384 * These methods assume that a value with the given `id` has already
385 * been created in the test with `check*()` methods, and that it has
388 * Currently, these methods do not work correctly if the reference data
389 * file does not exist, so a test using them may fail with exceptions
390 * before the reference data has been generated.
393 //! Reads an unsigned char value.
394 unsigned char readUChar(const char *id);
395 //! Reads an integer value.
396 int readInteger(const char *id);
397 //! Reads a float value.
398 float readFloat(const char *id);
399 //! Reads a double value.
400 double readDouble(const char *id);
401 //! Reads a string value.
402 std::string readString(const char *id);
405 /*! \name Overloaded versions of simple checker methods
407 * These methods provide overloads under a single name for all the
408 * methods checkBoolean(), checkString(), checkReal() and checkVector().
409 * They are provided mainly to allow template implementations (such as
410 * checkSequence()). Typically callers should use the individually
411 * named versions for greater clarity.
414 //! Check a single boolean value.
415 void checkValue(bool value, const char *id)
417 checkBoolean(value, id);
419 //! Check a single string value.
420 void checkValue(const char *value, const char *id)
422 checkString(value, id);
424 //! Check a single string value.
425 void checkValue(const std::string &value, const char *id)
427 checkString(value, id);
429 //! Check a single integer value.
430 void checkValue(int value, const char *id)
432 checkInteger(value, id);
434 //! Check a single integer value.
435 void checkValue(gmx_int64_t value, const char *id)
437 checkInt64(value, id);
439 //! Check a single integer value.
440 void checkValue(gmx_uint64_t value, const char *id)
442 checkUInt64(value, id);
444 //! Check a single single-precision floating point value.
445 void checkValue(float value, const char *id)
447 checkFloat(value, id);
449 //! Check a single double-precision floating point value.
450 void checkValue(double value, const char *id)
452 checkDouble(value, id);
454 //! Check a vector of three integer values.
455 void checkValue(const int value[3], const char *id)
457 checkVector(value, id);
459 //! Check a vector of three single-precision floating point values.
460 void checkValue(const float value[3], const char *id)
462 checkVector(value, id);
464 //! Check a vector of three double-precision floating point values.
465 void checkValue(const double value[3], const char *id)
467 checkVector(value, id);
469 //! Check a generic key-value tree value.
470 void checkValue(const KeyValueTreeValue &value, const char *id)
472 checkKeyValueTreeValue(value, id);
477 * Generic method to check a sequence of simple values.
479 * \tparam Iterator Input iterator that allows multiple (two) passes.
480 * Value type must be one of those accepted by checkValue(), or
481 * implicitly convertible to one.
482 * \param[in] begin Iterator to the start of the range to check.
483 * \param[in] end Iterator to the end of the range to check.
484 * \param[in] id Unique identifier for the sequence among its
487 template <class Iterator>
488 void checkSequence(Iterator begin, Iterator end, const char *id)
490 typename std::iterator_traits<Iterator>::difference_type length
491 = std::distance(begin, end);
492 TestReferenceChecker compound(checkSequenceCompound(id, length));
493 for (Iterator i = begin; i != end; ++i)
495 compound.checkValue(*i, nullptr);
499 * Generic method to check a sequence of custom values.
501 * \tparam Iterator Input iterator that allows multiple (two) passes.
502 * \tparam ItemChecker Functor to check an individual value. Signature
503 * void(TestReferenceChecker *, const T &), where T is the value
504 * type of \p Iterator.
505 * \param[in] begin Iterator to the start of the range to check.
506 * \param[in] end Iterator to the end of the range to check.
507 * \param[in] id Unique identifier for the sequence among its
509 * \param[in] checkItem Functor to check an individual item.
511 * This method creates a compound checker \c compound within which all
512 * values of the sequence are checked. Calls \c checkItem(&compound, *i)
513 * with that compound for each iterator \c i in the range [begin, end).
514 * \p checkItem should use the various check methods in the passed
515 * checker to check each value.
517 * This method can be used to check a sequence made of compound types.
518 * Typically \p checkItem will create a compound within the passed
519 * checker to check different aspects of the value that was passed
520 * to it. Either NULL or a unique identifier string must be used for
521 * the id value of that compound. */
522 template <class Iterator, class ItemChecker>
523 void checkSequence(Iterator begin, Iterator end, const char *id,
524 ItemChecker checkItem)
526 typename std::iterator_traits<Iterator>::difference_type length
527 = std::distance(begin, end);
528 TestReferenceChecker compound(checkSequenceCompound(id, length));
529 for (Iterator i = begin; i != end; ++i)
531 checkItem(&compound, *i);
535 * Check an array of values.
537 * \tparam T Type of values to check. Should be one of those accepted
538 * by checkValue(), or implicitly convertible to one.
540 * \param[in] length Number of values to check.
541 * \param[in] values Pointer to the first value to check.
542 * \param[in] id Unique identifier for the sequence among its
545 * This is a convenience method that delegates all work to
548 template <typename T>
549 void checkSequenceArray(size_t length, const T *values, const char *id)
551 checkSequence(values, values + length, id);
554 * Convenience method for checking that a sequence is empty.
556 * \param[in] id Unique identifier for the sequence among its
559 * This method provides a convenient solution for a case where there is
560 * implicitly a sequence to be checked, but there is no pointer
561 * available to the values since the sequence is empty.
562 * Since this method does not require the type of the values, it can be
563 * used in such cases easily.
565 void checkEmptySequence(const char *id);
567 * Initializes a compound for a sequence of items.
569 * \param[in] id Unique identifier for the sequence among its
571 * \param[in] length Number of items that will be in the sequence.
572 * \returns Checker to use for comparison within the sequence.
574 * This method can be used to check custom sequences where
575 * checkSequence() is not appropriate.
577 TestReferenceChecker checkSequenceCompound(const char *id, size_t length);
583 * Constructs a checker with a specific internal state.
585 * Is private to only allow users of this class to create instances
586 * using TestReferenceData::rootChecker() or checkCompound()
589 explicit TestReferenceChecker(Impl *impl);
591 PrivateImplPointer<Impl> impl_;
594 * Needed to expose the constructor only to TestReferenceData.
596 friend class TestReferenceData;