2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2011,2012,2013,2014, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
35 /*! \libinternal \file
37 * Functionality for writing tests that can produce their own reference data.
39 * \author Teemu Murtola <teemu.murtola@gmail.com>
41 * \ingroup module_testutils
43 #ifndef GMX_TESTUTILS_REFDATA_H
44 #define GMX_TESTUTILS_REFDATA_H
49 #include "gromacs/utility/common.h"
59 class FloatingPointTolerance;
61 /*! \libinternal \brief
62 * Mode of operation for reference data handling.
64 * There should be no need to use this type outside the test utility module.
66 enum ReferenceDataMode
69 * Compare to existing reference data.
71 * If reference data does not exist, or if the test results differ from
72 * those in the reference data, the test fails.
76 * Create missing reference data.
78 * If reference data does not exist for a test, that test behaves as if
79 * ::erefdataUpdateAll had been specified. Tests for which reference data
80 * exists, behave like with ::erefdataCompare.
82 erefdataCreateMissing,
84 * Update reference data, overwriting old data.
86 * Tests utilizing reference data should always pass in this mode unless
87 * there is an I/O error.
92 /*! \libinternal \brief
93 * Returns the global reference data mode.
95 * There should be no need to use this function outside the test utility module.
97 ReferenceDataMode getReferenceDataMode();
98 /*! \libinternal \brief
99 * Sets the global reference data mode.
101 * There should be no need to use this function outside the test utility module.
103 void setReferenceDataMode(ReferenceDataMode mode);
104 /*! \libinternal \brief
105 * Returns the directory where reference data files are stored.
107 * There should be no need to use this function outside the test utility module.
109 std::string getReferenceDataPath();
110 /*! \libinternal \brief
111 * Initializes reference data handling.
113 * Adds command-line options to \p options to set the reference data mode.
114 * By default, ::erefdataCompare is used, but \c "--ref-data create" or
115 * \c "--ref-data update" can be used to change it.
117 * This function is automatically called by initTestUtils().
119 void initReferenceData(Options *options);
122 class TestReferenceChecker;
124 /*! \libinternal \brief
125 * Handles creation of and comparison to test reference data.
127 * This class provides functionality to use the same code to generate reference
128 * data and then on later runs compare the results of the code against that
129 * reference. The mode in which the class operates (writing reference data or
130 * comparing against existing data) is set with parseReferenceDataArgs(), which
131 * is automatically called when using the testutils module to implement tests.
132 * Tests only need to create an instance of TestReferenceData, obtain a
133 * TestReferenceChecker using the rootChecker() method and use the various
134 * check*() methods in TestReferenceChecker to indicate values to check. If
135 * the test is running in reference data creation mode, it will produce an XML
136 * file with the values recorder. In comparison mode, it will read that same
137 * XML file and produce a Google Test non-fatal assertion for every discrepancy
138 * it detects with the reference data (including missing reference data file or
139 * individual item). Exceptions derived from TestException are thrown for I/O
140 * errors and syntax errors in the reference data.
142 * Simple example (using Google Test):
144 int functionToTest(int param);
146 TEST(MyTest, SimpleTest)
148 gmx::test::TestReferenceData data;
150 gmx::test::TestReferenceChecker checker(data.rootChecker());
151 checker.checkInteger(functionToTest(3), "ValueWith3");
152 checker.checkInteger(functionToTest(5), "ValueWith5");
153 gmx::test::TestReferenceChecker compound(
154 checker.checkCompound("CustomCompound", "Item"));
155 compound.checkInteger(function2ToTest(3), "ValueWith3");
156 compound.checkInteger(function2ToTest(5), "ValueWith5");
157 checker.checkInteger(functionToTest(4), "ValueWith4");
161 * If rootChecker() is never called, no comparison is done (i.e., missing
162 * reference data file is not reported as an error, nor is empty reference data
163 * file created in write mode).
165 * For floating-point comparisons, the reference data should be generated in
166 * double precision (currently, no warning is provided even if this is not the
167 * case, but the double precision tests will then very likely fail).
170 * \ingroup module_testutils
172 class TestReferenceData
176 * Initializes the reference data in the global mode.
180 * Initializes the reference data in a specific mode.
182 * This function is mainly useful for self-testing the reference data
183 * framework. As such, it also puts the framework in a state where it
184 * logs additional internal information for failures to help diagnosing
185 * problems in the framework.
186 * The default constructor should be used in tests utilizing this class.
188 explicit TestReferenceData(ReferenceDataMode mode);
190 * Frees reference data structures.
192 * In the current implementation, this function writes the reference
193 * data out if necessary.
195 ~TestReferenceData();
197 //! Returns true if reference data is currently being written.
198 bool isWriteMode() const;
201 * Returns a root-level checker object for comparisons.
203 * Each call returns an independent instance.
205 TestReferenceChecker rootChecker();
210 PrivateImplPointer<Impl> impl_;
213 /*! \libinternal \brief
214 * Handles comparison to test reference data.
216 * Every check*() method takes an id string as the last parameter. This id is
217 * used to uniquely identify the value in the reference data, and it makes the
218 * output XML more human-friendly and more robust to errors. The id can be
219 * NULL; in this case, multiple elements with no id are created, and they will
220 * be matched in the same order as in which they are created. The
221 * checkCompound() method can be used to create a set of reference values
222 * grouped together. In this case, all check*() calls using the returned child
223 * TestReferenceChecker object will create the reference data within this
224 * group, and the ids only need to be unique within the compound. Compounds
227 * For usage example, see TestReferenceData.
229 * Copies of this class behave have independent internal state.
232 * \ingroup module_testutils
234 class TestReferenceChecker
238 * Creates a deep copy of the other checker.
240 TestReferenceChecker(const TestReferenceChecker &other);
241 ~TestReferenceChecker();
243 //! Assigns a test reference checker.
244 TestReferenceChecker &operator=(const TestReferenceChecker &other);
246 //! Returns true if reference data is currently being written.
247 bool isWriteMode() const;
250 * Sets the tolerance for floating-point comparisons.
252 * All following floating-point comparisons using this checker will use
253 * the new tolerance. Child checkers created with checkCompound()
254 * will inherit the tolerance from their parent checker at the time
255 * checkCompound() is called.
259 void setDefaultTolerance(const FloatingPointTolerance &tolerance);
262 * Checks whether a data item is present.
264 * \param[in] bPresent Whether to check for presence or absence.
265 * \param[in] id Unique identifier of the item to check.
266 * \returns true if bPresent was true and the data item was found.
268 * If \p bPresent is true, checks that a data item with \p id is
269 * present, otherwise checks that the data item is absent.
270 * If the check fails, a non-fatal Google Test assertion is generated.
272 * If isWriteMode() returns true, the check always succeeds and the
273 * return value is \p bPresent.
275 * The main use of this method is to assign meaning for missing
276 * reference data. Example use:
278 if (checker.checkPresent(bHaveVelocities, "Velocities"))
280 // <check the velocities>
284 bool checkPresent(bool bPresent, const char *id);
287 * Initializes comparison of a group of related data items.
289 * \param[in] type Informational type for the compound.
290 * \param[in] id Unique identifier for the compound among its
292 * \returns Checker to use for comparison within the compound.
294 * All checks performed with the returned checker only
295 * need to have unique ids within the compound, not globally.
297 * Compound structures can be nested.
299 TestReferenceChecker checkCompound(const char *type, const char *id);
301 //! Check a single boolean value.
302 void checkBoolean(bool value, const char *id);
303 //! Check a single string value.
304 void checkString(const char *value, const char *id);
305 //! Check a single string value.
306 void checkString(const std::string &value, const char *id);
308 * Check a multi-line string value.
310 * This method works as checkString(), but should be used for long
311 * strings that may contain, e.g., newlines. Typically used to check
312 * formatted output, and attempts to make the output XML such that it
313 * is easier to edit by hand to set the desired output formatting.
315 void checkStringBlock(const std::string &value, const char *id);
316 //! Check a single integer value.
317 void checkInteger(int value, const char *id);
318 //! Check a single single-precision floating point value.
319 void checkFloat(float value, const char *id);
320 //! Check a single double-precision floating point value.
321 void checkDouble(double value, const char *id);
322 //! Check a single floating point value.
323 void checkReal(float value, const char *id);
324 //! Check a single floating point value.
325 void checkReal(double value, const char *id);
326 //! Check a vector of three integer values.
327 void checkVector(const int value[3], const char *id);
328 //! Check a vector of three single-precision floating point values.
329 void checkVector(const float value[3], const char *id);
330 //! Check a vector of three double-precision floating point values.
331 void checkVector(const double value[3], const char *id);
333 /*! \name Overloaded versions of simple checker methods
335 * These methods provide overloads under a single name for all the
336 * methods checkBoolean(), checkString(), checkReal() and checkVector().
337 * They are provided mainly to allow template implementations (such as
338 * checkSequence()). Typically callers should use the individually
339 * named versions for greater clarity.
342 //! Check a single boolean value.
343 void checkValue(bool value, const char *id)
345 checkBoolean(value, id);
347 //! Check a single string value.
348 void checkValue(const char *value, const char *id)
350 checkString(value, id);
352 //! Check a single string value.
353 void checkValue(const std::string &value, const char *id)
355 checkString(value, id);
357 //! Check a single integer value.
358 void checkValue(int value, const char *id)
360 checkInteger(value, id);
362 //! Check a single single-precision floating point value.
363 void checkValue(float value, const char *id)
365 checkFloat(value, id);
367 //! Check a single double-precision floating point value.
368 void checkValue(double value, const char *id)
370 checkDouble(value, id);
372 //! Check a vector of three integer values.
373 void checkValue(const int value[3], const char *id)
375 checkVector(value, id);
377 //! Check a vector of three single-precision floating point values.
378 void checkValue(const float value[3], const char *id)
380 checkVector(value, id);
382 //! Check a vector of three double-precision floating point values.
383 void checkValue(const double value[3], const char *id)
385 checkVector(value, id);
390 * Generic method to check a sequence of simple values.
392 * \tparam Iterator Input iterator that allows multiple (two) passes.
393 * Value type must be one of those accepted by checkValue(), or
394 * implicitly convertible to one.
395 * \param[in] begin Iterator to the start of the range to check.
396 * \param[in] end Iterator to the end of the range to check.
397 * \param[in] id Unique identifier for the sequence among its
400 template <class Iterator>
401 void checkSequence(Iterator begin, Iterator end, const char *id)
403 typename std::iterator_traits<Iterator>::difference_type length
404 = std::distance(begin, end);
405 TestReferenceChecker compound(checkSequenceCompound(id, length));
406 for (Iterator i = begin; i != end; ++i)
408 compound.checkValue(*i, NULL);
412 * Generic method to check a sequence of custom values.
414 * \tparam Iterator Input iterator that allows multiple (two) passes.
415 * \tparam ItemChecker Functor to check an individual value. Signature
416 * void(TestReferenceChecker *, const T &), where T is the value
417 * type of \p Iterator.
418 * \param[in] begin Iterator to the start of the range to check.
419 * \param[in] end Iterator to the end of the range to check.
420 * \param[in] id Unique identifier for the sequence among its
422 * \param[in] checkItem Functor to check an individual item.
424 * This method creates a compound checker \c compound within which all
425 * values of the sequence are checked. Calls \c checkItem(&compound, *i)
426 * with that compound for each iterator \c i in the range [begin, end).
427 * \p checkItem should use the various check methods in the passed
428 * checker to check each value.
430 * This method can be used to check a sequence made of compound types.
431 * Typically \p checkItem will create a compound within the passed
432 * checker to check different aspects of the passed in value.
434 template <class Iterator, class ItemChecker>
435 void checkSequence(Iterator begin, Iterator end, const char *id,
436 ItemChecker checkItem)
438 typename std::iterator_traits<Iterator>::difference_type length
439 = std::distance(begin, end);
440 TestReferenceChecker compound(checkSequenceCompound(id, length));
441 for (Iterator i = begin; i != end; ++i)
443 checkItem(&compound, *i);
447 * Check an array of values.
449 * \tparam T Type of values to check. Should be one of those accepted
450 * by checkValue(), or implicitly convertible to one.
452 * \param[in] length Number of values to check.
453 * \param[in] values Pointer to the first value to check.
454 * \param[in] id Unique identifier for the sequence among its
457 * This is a convenience method that delegates all work to
460 template <typename T>
461 void checkSequenceArray(size_t length, const T *values, const char *id)
463 checkSequence(values, values + length, id);
466 * Convenience method for checking that a sequence is empty.
468 * \param[in] id Unique identifier for the sequence among its
471 * This method provides a convenient solution for a case where there is
472 * implicitly a sequence to be checked, but there is no pointer
473 * available to the values since the sequence is empty.
474 * Since this method does not require the type of the values, it can be
475 * used in such cases easily.
477 void checkEmptySequence(const char *id);
479 * Initializes a compound for a sequence of items.
481 * \param[in] id Unique identifier for the sequence among its
483 * \param[in] length Number of items that will be in the sequence.
484 * \returns Checker to use for comparison within the sequence.
486 * This method can be used to check custom sequences where
487 * checkSequence() is not appropriate.
489 TestReferenceChecker checkSequenceCompound(const char *id, size_t length);
495 * Constructs a checker with a specific internal state.
497 * Is private to only allow users of this class to create instances
498 * using TestReferenceData::rootChecker() or checkCompound()
501 explicit TestReferenceChecker(Impl *impl);
503 PrivateImplPointer<Impl> impl_;
506 * Needed to expose the constructor only to TestReferenceData.
508 friend class TestReferenceData;