*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#include "gmxfio_xdr.h"
+#include <cstddef>
#include <cstdio>
#include <cstring>
+#include <limits>
+
#include "gromacs/fileio/gmxfio.h"
#include "gromacs/fileio/xdrf.h"
#include "gromacs/utility/fatalerror.h"
eioNRVEC,
eioIVEC,
eioSTRING,
+ eioOPAQUE,
eioNR
};
-static const char* eioNames[eioNR] = { "REAL", "FLOAT", "DOUBLE", "INT", "INT32",
- "INT64", "UCHAR", "CHAR", "NCHAR", "NUCHAR",
- "USHORT", "RVEC", "NRVEC", "IVEC", "STRING" };
+static const char* eioNames[eioNR] = { "REAL", "FLOAT", "DOUBLE", "INT", "INT32", "INT64",
+ "UCHAR", "CHAR", "NCHAR", "NUCHAR", "USHORT", "RVEC",
+ "NRVEC", "IVEC", "STRING", "OPAQUE" };
void gmx_fio_setprecision(t_fileio* fio, gmx_bool bDouble)
{
}
/* check the number of items given against the type */
-static void gmx_fio_check_nitem(int eio, int nitem, const char* file, int line)
+static void gmx_fio_check_nitem(int eio, std::size_t nitem, const char* file, int line)
{
- if ((nitem != 1) && !((eio == eioNRVEC) || (eio == eioNUCHAR) || (eio == eioNCHAR)))
+ if ((nitem != 1)
+ && !((eio == eioNRVEC) || (eio == eioNUCHAR) || (eio == eioNCHAR) || (eio == eioOPAQUE)))
{
gmx_fatal(FARGS,
- "nitem (%d) may differ from 1 only for %s, %s or %s, not for %s"
+ "nitem may differ from 1 only for %s, %s, %s or %s, not for %s"
"(%s, %d)",
- nitem, eioNames[eioNUCHAR], eioNames[eioNRVEC], eioNames[eioNCHAR], eioNames[eio],
- file, line);
+ eioNames[eioNUCHAR], eioNames[eioNRVEC], eioNames[eioNCHAR], eioNames[eioOPAQUE],
+ eioNames[eio], file, line);
}
}
/* This is the part that reads xdr files. */
-static gmx_bool do_xdr(t_fileio* fio, void* item, int nitem, int eio, const char* desc, const char* srcfile, int line)
+static gmx_bool
+do_xdr(t_fileio* fio, void* item, std::size_t nitem, int eio, const char* desc, const char* srcfile, int line)
{
unsigned char ucdum, *ucptr;
char cdum, *cptr;
bool_t res = 0;
float fvec[DIM];
double dvec[DIM];
- int j, m, *iptr, idum;
+ int m, *iptr, idum;
int32_t s32dum;
int64_t s64dum;
real* ptr;
break;
case eioNCHAR:
cptr = static_cast<char*>(item);
- res = xdr_vector(fio->xdr, cptr, nitem, static_cast<unsigned int>(sizeof(char)),
+ GMX_RELEASE_ASSERT(nitem < static_cast<std::size_t>(std::numeric_limits<int>::max()),
+ "The XDR interface cannot handle array lengths > 2^31");
+ res = xdr_vector(fio->xdr, cptr, static_cast<int>(nitem),
+ static_cast<unsigned int>(sizeof(char)),
reinterpret_cast<xdrproc_t>(xdr_char));
break;
case eioNUCHAR:
ucptr = static_cast<unsigned char*>(item);
- res = xdr_vector(fio->xdr, reinterpret_cast<char*>(ucptr), nitem,
+ GMX_RELEASE_ASSERT(nitem < static_cast<std::size_t>(std::numeric_limits<int>::max()),
+ "The XDR interface cannot handle array lengths > 2^31");
+ res = xdr_vector(fio->xdr, reinterpret_cast<char*>(ucptr), static_cast<int>(nitem),
static_cast<unsigned int>(sizeof(unsigned char)),
reinterpret_cast<xdrproc_t>(xdr_u_char));
break;
case eioNRVEC:
ptr = nullptr;
res = 1;
- for (j = 0; (j < nitem) && res; j++)
+ for (std::size_t j = 0; j < nitem && res; j++)
{
if (item)
{
}
break;
}
+ case eioOPAQUE:
+ {
+ if (item == nullptr && nitem > 0)
+ {
+ gmx_fatal(FARGS, "Null pointer provided for non-zero length XDR opaque data.");
+ }
+
+ if (nitem > 0)
+ {
+ // We need to support very large opaque data objects although the default
+ // XDR interface only uses integers for the size field, since gromacs-2020
+ // e.g. embeds the entire TPR body as a single such object, which would break all
+ // TPR files larger than 2GB unless we handle it as a special case.
+ // To avoid inserting extra padding, we calculate the chunk size as:
+ // - The max value of a signed integer + 1
+ // - Subtract 4 (the XDR object size) to get a size within the range of the signed int.
+ const std::size_t maxChunk =
+ static_cast<std::size_t>(std::numeric_limits<int>::max()) + 1 - 4;
+
+ for (res = 1; res > 0 && nitem > 0;)
+ {
+ std::size_t thisChunk = std::min(maxChunk, nitem);
+ res = xdr_opaque(fio->xdr, reinterpret_cast<char*>(item), thisChunk);
+ nitem -= thisChunk;
+ }
+ }
+ else
+ {
+ res = 1;
+ }
+ break;
+ }
default: gmx_fio_fe(fio, eio, desc, srcfile, line);
}
return ret;
}
+gmx_bool gmx_fio_doe_opaque(t_fileio* fio, char* data, std::size_t size, const char* desc, const char* srcfile, int line)
+{
+ gmx_bool ret;
+ gmx_fio_lock(fio);
+ ret = do_xdr(fio, data, size, eioOPAQUE, desc, srcfile, line);
+ gmx_fio_unlock(fio);
+ return ret;
+}
/* Array reading & writing */
}
}
+void FileIOXdrSerializer::doOpaque(char* data, std::size_t size)
+{
+ gmx_fio_do_opaque(fio_, data, size);
+}
+
} // namespace gmx
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#ifndef GMX_FILEIO_GMXFIO_XDR_H
#define GMX_FILEIO_GMXFIO_XDR_H
+#include <cstddef>
+
#include <string>
#include "gromacs/fileio/xdrf.h"
gmx_bool gmx_fio_doe_rvec(struct t_fileio* fio, rvec* item, const char* desc, const char* srcfile, int line);
gmx_bool gmx_fio_doe_ivec(struct t_fileio* fio, ivec* item, const char* desc, const char* srcfile, int line);
gmx_bool gmx_fio_doe_string(struct t_fileio* fio, char* item, const char* desc, const char* srcfile, int line);
+gmx_bool gmx_fio_doe_opaque(struct t_fileio* fio,
+ char* item,
+ std::size_t size,
+ const char* desc,
+ const char* srcfile,
+ int line);
/* array reading & writing */
gmx_bool gmx_fio_ndoe_real(struct t_fileio* fio, real* item, int n, const char* desc, const char* srcfile, int line);
#define gmx_fio_do_rvec(fio, item) gmx_fio_doe_rvec(fio, &(item), (#item), __FILE__, __LINE__)
#define gmx_fio_do_ivec(fio, item) gmx_fio_doe_ivec(fio, &(item), (#item), __FILE__, __LINE__)
#define gmx_fio_do_string(fio, item) gmx_fio_doe_string(fio, item, (#item), __FILE__, __LINE__)
+#define gmx_fio_do_opaque(fio, item, size) \
+ gmx_fio_doe_opaque(fio, item, size, (#item), __FILE__, __LINE__)
#define gmx_fio_ndo_real(fio, item, n) gmx_fio_ndoe_real(fio, item, n, (#item), __FILE__, __LINE__)
void doRvec(rvec* value) override;
//! Handle I/O if string.
void doString(std::string* value) override;
+ //! Handle opaque data.
+ void doOpaque(char* data, std::size_t size) override;
//! Special case for handling I/O of a vector of characters.
void doCharArray(char* values, int elements) override;
//! Special case for handling I/O of a vector of unsigned characters.
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2019, by the GROMACS development team, led by
+ * Copyright (c) 2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
{
if (!mrcHeaderIsSane(reader_->header()))
{
- serializer_ = std::make_unique<InMemoryDeserializer>(buffer_, false, EndianSwapBehavior::DoSwap);
- reader_ = std::make_unique<MrcDensityMapOfFloatReader>(serializer_.get());
+ serializer_ = std::make_unique<InMemoryDeserializer>(buffer_, false, EndianSwapBehavior::Swap);
+ reader_ = std::make_unique<MrcDensityMapOfFloatReader>(serializer_.get());
if (!mrcHeaderIsSane(reader_->header()))
{
GMX_THROW(FileIOError(
*
* Copyright (c) 1991-2000, University of Groningen, The Netherlands.
* Copyright (c) 2001-2004, The GROMACS development team.
- * Copyright (c) 2013,2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2013,2014,2015,2016,2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
}
/*! \brief
- * Process the body of a TPR file as char buffer.
+ * Process the body of a TPR file as an opaque data buffer.
*
* Reads/writes the information in \p buffer from/to the \p serializer
* provided to the function. Does not interact with the actual
*/
static void doTpxBodyBuffer(gmx::ISerializer* serializer, gmx::ArrayRef<char> buffer)
{
- serializer->doCharArray(buffer.data(), buffer.size());
+ serializer->doOpaque(buffer.data(), buffer.size());
}
/*! \brief
// we prepare a new char buffer with the information we have already read
// in on master.
partialDeserializedTpr.header = populateTpxHeader(*state, ir, mtop);
- gmx::InMemorySerializer tprBodySerializer;
+ // Long-term we should move to use little endian in files to avoid extra byte swapping,
+ // but since we just used the default XDR format (which is big endian) for the TPR
+ // header it would cause third-party libraries reading our raw data to tear their hair
+ // if we swap the endian in the middle of the file, so we stick to big endian in the
+ // TPR file for now - and thus we ask the serializer to swap if this host is little endian.
+ gmx::InMemorySerializer tprBodySerializer(gmx::EndianSwapBehavior::SwapIfHostIsLittleEndian);
do_tpx_body(&tprBodySerializer, &partialDeserializedTpr.header, ir, mtop);
partialDeserializedTpr.body = tprBodySerializer.finishAndGetBuffer();
t_fileio* fio;
TpxFileHeader tpx = populateTpxHeader(*state, ir, mtop);
-
- gmx::InMemorySerializer tprBodySerializer;
+ // Long-term we should move to use little endian in files to avoid extra byte swapping,
+ // but since we just used the default XDR format (which is big endian) for the TPR
+ // header it would cause third-party libraries reading our raw data to tear their hair
+ // if we swap the endian in the middle of the file, so we stick to big endian in the
+ // TPR file for now - and thus we ask the serializer to swap if this host is little endian.
+ gmx::InMemorySerializer tprBodySerializer(gmx::EndianSwapBehavior::SwapIfHostIsLittleEndian);
do_tpx_body(&tprBodySerializer, &tpx, const_cast<t_inputrec*>(ir), const_cast<t_state*>(state),
nullptr, nullptr, const_cast<gmx_mtop_t*>(mtop));
rvec* v,
gmx_mtop_t* mtop)
{
+ // Long-term we should move to use little endian in files to avoid extra byte swapping,
+ // but since we just used the default XDR format (which is big endian) for the TPR
+ // header it would cause third-party libraries reading our raw data to tear their hair
+ // if we swap the endian in the middle of the file, so we stick to big endian in the
+ // TPR file for now - and thus we ask the serializer to swap if this host is little endian.
gmx::InMemoryDeserializer tprBodyDeserializer(partialDeserializedTpr->body,
- partialDeserializedTpr->header.isDouble);
+ partialDeserializedTpr->header.isDouble,
+ gmx::EndianSwapBehavior::SwapIfHostIsLittleEndian);
return do_tpx_body(&tprBodyDeserializer, &partialDeserializedTpr->header, ir, state, x, v, mtop);
}
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2016,2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2016,2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#include "inmemoryserializer.h"
+#include "config.h"
+
#include <algorithm>
#include <vector>
return endianessSwappedValue.value_;
}
+//! \brief Change the host-dependent endian settings to either Swap or DoNotSwap.
+//
+// \param endianSwapBehavior input swap behavior, might depend on host.
+//
+// \return Host-independent setting, either Swap or DoNotSwap.
+EndianSwapBehavior setEndianSwapBehaviorFromHost(EndianSwapBehavior endianSwapBehavior)
+{
+ if (endianSwapBehavior == EndianSwapBehavior::SwapIfHostIsBigEndian)
+ {
+ return GMX_INTEGER_BIG_ENDIAN ? EndianSwapBehavior::Swap : EndianSwapBehavior::DoNotSwap;
+ }
+ else if (endianSwapBehavior == EndianSwapBehavior::SwapIfHostIsLittleEndian)
+ {
+ return GMX_INTEGER_BIG_ENDIAN ? EndianSwapBehavior::DoNotSwap : EndianSwapBehavior::Swap;
+ }
+ else
+ {
+ return endianSwapBehavior;
+ }
+}
+
} // namespace
/********************************************************************
class InMemorySerializer::Impl
{
public:
- Impl(EndianSwapBehavior endianSwapBehavior) : endianSwapBehavior_(endianSwapBehavior) {}
+ Impl(EndianSwapBehavior endianSwapBehavior) :
+ endianSwapBehavior_(setEndianSwapBehaviorFromHost(endianSwapBehavior))
+ {
+ }
+
template<typename T>
void doValue(T value)
{
- if (endianSwapBehavior_ == EndianSwapBehavior::DoSwap)
+ if (endianSwapBehavior_ == EndianSwapBehavior::Swap)
{
CharBuffer<T>(swapEndian(value)).appendTo(&buffer_);
}
doValue<uint64_t>(value.size());
buffer_.insert(buffer_.end(), value.begin(), value.end());
}
+ void doOpaque(const char* data, std::size_t size)
+ {
+ buffer_.insert(buffer_.end(), data, data + size);
+ }
std::vector<char> buffer_;
EndianSwapBehavior endianSwapBehavior_;
impl_->doString(*value);
}
+void InMemorySerializer::doOpaque(char* data, std::size_t size)
+{
+ impl_->doOpaque(data, size);
+}
+
/********************************************************************
* InMemoryDeserializer
*/
buffer_(buffer),
sourceIsDouble_(sourceIsDouble),
pos_(0),
- endianSwapBehavior_(endianSwapBehavior)
+ endianSwapBehavior_(setEndianSwapBehaviorFromHost(endianSwapBehavior))
{
}
template<typename T>
void doValue(T* value)
{
- if (endianSwapBehavior_ == EndianSwapBehavior::DoSwap)
+ if (endianSwapBehavior_ == EndianSwapBehavior::Swap)
{
*value = swapEndian(CharBuffer<T>(&buffer_[pos_]).value());
}
*value = std::string(&buffer_[pos_], size);
pos_ += size;
}
+ void doOpaque(char* data, std::size_t size)
+ {
+ std::copy(&buffer_[pos_], &buffer_[pos_ + size], data);
+ pos_ += size;
+ }
ArrayRef<const char> buffer_;
bool sourceIsDouble_;
impl_->doString(value);
}
+void InMemoryDeserializer::doOpaque(char* data, std::size_t size)
+{
+ impl_->doOpaque(data, size);
+}
+
} // namespace gmx
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2016,2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2016,2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#ifndef GMX_UTILITY_INMEMORYSERIALIZER_H
#define GMX_UTILITY_INMEMORYSERIALIZER_H
+#include <cstddef>
+
#include <vector>
#include "gromacs/utility/arrayref.h"
namespace gmx
{
-//! Specify endian swapping behvaior
+//! Specify endian swapping behavoir.
+//
+// The host-dependent choices avoid the calling file having to
+// depend on config.h.
+//
enum class EndianSwapBehavior : int
{
- DoSwap, //!< Swap the bytes
- DoNotSwap, //!< Do not swap the bytes
- Count //!< Number of possible behaviors
+ DoNotSwap, //!< Don't touch anything
+ Swap, //!< User-enforced swapping
+ SwapIfHostIsBigEndian, //!< Only swap if machine we execute on is big-endian
+ SwapIfHostIsLittleEndian, //!< Only swap if machine we execute on is little-endian
+ Count //!< Number of possible behaviors
};
class InMemorySerializer : public ISerializer
void doIvec(ivec* value) override;
void doRvec(rvec* value) override;
void doString(std::string* value) override;
+ void doOpaque(char* data, std::size_t size) override;
private:
class Impl;
void doIvec(ivec* value) override;
void doRvec(rvec* value) override;
void doString(std::string* value) override;
+ void doOpaque(char* data, std::size_t size) override;
private:
class Impl;
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2016,2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2016,2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#ifndef GMX_UTILITY_ISERIALIZER_H
#define GMX_UTILITY_ISERIALIZER_H
+#include <cstddef>
+
#include <string>
#include "gromacs/math/vectypes.h"
virtual bool reading() const = 0;
//! \brief Serialize values of different types.
///@{
- virtual void doBool(bool* value) = 0;
- virtual void doUChar(unsigned char* value) = 0;
- virtual void doChar(char* value) = 0;
- virtual void doUShort(unsigned short* value) = 0;
- virtual void doInt(int* value) = 0;
- virtual void doInt32(int32_t* value) = 0;
- virtual void doInt64(int64_t* value) = 0;
- virtual void doFloat(float* value) = 0;
- virtual void doDouble(double* value) = 0;
- virtual void doReal(real* value) = 0;
- virtual void doIvec(ivec* value) = 0;
- virtual void doRvec(rvec* value) = 0;
- virtual void doString(std::string* value) = 0;
+ virtual void doBool(bool* value) = 0;
+ virtual void doUChar(unsigned char* value) = 0;
+ virtual void doChar(char* value) = 0;
+ virtual void doUShort(unsigned short* value) = 0;
+ virtual void doInt(int* value) = 0;
+ virtual void doInt32(int32_t* value) = 0;
+ virtual void doInt64(int64_t* value) = 0;
+ virtual void doFloat(float* value) = 0;
+ virtual void doDouble(double* value) = 0;
+ virtual void doReal(real* value) = 0;
+ virtual void doIvec(ivec* value) = 0;
+ virtual void doRvec(rvec* value) = 0;
+ virtual void doString(std::string* value) = 0;
+ virtual void doOpaque(char* data, std::size_t size) = 0;
///@}
//! \brief Serialize arrays of values of different types.
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2019, by the GROMACS development team, led by
+ * Copyright (c) 2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
TEST_F(InMemorySerializerTest, RoundtripWithEndianessSwap)
{
- InMemorySerializer serializerWithSwap(EndianSwapBehavior::DoSwap);
+ InMemorySerializer serializerWithSwap(EndianSwapBehavior::Swap);
SerializerValues values = defaultValues_;
serialize(&serializerWithSwap, &values);
auto buffer = serializerWithSwap.finishAndGetBuffer();
InMemoryDeserializer deserializerWithSwap(buffer, std::is_same<real, double>::value,
- EndianSwapBehavior::DoSwap);
+ EndianSwapBehavior::Swap);
SerializerValues deserialisedValues = deserialize(&deserializerWithSwap);
TEST_F(InMemorySerializerTest, SerializerExplicitEndianessSwap)
{
- InMemorySerializer serializerWithSwap(EndianSwapBehavior::DoSwap);
+ InMemorySerializer serializerWithSwap(EndianSwapBehavior::Swap);
SerializerValues values = defaultValues_;
serialize(&serializerWithSwap, &values);
auto buffer = serializer.finishAndGetBuffer();
InMemoryDeserializer deserializerWithSwap(buffer, std::is_same<real, double>::value,
- EndianSwapBehavior::DoSwap);
+ EndianSwapBehavior::Swap);
SerializerValues deserialisedValues = deserialize(&deserializerWithSwap);
checkSerializerValuesforEquality(endianessSwappedValues_, deserialisedValues);
/*
* This file is part of the GROMACS molecular simulation package.
*
- * Copyright (c) 2016,2017,2018,2019, by the GROMACS development team, led by
+ * Copyright (c) 2016,2017,2018,2019,2020, by the GROMACS development team, led by
* Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
* and including many others, as listed in the AUTHORS file in the
* top-level source directory and at http://www.gromacs.org.
#include "gromacs/utility/keyvaluetreeserializer.h"
+#include <cstddef>
+
#include <gtest/gtest.h>
#include "gromacs/utility/inmemoryserializer.h"
void doFloat(float* value) override { checker_.checkFloat(*value, nullptr); }
void doDouble(double* value) override { checker_.checkDouble(*value, nullptr); }
void doString(std::string* value) override { checker_.checkString(*value, nullptr); }
+ void doOpaque(char* /* value */, std::size_t /* size */) override { raiseAssert(); }
void doReal(real* /* value */) override { raiseAssert(); }
void doIvec(ivec* /* value */) override { raiseAssert(); }
void doRvec(rvec* /* value */) override { raiseAssert(); }
biod.pnpi.spb.ru / alexxy / gromacs.git / commitdiff