Field3D
SparseFieldIO Class Reference

#include <SparseFieldIO.h>

Inheritance diagram for SparseFieldIO:
FieldIO RefBase

Public Types

typedef SparseFieldIO class_type
 
typedef boost::intrusive_ptr
< SparseFieldIO
Ptr
 
- Public Types inherited from FieldIO
typedef FieldIO class_type
 
typedef boost::intrusive_ptr
< FieldIO
Ptr
 
- Public Types inherited from RefBase
typedef boost::intrusive_ptr
< RefBase
Ptr
 

Public Member Functions

virtual std::string className () const
 Returns the class name.
 
const char * classType () const
 
virtual FieldBase::Ptr read (hid_t layerGroup, const std::string &filename, const std::string &layerPath, DataTypeEnum typeEnum)
 Reads the field at the given location and tries to create a SparseField object from it.
 
 SparseFieldIO ()
 Ctor.
 
virtual bool write (hid_t layerGroup, FieldBase::Ptr field)
 Writes the given field to disk.
 
virtual ~SparseFieldIO ()
 Dtor.
 
- Public Member Functions inherited from FieldIO
 FieldIO ()
 Ctor.
 
virtual ~FieldIO ()
 Dtor.
 
- Public Member Functions inherited from RefBase
void ref () const
 Used by boost::intrusive_pointer.
 
size_t refcnt ()
 Used by boost::intrusive_pointer.
 
void unref () const
 Used by boost::intrusive_pointer.
 
 RefBase ()
 
 RefBase (const RefBase &)
 Copy constructor.
 
RefBaseoperator= (const RefBase &)
 Assignment operator.
 
virtual ~RefBase ()
 Destructor.
 
virtual bool checkRTTI (const char *typenameStr)=0
 This function is only implemented by concrete classes and triggers the actual RTTI check through matchRTTI();.
 
bool matchRTTI (const char *typenameStr)
 Performs a check to see if the given typename string matches this class' This needs to be implemented in -all- subclasses, even abstract ones.
 

Static Public Member Functions

static FieldIO::Ptr create ()
 
- Static Public Member Functions inherited from FieldIO
static const char * classType ()
 
- Static Public Member Functions inherited from RefBase
static const char * classType ()
 

Public Attributes

 DEFINE_FIELD_RTTI_CONCRETE_CLASS
 
- Public Attributes inherited from FieldIO
 DEFINE_FIELD_RTTI_ABSTRACT_CLASS
 

Private Types

typedef FieldIO base
 Convenience typedef for referring to base class.
 

Private Member Functions

template<class Data_T >
bool readData (hid_t location, int numBlocks, const std::string &filename, const std::string &layerPath, typename SparseField< Data_T >::Ptr result)
 Reads the data that is dependent on the data type on disk.
 
template<class Data_T >
bool writeInternal (hid_t layerGroup, typename SparseField< Data_T >::Ptr field)
 This call writes all the attributes and sets up the data space.
 

Static Private Attributes

static const std::string k_bitsPerComponentStr
 
static const std::string k_blockOrderStr
 
static const std::string k_blockResStr
 
static const std::string k_componentsStr
 
static const std::string k_dataStr
 
static const std::string k_dataWindowStr
 
static const std::string k_extentsStr
 
static const std::string k_numBlocksStr
 
static const std::string k_numOccupiedBlocksStr
 
static const std::string k_versionAttrName
 
static const int k_versionNumber
 

Detailed Description

Defines the IO for a SparseField object

Definition at line 79 of file SparseFieldIO.h.

Member Typedef Documentation

typedef boost::intrusive_ptr<SparseFieldIO> SparseFieldIO::Ptr

Definition at line 86 of file SparseFieldIO.h.

Definition at line 90 of file SparseFieldIO.h.

typedef FieldIO SparseFieldIO::base
private

Convenience typedef for referring to base class.

Definition at line 163 of file SparseFieldIO.h.

Constructor & Destructor Documentation

SparseFieldIO::SparseFieldIO ( )
inline

Ctor.

Definition at line 101 of file SparseFieldIO.h.

: FieldIO()
{ }
virtual SparseFieldIO::~SparseFieldIO ( )
inlinevirtual

Dtor.

Definition at line 106 of file SparseFieldIO.h.

{ /* Empty */ }

Member Function Documentation

const char* SparseFieldIO::classType ( ) const
inline

Definition at line 93 of file SparseFieldIO.h.

{
return "SparseFieldIO";
}
static FieldIO::Ptr SparseFieldIO::create ( )
inlinestatic

Definition at line 110 of file SparseFieldIO.h.

Referenced by initIO().

{ return Ptr(new SparseFieldIO); }
FieldBase::Ptr SparseFieldIO::read ( hid_t  layerGroup,
const std::string &  filename,
const std::string &  layerPath,
DataTypeEnum  typeEnum 
)
virtual

Reads the field at the given location and tries to create a SparseField object from it.

Returns
Null if no object was read

Implements FieldIO.

Definition at line 84 of file SparseFieldIO.cpp.

References DataTypeDouble, DataTypeFloat, DataTypeHalf, DataTypeVecDouble, DataTypeVecFloat, DataTypeVecHalf, Msg::print(), Hdf5Util::readAttribute(), SparseField< Data_T >::setBlockOrder(), ResizableField< Data_T >::setSize(), and Msg::SevWarning.

{
Box3i extents, dataW;
int components;
int blockOrder;
int numBlocks;
V3i blockRes;
if (layerGroup == -1) {
Msg::print(Msg::SevWarning, "Bad layerGroup.");
return FieldBase::Ptr();
}
int version;
if (!readAttribute(layerGroup, k_versionAttrName, 1, version))
throw MissingAttributeException("Couldn't find attribute: " +
if (version != k_versionNumber)
throw UnsupportedVersionException("SparseField version not supported: " +
lexical_cast<std::string>(version));
if (!readAttribute(layerGroup, k_extentsStr, 6, extents.min.x))
throw MissingAttributeException("Couldn't find attribute: " +
if (!readAttribute(layerGroup, k_dataWindowStr, 6, dataW.min.x))
throw MissingAttributeException("Couldn't find attribute: " +
if (!readAttribute(layerGroup, k_componentsStr, 1, components))
throw MissingAttributeException("Couldn't find attribute: " +
// Read block order
if (!readAttribute(layerGroup, k_blockOrderStr, 1, blockOrder))
throw MissingAttributeException("Couldn't find attribute: " +
// Read number of blocks total
if (!readAttribute(layerGroup, k_numBlocksStr, 1, numBlocks))
throw MissingAttributeException("Couldn't find attribute: " +
// Read block resolution in each dimension
if (!readAttribute(layerGroup, k_blockResStr, 3, blockRes.x))
throw MissingAttributeException("Couldn't find attribute: " +
// ... Check that it matches the # reported by summing the active blocks
int numCalculatedBlocks = blockRes.x * blockRes.y * blockRes.z;
if (numCalculatedBlocks != numBlocks)
throw FileIntegrityException("Incorrect block count in SparseFieldIO::read");
// Call the appropriate read function based on the data type ---
int occupiedBlocks;
if (!readAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks))
throw MissingAttributeException("Couldn't find attribute: " +
// Check the data type ---
int bits;
if (!readAttribute(layerGroup, k_bitsPerComponentStr, 1, bits))
throw MissingAttributeException("Couldn't find attribute: " +
bool isHalf = false;
bool isFloat = false;
bool isDouble = false;
switch (bits) {
case 16:
isHalf = true;
break;
case 64:
isDouble = true;
break;
case 32:
default:
isFloat = true;
}
// Finally, read the data ---
if (components == 1) {
if (isHalf && typeEnum == DataTypeHalf) {
field->setSize(extents, dataW);
field->setBlockOrder(blockOrder);
readData<half>(layerGroup, numBlocks, filename, layerPath, field);
result = field;
} else if (isFloat && typeEnum == DataTypeFloat) {
field->setSize(extents, dataW);
field->setBlockOrder(blockOrder);
readData<float>(layerGroup, numBlocks, filename, layerPath, field);
result = field;
} else if (isDouble && typeEnum == DataTypeDouble) {
field->setSize(extents, dataW);
field->setBlockOrder(blockOrder);
readData<double>(layerGroup, numBlocks, filename, layerPath, field);
result = field;
}
} else if (components == 3) {
if (isHalf && typeEnum == DataTypeVecHalf) {
field->setSize(extents, dataW);
field->setBlockOrder(blockOrder);
readData<V3h>(layerGroup, numBlocks, filename, layerPath, field);
result = field;
} else if (isFloat && typeEnum == DataTypeVecFloat) {
field->setSize(extents, dataW);
field->setBlockOrder(blockOrder);
readData<V3f>(layerGroup, numBlocks, filename, layerPath, field);
result = field;
} else if (isDouble && typeEnum == DataTypeVecDouble) {
field->setSize(extents, dataW);
field->setBlockOrder(blockOrder);
readData<V3d>(layerGroup, numBlocks, filename, layerPath, field);
result = field;
}
}
return result;
}
bool SparseFieldIO::write ( hid_t  layerGroup,
FieldBase::Ptr  field 
)
virtual

Writes the given field to disk.

Returns
true if successful, otherwise false

Implements FieldIO.

Definition at line 223 of file SparseFieldIO.cpp.

References field_dynamic_cast(), Msg::print(), Msg::SevWarning, and Hdf5Util::writeAttribute().

{
if (layerGroup == -1) {
Msg::print(Msg::SevWarning, "Bad layerGroup.");
return false;
}
// Add version attribute
if (!writeAttribute(layerGroup, k_versionAttrName,
Msg::print(Msg::SevWarning, "Error adding version attribute.");
return false;
}
SparseField<double>::Ptr doubleField =
SparseField<V3h>::Ptr vecHalfField =
SparseField<V3f>::Ptr vecFloatField =
SparseField<V3d>::Ptr vecDoubleField =
bool success = true;
if (halfField) {
success = writeInternal<half>(layerGroup, halfField);
} else if (floatField) {
success = writeInternal<float>(layerGroup, floatField);
} else if (doubleField) {
success = writeInternal<double>(layerGroup, doubleField);
} else if (vecHalfField) {
success = writeInternal<V3h>(layerGroup, vecHalfField);
} else if (vecFloatField) {
success = writeInternal<V3f>(layerGroup, vecFloatField);
} else if (vecDoubleField) {
success = writeInternal<V3d>(layerGroup, vecDoubleField);
} else {
throw WriteLayerException("SparseFieldIO::write does not support the given "
"SparseField template parameter");
}
return success;
}
virtual std::string SparseFieldIO::className ( ) const
inlinevirtual

Returns the class name.

Implements FieldIO.

Definition at line 127 of file SparseFieldIO.h.

{ return "SparseField"; }
template<class Data_T >
bool SparseFieldIO::writeInternal ( hid_t  layerGroup,
typename SparseField< Data_T >::Ptr  field 
)
private

This call writes all the attributes and sets up the data space.

Todo:
Tune the chunk size of the gzip call

Definition at line 172 of file SparseFieldIO.h.

References Hdf5Util::checkHdf5Gzip(), FieldTraits< Data_T >::dataDims(), FieldRes::dataWindow(), FieldRes::extents(), DataTypeTraits< T >::h5bits(), DataTypeTraits< T >::h5type(), Hdf5Util::H5Base::id(), k_bitsPerComponentStr, k_blockOrderStr, k_blockResStr, k_componentsStr, k_dataStr, k_dataWindowStr, k_extentsStr, k_numBlocksStr, k_numOccupiedBlocksStr, SparseField< Data_T >::m_blockOrder, SparseField< Data_T >::m_blockRes, SparseField< Data_T >::m_blocks, Msg::print(), Msg::SevWarning, and Hdf5Util::writeAttribute().

{
using namespace std;
using namespace Exc;
using namespace Hdf5Util;
using namespace Sparse;
Box3i ext(field->extents()), dw(field->dataWindow());
int components = FieldTraits<Data_T>::dataDims();
int valuesPerBlock = (1 << (field->m_blockOrder * 3)) * components;
int size[3];
size[0] = dw.max.x - dw.min.x + 1;
size[1] = dw.max.y - dw.min.y + 1;
size[2] = dw.max.z - dw.min.z + 1;
hsize_t totalSize[1];
totalSize[0] = size[0] * size[1] * size[2] * components;
// Add extents attribute ---
int extents[6] =
{ ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z };
if (!writeAttribute(layerGroup, k_extentsStr, 6, extents[0])) {
Msg::print(Msg::SevWarning, "Error adding size attribute.");
return false;
}
// Add data window attribute ---
int dataWindow[6] =
{ dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z };
if (!writeAttribute(layerGroup, k_dataWindowStr, 6, dataWindow[0])) {
Msg::print(Msg::SevWarning, "Error adding size attribute.");
return false;
}
// Add components attribute ---
if (!writeAttribute(layerGroup, k_componentsStr, 1, components)) {
Msg::print(Msg::SevWarning, "Error adding components attribute.");
return false;
}
// Add block order attribute ---
int blockOrder = field->m_blockOrder;
if (!writeAttribute(layerGroup, k_blockOrderStr, 1, blockOrder)) {
Msg::print(Msg::SevWarning, "Error adding block order attribute.");
return false;
}
// Add number of blocks attribute ---
V3i &blockRes = field->m_blockRes;
int numBlocks = blockRes.x * blockRes.y * blockRes.z;
if (!writeAttribute(layerGroup, k_numBlocksStr, 1, numBlocks)) {
Msg::print(Msg::SevWarning, "Error adding number of blocks attribute.");
return false;
}
// Add block resolution in each dimension ---
if (!writeAttribute(layerGroup, k_blockResStr, 3, blockRes.x)) {
Msg::print(Msg::SevWarning, "Error adding block res attribute.");
return false;
}
// Add the bits per component attribute ---
if (!writeAttribute(layerGroup, k_bitsPerComponentStr, 1, bits)) {
Msg::print(Msg::SevWarning, "Error adding bits per component attribute.");
return false;
}
// Write the block info data sets ---
// ... Write the isAllocated array
{
vector<char> isAllocated(numBlocks);
vector<char>::iterator i = isAllocated.begin();
typename vector<SparseBlock<Data_T> >::const_iterator b =
field->m_blocks.begin();
for (; i != isAllocated.end(); ++i, ++b)
*i = static_cast<char>(b->isAllocated);
writeSimpleData<char>(layerGroup, "block_is_allocated_data", isAllocated);
}
// ... Write the emptyValue array
{
vector<Data_T> emptyValue(numBlocks);
typename vector<Data_T>::iterator i = emptyValue.begin();
typename vector<SparseBlock<Data_T> >::const_iterator b =
field->m_blocks.begin();
for (; i != emptyValue.end(); ++i, ++b)
*i = static_cast<Data_T>(b->emptyValue);
writeSimpleData<Data_T>(layerGroup, "block_empty_value_data", emptyValue);
}
// Count the number of occupied blocks ---
int occupiedBlocks = 0;
typename vector<SparseBlock<Data_T> >::iterator b =
field->m_blocks.begin();
for (; b != field->m_blocks.end(); ++b) {
if (b->isAllocated)
occupiedBlocks++;
}
if (!writeAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks)) {
throw WriteAttributeException("Couldn't add attribute " +
}
if (occupiedBlocks > 0) {
// Make the memory data space
hsize_t memDims[1];
memDims[0] = valuesPerBlock;
H5ScopedScreate memDataSpace(H5S_SIMPLE);
H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
// Make the file data space
hsize_t fileDims[2];
fileDims[0] = occupiedBlocks;
fileDims[1] = valuesPerBlock;
H5ScopedScreate fileDataSpace(H5S_SIMPLE);
H5Sset_extent_simple(fileDataSpace.id(), 2, fileDims, NULL);
// Set up gzip property list
bool gzipAvailable = checkHdf5Gzip();
hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
hsize_t chunkSize[2];
chunkSize[0] = 1;
chunkSize[1] = valuesPerBlock;
if (gzipAvailable) {
herr_t status = H5Pset_deflate(dcpl, 9);
if (status < 0) {
return false;
}
status = H5Pset_chunk(dcpl, 2, chunkSize);
if (status < 0) {
return false;
}
}
// Add the data set
H5ScopedDcreate dataSet(layerGroup, k_dataStr,
fileDataSpace.id(),
H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (dataSet.id() < 0)
throw CreateDataSetException("Couldn't create data set in "
"SparseFieldIO::writeInternal");
// For each allocated block ---
int nextBlockIdx = 0;
hsize_t offset[2];
hsize_t count[2];
herr_t status;
for (b = field->m_blocks.begin(); b != field->m_blocks.end(); ++b) {
if (b->isAllocated) {
offset[0] = nextBlockIdx; // Index of next block
offset[1] = 0; // Index of first data in block. Always 0
count[0] = 1; // Number of columns to read. Always 1
count[1] = valuesPerBlock; // Number of values in one column
status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
offset, NULL, count, NULL);
if (status < 0) {
throw WriteHyperSlabException(
"Couldn't select slab " +
boost::lexical_cast<std::string>(nextBlockIdx));
}
Data_T *data = &b->data[0];
status = H5Dwrite(dataSet.id(), DataTypeTraits<Data_T>::h5type(),
memDataSpace.id(),
fileDataSpace.id(), H5P_DEFAULT, data);
if (status < 0) {
throw WriteHyperSlabException(
"Couldn't write slab " +
boost::lexical_cast<std::string>(nextBlockIdx));
}
// Increment nextBlockIdx
nextBlockIdx++;
}
}
} // if occupiedBlocks > 0
return true;
}
template<class Data_T >
bool SparseFieldIO::readData ( hid_t  location,
int  numBlocks,
const std::string &  filename,
const std::string &  layerPath,
typename SparseField< Data_T >::Ptr  result 
)
private

Reads the data that is dependent on the data type on disk.

Definition at line 377 of file SparseFieldIO.h.

References SparseField< Data_T >::addReference(), FieldTraits< Data_T >::dataDims(), SparseFileManager::doLimitMemUse(), k_numOccupiedBlocksStr, SparseField< Data_T >::m_blockOrder, SparseField< Data_T >::m_blocks, Hdf5Util::readAttribute(), SparseDataReader< Data_T >::readBlockList(), SparseField< Data_T >::setupReferenceBlocks(), and SparseFileManager::singleton().

{
using namespace std;
using namespace Exc;
using namespace Hdf5Util;
using namespace Sparse;
int occupiedBlocks;
bool dynamicLoading = SparseFileManager::singleton().doLimitMemUse();
int components = FieldTraits<Data_T>::dataDims();
int valuesPerBlock = (1 << (result->m_blockOrder * 3)) * components;
// Read the number of occupied blocks ---
if (!readAttribute(location, k_numOccupiedBlocksStr, 1, occupiedBlocks))
throw MissingAttributeException("Couldn't find attribute: " +
// Set up the dynamic read info ---
if (dynamicLoading) {
// Set up the field reference
result->addReference(filename, layerPath,
valuesPerBlock,
occupiedBlocks);
}
// Read the block info data sets ---
// ... Read the isAllocated array
{
vector<char> isAllocated(numBlocks);
vector<char>::iterator i = isAllocated.begin();
readSimpleData<char>(location, "block_is_allocated_data", isAllocated);
typename vector<SparseBlock<Data_T> >::iterator b =
result->m_blocks.begin();
typename vector<SparseBlock<Data_T> >::iterator bend =
result->m_blocks.end();
// We're assuming there are as many blocks in isAllocated as in the field.
for (; b != bend; ++b, ++i) {
b->isAllocated = static_cast<bool>(*i);
if (*i && !dynamicLoading) {
b->data.resize(valuesPerBlock);
}
}
}
// ... Read the emptyValue array ---
{
vector<Data_T> emptyValue(numBlocks);
readSimpleData<Data_T>(location, "block_empty_value_data", emptyValue);
typename vector<SparseBlock<Data_T> >::iterator b =
result->m_blocks.begin();
typename vector<SparseBlock<Data_T> >::iterator bend =
result->m_blocks.end();
typename vector<Data_T>::iterator i = emptyValue.begin();
// We're assuming there are as many blocks in isAllocated as in the field.
for (; b != bend; ++b, ++i) {
b->emptyValue = *i;
}
}
// Read the data ---
if (occupiedBlocks > 0) {
if (dynamicLoading) {
result->setupReferenceBlocks();
} else {
typename vector<SparseBlock<Data_T> >::iterator b =
result->m_blocks.begin();
typename vector<SparseBlock<Data_T> >::iterator bend =
result->m_blocks.end();
SparseDataReader<Data_T> reader(location, valuesPerBlock, occupiedBlocks);
// We'll read at most 50meg at a time
static const long maxMemPerPass = 50*1024*1024;
for (int nextBlockIdx = 0;;) {
long mem = 0;
std::vector<Data_T*> memoryList;
for (; b != bend && mem < maxMemPerPass; ++b) {
if (b->isAllocated) {
mem += sizeof(Data_T)*valuesPerBlock;
memoryList.push_back(&b->data[0]);
}
}
// all done.
if (!memoryList.size()) {
break;
}
reader.readBlockList(nextBlockIdx, memoryList);
nextBlockIdx += memoryList.size();
}
}
} // if occupiedBlocks > 0
return true;
}

Member Data Documentation

SparseFieldIO::DEFINE_FIELD_RTTI_CONCRETE_CLASS

Definition at line 91 of file SparseFieldIO.h.

const int SparseFieldIO::k_versionNumber
staticprivate

Definition at line 148 of file SparseFieldIO.h.

const std::string SparseFieldIO::k_versionAttrName
staticprivate

Definition at line 149 of file SparseFieldIO.h.

const std::string SparseFieldIO::k_extentsStr
staticprivate

Definition at line 150 of file SparseFieldIO.h.

Referenced by writeInternal().

const std::string SparseFieldIO::k_dataWindowStr
staticprivate

Definition at line 151 of file SparseFieldIO.h.

Referenced by writeInternal().

const std::string SparseFieldIO::k_componentsStr
staticprivate

Definition at line 152 of file SparseFieldIO.h.

Referenced by writeInternal().

const std::string SparseFieldIO::k_blockOrderStr
staticprivate

Definition at line 153 of file SparseFieldIO.h.

Referenced by writeInternal().

const std::string SparseFieldIO::k_numBlocksStr
staticprivate

Definition at line 154 of file SparseFieldIO.h.

Referenced by writeInternal().

const std::string SparseFieldIO::k_blockResStr
staticprivate

Definition at line 155 of file SparseFieldIO.h.

Referenced by writeInternal().

const std::string SparseFieldIO::k_bitsPerComponentStr
staticprivate

Definition at line 156 of file SparseFieldIO.h.

Referenced by writeInternal().

const std::string SparseFieldIO::k_numOccupiedBlocksStr
staticprivate

Definition at line 157 of file SparseFieldIO.h.

Referenced by readData(), and writeInternal().

const std::string SparseFieldIO::k_dataStr
staticprivate

Definition at line 158 of file SparseFieldIO.h.

Referenced by writeInternal().


The documentation for this class was generated from the following files: