Field3D
SparseFieldIO.h
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
44 //----------------------------------------------------------------------------//
45 
46 #ifndef _INCLUDED_Field3D_SparseFieldIO_H_
47 #define _INCLUDED_Field3D_SparseFieldIO_H_
48 
49 //----------------------------------------------------------------------------//
50 
51 #include <string>
52 #include <cmath>
53 
54 #include <hdf5.h>
55 
56 #include "SparseDataReader.h"
57 #include "SparseField.h"
58 #include "SparseFile.h"
59 #include "FieldIO.h"
60 #include "Field3DFile.h"
61 
62 //----------------------------------------------------------------------------//
63 
64 #include "ns.h"
65 
67 
68 //----------------------------------------------------------------------------//
69 // SparseFieldIO
70 //----------------------------------------------------------------------------//
71 
77 //----------------------------------------------------------------------------//
78 
79 class SparseFieldIO : public FieldIO
80 {
81 
82 public:
83 
84  // Typedefs ------------------------------------------------------------------
85 
86  typedef boost::intrusive_ptr<SparseFieldIO> Ptr;
87 
88  // RTTI replacement ----------------------------------------------------------
89 
92 
93  const char *classType() const
94  {
95  return "SparseFieldIO";
96  }
97 
98  // Constructors --------------------------------------------------------------
99 
102  : FieldIO()
103  { }
104 
106  virtual ~SparseFieldIO()
107  { /* Empty */ }
108 
109 
111  { return Ptr(new SparseFieldIO); }
112 
113  // From FieldIO --------------------------------------------------------------
114 
118  virtual FieldBase::Ptr read(hid_t layerGroup, const std::string &filename,
119  const std::string &layerPath,
120  DataTypeEnum typeEnum);
121 
124  virtual bool write(hid_t layerGroup, FieldBase::Ptr field);
125 
127  virtual std::string className() const
128  { return "SparseField"; }
129 
130 private:
131 
132  // Internal methods ----------------------------------------------------------
133 
135  template <class Data_T>
136  bool writeInternal(hid_t layerGroup, typename SparseField<Data_T>::Ptr field);
137 
139  template <class Data_T>
140  bool readData(hid_t location,
141  int numBlocks,
142  const std::string &filename,
143  const std::string &layerPath,
144  typename SparseField<Data_T>::Ptr result);
145 
146  // Strings -------------------------------------------------------------------
147 
148  static const int k_versionNumber;
149  static const std::string k_versionAttrName;
150  static const std::string k_extentsStr;
151  static const std::string k_dataWindowStr;
152  static const std::string k_componentsStr;
153  static const std::string k_blockOrderStr;
154  static const std::string k_numBlocksStr;
155  static const std::string k_blockResStr;
156  static const std::string k_bitsPerComponentStr;
157  static const std::string k_numOccupiedBlocksStr;
158  static const std::string k_dataStr;
159 
160  // Typedefs ------------------------------------------------------------------
161 
163  typedef FieldIO base;
164 };
165 
166 //----------------------------------------------------------------------------//
167 // Template methods
168 //----------------------------------------------------------------------------//
169 
171 template <class Data_T>
172 bool SparseFieldIO::writeInternal(hid_t layerGroup,
173  typename SparseField<Data_T>::Ptr field)
174 {
175  using namespace std;
176  using namespace Exc;
177  using namespace Hdf5Util;
178  using namespace Sparse;
179 
180  Box3i ext(field->extents()), dw(field->dataWindow());
181 
182  int components = FieldTraits<Data_T>::dataDims();
183 
184  int valuesPerBlock = (1 << (field->m_blockOrder * 3)) * components;
185 
186  int size[3];
187  size[0] = dw.max.x - dw.min.x + 1;
188  size[1] = dw.max.y - dw.min.y + 1;
189  size[2] = dw.max.z - dw.min.z + 1;
190 
191 
192  hsize_t totalSize[1];
193  totalSize[0] = size[0] * size[1] * size[2] * components;
194 
195  // Add extents attribute ---
196 
197  int extents[6] =
198  { ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z };
199 
200  if (!writeAttribute(layerGroup, k_extentsStr, 6, extents[0])) {
201  Msg::print(Msg::SevWarning, "Error adding size attribute.");
202  return false;
203  }
204 
205  // Add data window attribute ---
206 
207  int dataWindow[6] =
208  { dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z };
209 
210  if (!writeAttribute(layerGroup, k_dataWindowStr, 6, dataWindow[0])) {
211  Msg::print(Msg::SevWarning, "Error adding size attribute.");
212  return false;
213  }
214 
215  // Add components attribute ---
216 
217  if (!writeAttribute(layerGroup, k_componentsStr, 1, components)) {
218  Msg::print(Msg::SevWarning, "Error adding components attribute.");
219  return false;
220  }
221 
222  // Add block order attribute ---
223 
224  int blockOrder = field->m_blockOrder;
225 
226  if (!writeAttribute(layerGroup, k_blockOrderStr, 1, blockOrder)) {
227  Msg::print(Msg::SevWarning, "Error adding block order attribute.");
228  return false;
229  }
230 
231  // Add number of blocks attribute ---
232 
233  V3i &blockRes = field->m_blockRes;
234  int numBlocks = blockRes.x * blockRes.y * blockRes.z;
235 
236  if (!writeAttribute(layerGroup, k_numBlocksStr, 1, numBlocks)) {
237  Msg::print(Msg::SevWarning, "Error adding number of blocks attribute.");
238  return false;
239  }
240 
241  // Add block resolution in each dimension ---
242 
243  if (!writeAttribute(layerGroup, k_blockResStr, 3, blockRes.x)) {
244  Msg::print(Msg::SevWarning, "Error adding block res attribute.");
245  return false;
246  }
247 
248  // Add the bits per component attribute ---
249 
250  int bits = DataTypeTraits<Data_T>::h5bits();
251  if (!writeAttribute(layerGroup, k_bitsPerComponentStr, 1, bits)) {
252  Msg::print(Msg::SevWarning, "Error adding bits per component attribute.");
253  return false;
254  }
255 
256  // Write the block info data sets ---
257 
258  // ... Write the isAllocated array
259  {
260  vector<char> isAllocated(numBlocks);
261  vector<char>::iterator i = isAllocated.begin();
262  typename vector<SparseBlock<Data_T> >::const_iterator b =
263  field->m_blocks.begin();
264  for (; i != isAllocated.end(); ++i, ++b)
265  *i = static_cast<char>(b->isAllocated);
266  writeSimpleData<char>(layerGroup, "block_is_allocated_data", isAllocated);
267  }
268 
269  // ... Write the emptyValue array
270  {
271  vector<Data_T> emptyValue(numBlocks);
272  typename vector<Data_T>::iterator i = emptyValue.begin();
273  typename vector<SparseBlock<Data_T> >::const_iterator b =
274  field->m_blocks.begin();
275  for (; i != emptyValue.end(); ++i, ++b)
276  *i = static_cast<Data_T>(b->emptyValue);
277  writeSimpleData<Data_T>(layerGroup, "block_empty_value_data", emptyValue);
278  }
279 
280  // Count the number of occupied blocks ---
281  int occupiedBlocks = 0;
282  typename vector<SparseBlock<Data_T> >::iterator b =
283  field->m_blocks.begin();
284  for (; b != field->m_blocks.end(); ++b) {
285  if (b->isAllocated)
286  occupiedBlocks++;
287  }
288  if (!writeAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks)) {
289  throw WriteAttributeException("Couldn't add attribute " +
291  }
292 
293  if (occupiedBlocks > 0) {
294 
295  // Make the memory data space
296  hsize_t memDims[1];
297  memDims[0] = valuesPerBlock;
298  H5ScopedScreate memDataSpace(H5S_SIMPLE);
299  H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
300 
301  // Make the file data space
302  hsize_t fileDims[2];
303  fileDims[0] = occupiedBlocks;
304  fileDims[1] = valuesPerBlock;
305  H5ScopedScreate fileDataSpace(H5S_SIMPLE);
306  H5Sset_extent_simple(fileDataSpace.id(), 2, fileDims, NULL);
307 
308  // Set up gzip property list
309  bool gzipAvailable = checkHdf5Gzip();
310  hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
311  hsize_t chunkSize[2];
312  chunkSize[0] = 1;
313  chunkSize[1] = valuesPerBlock;
314  if (gzipAvailable) {
315  herr_t status = H5Pset_deflate(dcpl, 9);
316  if (status < 0) {
317  return false;
318  }
319  status = H5Pset_chunk(dcpl, 2, chunkSize);
320  if (status < 0) {
321  return false;
322  }
323  }
324 
325  // Add the data set
326  H5ScopedDcreate dataSet(layerGroup, k_dataStr,
328  fileDataSpace.id(),
329  H5P_DEFAULT, dcpl, H5P_DEFAULT);
330  if (dataSet.id() < 0)
331  throw CreateDataSetException("Couldn't create data set in "
332  "SparseFieldIO::writeInternal");
333 
334  // For each allocated block ---
335 
336  int nextBlockIdx = 0;
337  hsize_t offset[2];
338  hsize_t count[2];
339  herr_t status;
340 
341  for (b = field->m_blocks.begin(); b != field->m_blocks.end(); ++b) {
342  if (b->isAllocated) {
343  offset[0] = nextBlockIdx; // Index of next block
344  offset[1] = 0; // Index of first data in block. Always 0
345  count[0] = 1; // Number of columns to read. Always 1
346  count[1] = valuesPerBlock; // Number of values in one column
347  status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
348  offset, NULL, count, NULL);
349  if (status < 0) {
350  throw WriteHyperSlabException(
351  "Couldn't select slab " +
352  boost::lexical_cast<std::string>(nextBlockIdx));
353  }
354  Data_T *data = &b->data[0];
355  status = H5Dwrite(dataSet.id(), DataTypeTraits<Data_T>::h5type(),
356  memDataSpace.id(),
357  fileDataSpace.id(), H5P_DEFAULT, data);
358  if (status < 0) {
359  throw WriteHyperSlabException(
360  "Couldn't write slab " +
361  boost::lexical_cast<std::string>(nextBlockIdx));
362  }
363  // Increment nextBlockIdx
364  nextBlockIdx++;
365  }
366  }
367 
368  } // if occupiedBlocks > 0
369 
370  return true;
371 
372 }
373 
374 //----------------------------------------------------------------------------//
375 
376 template <class Data_T>
377 bool SparseFieldIO::readData(hid_t location,
378  int numBlocks,
379  const std::string &filename,
380  const std::string &layerPath,
381  typename SparseField<Data_T>::Ptr result)
382 {
383  using namespace std;
384  using namespace Exc;
385  using namespace Hdf5Util;
386  using namespace Sparse;
387 
388  int occupiedBlocks;
389 
390  bool dynamicLoading = SparseFileManager::singleton().doLimitMemUse();
391 
392  int components = FieldTraits<Data_T>::dataDims();
393  int valuesPerBlock = (1 << (result->m_blockOrder * 3)) * components;
394 
395  // Read the number of occupied blocks ---
396 
397  if (!readAttribute(location, k_numOccupiedBlocksStr, 1, occupiedBlocks))
398  throw MissingAttributeException("Couldn't find attribute: " +
400 
401  // Set up the dynamic read info ---
402 
403  if (dynamicLoading) {
404  // Set up the field reference
405  result->addReference(filename, layerPath,
406  valuesPerBlock,
407  occupiedBlocks);
408  }
409 
410  // Read the block info data sets ---
411 
412  // ... Read the isAllocated array
413 
414  {
415  vector<char> isAllocated(numBlocks);
416  vector<char>::iterator i = isAllocated.begin();
417  readSimpleData<char>(location, "block_is_allocated_data", isAllocated);
418  typename vector<SparseBlock<Data_T> >::iterator b =
419  result->m_blocks.begin();
420  typename vector<SparseBlock<Data_T> >::iterator bend =
421  result->m_blocks.end();
422  // We're assuming there are as many blocks in isAllocated as in the field.
423  for (; b != bend; ++b, ++i) {
424  b->isAllocated = static_cast<bool>(*i);
425  if (*i && !dynamicLoading) {
426  b->data.resize(valuesPerBlock);
427  }
428  }
429  }
430 
431  // ... Read the emptyValue array ---
432 
433  {
434  vector<Data_T> emptyValue(numBlocks);
435  readSimpleData<Data_T>(location, "block_empty_value_data", emptyValue);
436  typename vector<SparseBlock<Data_T> >::iterator b =
437  result->m_blocks.begin();
438  typename vector<SparseBlock<Data_T> >::iterator bend =
439  result->m_blocks.end();
440  typename vector<Data_T>::iterator i = emptyValue.begin();
441  // We're assuming there are as many blocks in isAllocated as in the field.
442  for (; b != bend; ++b, ++i) {
443  b->emptyValue = *i;
444  }
445  }
446 
447  // Read the data ---
448 
449  if (occupiedBlocks > 0) {
450 
451  if (dynamicLoading) {
452 
453  result->setupReferenceBlocks();
454 
455  } else {
456 
457  typename vector<SparseBlock<Data_T> >::iterator b =
458  result->m_blocks.begin();
459  typename vector<SparseBlock<Data_T> >::iterator bend =
460  result->m_blocks.end();
461 
462  SparseDataReader<Data_T> reader(location, valuesPerBlock, occupiedBlocks);
463 
464  // We'll read at most 50meg at a time
465  static const long maxMemPerPass = 50*1024*1024;
466 
467  for (int nextBlockIdx = 0;;) {
468 
469  long mem = 0;
470  std::vector<Data_T*> memoryList;
471 
472  for (; b != bend && mem < maxMemPerPass; ++b) {
473  if (b->isAllocated) {
474  mem += sizeof(Data_T)*valuesPerBlock;
475  memoryList.push_back(&b->data[0]);
476  }
477  }
478 
479  // all done.
480  if (!memoryList.size()) {
481  break;
482  }
483 
484  reader.readBlockList(nextBlockIdx, memoryList);
485  nextBlockIdx += memoryList.size();
486  }
487 
488  }
489 
490  } // if occupiedBlocks > 0
491 
492  return true;
493 
494 }
495 
496 //----------------------------------------------------------------------------//
497 
499 
500 //----------------------------------------------------------------------------//
501 
502 #endif
#define FIELD3D_NAMESPACE_HEADER_CLOSE
Definition: ns.h:58
SparseFieldIO()
Ctor.
Imath::Box3i Box3i
Definition: SpiMathLib.h:77
void addReference(const std::string &filename, const std::string &layerPath, int valuesPerBlock, int occupiedBlocks)
Internal function to create a Reference for the current field, for use in dynamic reading...
Definition: SparseField.h:1089
V3i m_blockRes
Block array resolution.
Definition: SparseField.h:396
const Box3i & extents() const
Returns the extents of the data. This signifies the relevant area that the data exists over...
Definition: Field.h:251
Contains the Field3DFile classesOSS sanitized.
static int dataDims()
Dimensions of the given data type. i.e. 3 for V3f, 1 for float.
static const std::string k_dataStr
void readBlockList(int idx, const std::vector< Data_T * > &memoryList)
Reads a series of blocks, storing each block of data in memoryList, which is assumed to contain enoug...
virtual std::string className() const
Returns the class name.
static const std::string k_versionAttrName
Contains functions controlling the loading of sparse fields.
virtual bool write(hid_t layerGroup, FieldBase::Ptr field)
Writes the given field to disk.
static const std::string k_dataWindowStr
void setupReferenceBlocks()
Internal function to setup the Reference&#39;s block pointers, for use with dynamic reading.
Definition: SparseField.h:1128
boost::intrusive_ptr< FieldBase > Ptr
Definition: Field.h:97
static const std::string k_numOccupiedBlocksStr
static const std::string k_extentsStr
virtual ~SparseFieldIO()
Dtor.
DEFINE_FIELD_RTTI_CONCRETE_CLASS
Definition: SparseFieldIO.h:91
void print(Severity severity, const std::string &message)
Sends the string to the assigned output, prefixing the message with the severity. ...
Definition: Log.cpp:62
int m_blockOrder
Block order (size = 2^blockOrder)
Definition: SparseField.h:394
boost::intrusive_ptr< SparseField > Ptr
Definition: SparseField.h:167
std::vector< Block > m_blocks
Information for all blocks in the field.
Definition: SparseField.h:400
boost::intrusive_ptr< SparseFieldIO > Ptr
Definition: SparseFieldIO.h:86
static const int k_versionNumber
Imath::V3i V3i
Definition: SpiMathLib.h:71
This class gets used by SparseFieldIO and SparseFileManager to read the block data. On creation it will open the data set and not close it until the object is destroyed.
bool readAttribute(hid_t location, const std::string &attrName, std::string &value)
Reads a string attribute.
bool checkHdf5Gzip()
Checks whether gzip is available in the current hdf5 library.
Definition: Hdf5Util.cpp:680
virtual FieldBase::Ptr read(hid_t layerGroup, const std::string &filename, const std::string &layerPath, DataTypeEnum typeEnum)
Reads the field at the given location and tries to create a SparseField object from it...
static const std::string k_blockOrderStr
static FieldIO::Ptr create()
static hid_t h5type()
static const std::string k_numBlocksStr
static const std::string k_componentsStr
const char * classType() const
Definition: SparseFieldIO.h:93
static const std::string k_blockResStr
bool writeAttribute(hid_t location, const std::string &attrName, const std::string &value)
Writes a string attribute.
Contains functions controlling the loading of sparse fields.
static int h5bits()
bool doLimitMemUse() const
Returns whether to limit memory usage and do dynamic loading for sparse fields.
Definition: SparseFile.cpp:80
SparseFieldIO class_type
Definition: SparseFieldIO.h:90
Scoped object - creates a dataspace on creation and closes it on destruction.
Definition: Hdf5Util.h:213
static const std::string k_bitsPerComponentStr
FieldIO base
Convenience typedef for referring to base class.
Contains the SparseField class.
Scoped object - creates a dataset on creation and closes it on destruction.
Definition: Hdf5Util.h:240
boost::intrusive_ptr< FieldIO > Ptr
Definition: FieldIO.h:90
DataTypeEnum
Definition: Traits.h:66
static SparseFileManager & singleton()
Returns a reference to the singleton instance.
Definition: SparseFile.cpp:63
const Box3i & dataWindow() const
Returns the data window. Any coordinate inside this window is safe to pass to value() in the Field su...
Definition: Field.h:255
Contains FieldIO class.
bool writeInternal(hid_t layerGroup, typename SparseField< Data_T >::Ptr field)
This call writes all the attributes and sets up the data space.
bool readData(hid_t location, int numBlocks, const std::string &filename, const std::string &layerPath, typename SparseField< Data_T >::Ptr result)
Reads the data that is dependent on the data type on disk.
hid_t id() const
Query the hid_t value.
Definition: Hdf5Util.h:90
#define FIELD3D_NAMESPACE_OPEN
Definition: ns.h:56