Field3D
SparseFieldIO.h
Go to the documentation of this file.
1 //----------------------------------------------------------------------------//
2 
3 /*
4  * Copyright (c) 2009 Sony Pictures Imageworks Inc
5  *
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  *
12  * Redistributions of source code must retain the above copyright
13  * notice, this list of conditions and the following disclaimer.
14  * Redistributions in binary form must reproduce the above copyright
15  * notice, this list of conditions and the following disclaimer in the
16  * documentation and/or other materials provided with the
17  * distribution. Neither the name of Sony Pictures Imageworks nor the
18  * names of its contributors may be used to endorse or promote
19  * products derived from this software without specific prior written
20  * permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
25  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
26  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
27  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
29  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
33  * OF THE POSSIBILITY OF SUCH DAMAGE.
34  */
35 
36 //----------------------------------------------------------------------------//
37 
44 //----------------------------------------------------------------------------//
45 
46 #ifndef _INCLUDED_Field3D_SparseFieldIO_H_
47 #define _INCLUDED_Field3D_SparseFieldIO_H_
48 
49 //----------------------------------------------------------------------------//
50 
51 #include <string>
52 #include <cmath>
53 
54 #include <hdf5.h>
55 
56 #include "SparseDataReader.h"
57 #include "SparseField.h"
58 #include "SparseFile.h"
59 #include "FieldIO.h"
60 #include "Field3DFile.h"
61 
62 //----------------------------------------------------------------------------//
63 
64 #include "ns.h"
65 
67 
68 //----------------------------------------------------------------------------//
69 // SparseFieldIO
70 //----------------------------------------------------------------------------//
71 
77 //----------------------------------------------------------------------------//
78 
79 class SparseFieldIO : public FieldIO
80 {
81 
82 public:
83 
84  // Typedefs ------------------------------------------------------------------
85 
86  typedef boost::intrusive_ptr<SparseFieldIO> Ptr;
87 
88  // RTTI replacement ----------------------------------------------------------
89 
92 
93  const char *classType() const
94  {
95  return "SparseFieldIO";
96  }
97 
98  // Constructors --------------------------------------------------------------
99 
102  : FieldIO()
103  { }
104 
106  virtual ~SparseFieldIO()
107  { /* Empty */ }
108 
109 
111  { return Ptr(new SparseFieldIO); }
112 
113  // From FieldIO --------------------------------------------------------------
114 
118  virtual FieldBase::Ptr read(hid_t layerGroup, const std::string &filename,
119  const std::string &layerPath,
120  DataTypeEnum typeEnum);
121 
124  virtual bool write(hid_t layerGroup, FieldBase::Ptr field);
125 
127  virtual std::string className() const
128  { return "SparseField"; }
129 
130 private:
131 
132  // Internal methods ----------------------------------------------------------
133 
135  template <class Data_T>
136  bool writeInternal(hid_t layerGroup, typename SparseField<Data_T>::Ptr field);
137 
139  template <class Data_T>
140  bool readData(hid_t location,
141  int numBlocks,
142  const std::string &filename,
143  const std::string &layerPath,
144  typename SparseField<Data_T>::Ptr result);
145 
146  // Strings -------------------------------------------------------------------
147 
148  static const int k_versionNumber;
149  static const std::string k_versionAttrName;
150  static const std::string k_extentsStr;
151  static const std::string k_dataWindowStr;
152  static const std::string k_componentsStr;
153  static const std::string k_blockOrderStr;
154  static const std::string k_numBlocksStr;
155  static const std::string k_blockResStr;
156  static const std::string k_bitsPerComponentStr;
157  static const std::string k_numOccupiedBlocksStr;
158  static const std::string k_dataStr;
159 
160  // Typedefs ------------------------------------------------------------------
161 
163  typedef FieldIO base;
164 };
165 
166 //----------------------------------------------------------------------------//
167 // Template methods
168 //----------------------------------------------------------------------------//
169 
171 template <class Data_T>
172 bool SparseFieldIO::writeInternal(hid_t layerGroup,
173  typename SparseField<Data_T>::Ptr field)
174 {
175  using namespace std;
176  using namespace Exc;
177  using namespace Hdf5Util;
178  using namespace Sparse;
179 
180  Box3i ext(field->extents()), dw(field->dataWindow());
181 
182  int components = FieldTraits<Data_T>::dataDims();
183 
184  int valuesPerBlock = (1 << (field->m_blockOrder * 3)) * components;
185 
186  int size[3];
187  size[0] = dw.max.x - dw.min.x + 1;
188  size[1] = dw.max.y - dw.min.y + 1;
189  size[2] = dw.max.z - dw.min.z + 1;
190 
191 
192  hsize_t totalSize[1];
193  totalSize[0] = size[0] * size[1] * size[2] * components;
194 
195  // Add extents attribute ---
196 
197  int extents[6] =
198  { ext.min.x, ext.min.y, ext.min.z, ext.max.x, ext.max.y, ext.max.z };
199 
200  if (!writeAttribute(layerGroup, k_extentsStr, 6, extents[0])) {
201  Msg::print(Msg::SevWarning, "Error adding size attribute.");
202  return false;
203  }
204 
205  // Add data window attribute ---
206 
207  int dataWindow[6] =
208  { dw.min.x, dw.min.y, dw.min.z, dw.max.x, dw.max.y, dw.max.z };
209 
210  if (!writeAttribute(layerGroup, k_dataWindowStr, 6, dataWindow[0])) {
211  Msg::print(Msg::SevWarning, "Error adding size attribute.");
212  return false;
213  }
214 
215  // Add components attribute ---
216 
217  if (!writeAttribute(layerGroup, k_componentsStr, 1, components)) {
218  Msg::print(Msg::SevWarning, "Error adding components attribute.");
219  return false;
220  }
221 
222  // Add block order attribute ---
223 
224  int blockOrder = field->m_blockOrder;
225 
226  if (!writeAttribute(layerGroup, k_blockOrderStr, 1, blockOrder)) {
227  Msg::print(Msg::SevWarning, "Error adding block order attribute.");
228  return false;
229  }
230 
231  // Add number of blocks attribute ---
232 
233  V3i &blockRes = field->m_blockRes;
234  int numBlocks = blockRes.x * blockRes.y * blockRes.z;
235 
236  if (!writeAttribute(layerGroup, k_numBlocksStr, 1, numBlocks)) {
237  Msg::print(Msg::SevWarning, "Error adding number of blocks attribute.");
238  return false;
239  }
240 
241  // Add block resolution in each dimension ---
242 
243  if (!writeAttribute(layerGroup, k_blockResStr, 3, blockRes.x)) {
244  Msg::print(Msg::SevWarning, "Error adding block res attribute.");
245  return false;
246  }
247 
248  // Add the bits per component attribute ---
249 
250  int bits = DataTypeTraits<Data_T>::h5bits();
251  if (!writeAttribute(layerGroup, k_bitsPerComponentStr, 1, bits)) {
252  Msg::print(Msg::SevWarning, "Error adding bits per component attribute.");
253  return false;
254  }
255 
256  // Write the block info data sets ---
257 
258  // ... Write the isAllocated array
259  {
260  vector<char> isAllocated(numBlocks);
261  vector<char>::iterator i = isAllocated.begin();
262  typename vector<SparseBlock<Data_T> >::const_iterator b =
263  field->m_blocks.begin();
264  for (; i != isAllocated.end(); ++i, ++b)
265  *i = static_cast<char>(b->isAllocated);
266  writeSimpleData<char>(layerGroup, "block_is_allocated_data", isAllocated);
267  }
268 
269  // ... Write the emptyValue array
270  {
271  vector<Data_T> emptyValue(numBlocks);
272  typename vector<Data_T>::iterator i = emptyValue.begin();
273  typename vector<SparseBlock<Data_T> >::const_iterator b =
274  field->m_blocks.begin();
275  for (; i != emptyValue.end(); ++i, ++b)
276  *i = static_cast<Data_T>(b->emptyValue);
277  writeSimpleData<Data_T>(layerGroup, "block_empty_value_data", emptyValue);
278  }
279 
280  // Count the number of occupied blocks ---
281  int occupiedBlocks = 0;
282  typename vector<SparseBlock<Data_T> >::iterator b =
283  field->m_blocks.begin();
284  for (; b != field->m_blocks.end(); ++b) {
285  if (b->isAllocated)
286  occupiedBlocks++;
287  }
288  if (!writeAttribute(layerGroup, k_numOccupiedBlocksStr, 1, occupiedBlocks)) {
289  throw WriteAttributeException("Couldn't add attribute " +
291  }
292 
293  if (occupiedBlocks > 0) {
294 
295  // Make the memory data space
296  hsize_t memDims[1];
297  memDims[0] = valuesPerBlock;
298  H5ScopedScreate memDataSpace(H5S_SIMPLE);
299  H5Sset_extent_simple(memDataSpace.id(), 1, memDims, NULL);
300 
301  // Make the file data space
302  hsize_t fileDims[2];
303  fileDims[0] = occupiedBlocks;
304  fileDims[1] = valuesPerBlock;
305  H5ScopedScreate fileDataSpace(H5S_SIMPLE);
306  H5Sset_extent_simple(fileDataSpace.id(), 2, fileDims, NULL);
307 
308  // Set up gzip property list
309  bool gzipAvailable = checkHdf5Gzip();
310  hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
311  hsize_t chunkSize[2];
312  chunkSize[0] = 1;
313  chunkSize[1] = valuesPerBlock;
314  if (gzipAvailable) {
315  herr_t status = H5Pset_deflate(dcpl, 9);
316  if (status < 0) {
317  return false;
318  }
319  status = H5Pset_chunk(dcpl, 2, chunkSize);
320  if (status < 0) {
321  return false;
322  }
323  }
324 
325  // Add the data set
326  H5ScopedDcreate dataSet(layerGroup, k_dataStr,
328  fileDataSpace.id(),
329  H5P_DEFAULT, dcpl, H5P_DEFAULT);
330  if (dataSet.id() < 0)
331  throw CreateDataSetException("Couldn't create data set in "
332  "SparseFieldIO::writeInternal");
333 
334  // For each allocated block ---
335 
336  int nextBlockIdx = 0;
337  hsize_t offset[2];
338  hsize_t count[2];
339  herr_t status;
340 
341  for (b = field->m_blocks.begin(); b != field->m_blocks.end(); ++b) {
342  if (b->isAllocated) {
343  offset[0] = nextBlockIdx; // Index of next block
344  offset[1] = 0; // Index of first data in block. Always 0
345  count[0] = 1; // Number of columns to read. Always 1
346  count[1] = valuesPerBlock; // Number of values in one column
347  status = H5Sselect_hyperslab(fileDataSpace.id(), H5S_SELECT_SET,
348  offset, NULL, count, NULL);
349  if (status < 0) {
350  throw WriteHyperSlabException(
351  "Couldn't select slab " +
352  boost::lexical_cast<std::string>(nextBlockIdx));
353  }
354  Data_T *data = &b->data[0];
355  status = H5Dwrite(dataSet.id(), DataTypeTraits<Data_T>::h5type(),
356  memDataSpace.id(),
357  fileDataSpace.id(), H5P_DEFAULT, data);
358  if (status < 0) {
359  throw WriteHyperSlabException(
360  "Couldn't write slab " +
361  boost::lexical_cast<std::string>(nextBlockIdx));
362  }
363  // Increment nextBlockIdx
364  nextBlockIdx++;
365  }
366  }
367 
368  } // if occupiedBlocks > 0
369 
370  return true;
371 
372 }
373 
374 //----------------------------------------------------------------------------//
375 
376 template <class Data_T>
377 bool SparseFieldIO::readData(hid_t location,
378  int numBlocks,
379  const std::string &filename,
380  const std::string &layerPath,
381  typename SparseField<Data_T>::Ptr result)
382 {
383  using namespace std;
384  using namespace Exc;
385  using namespace Hdf5Util;
386  using namespace Sparse;
387 
388  int occupiedBlocks;
389 
390  bool dynamicLoading = SparseFileManager::singleton().doLimitMemUse();
391 
392  int components = FieldTraits<Data_T>::dataDims();
393  int valuesPerBlock = (1 << (result->m_blockOrder * 3)) * components;
394 
395  // Read the number of occupied blocks ---
396 
397  if (!readAttribute(location, k_numOccupiedBlocksStr, 1, occupiedBlocks))
398  throw MissingAttributeException("Couldn't find attribute: " +
400 
401  // Set up the dynamic read info ---
402 
403  if (dynamicLoading) {
404  // Set up the field reference
405  result->addReference(filename, layerPath,
406  valuesPerBlock,
407  occupiedBlocks);
408  }
409 
410  // Read the block info data sets ---
411 
412  // ... Read the isAllocated array
413 
414  {
415  vector<char> isAllocated(numBlocks);
416  vector<char>::iterator i = isAllocated.begin();
417  readSimpleData<char>(location, "block_is_allocated_data", isAllocated);
418  typename vector<SparseBlock<Data_T> >::iterator b =
419  result->m_blocks.begin();
420  typename vector<SparseBlock<Data_T> >::iterator bend =
421  result->m_blocks.end();
422  // We're assuming there are as many blocks in isAllocated as in the field.
423  for (; b != bend; ++b, ++i) {
424  b->isAllocated = static_cast<bool>(*i);
425  if (*i && !dynamicLoading) {
426  b->data.resize(valuesPerBlock);
427  }
428  }
429  }
430 
431  // ... Read the emptyValue array ---
432 
433  {
434  vector<Data_T> emptyValue(numBlocks);
435  readSimpleData<Data_T>(location, "block_empty_value_data", emptyValue);
436  typename vector<SparseBlock<Data_T> >::iterator b =
437  result->m_blocks.begin();
438  typename vector<SparseBlock<Data_T> >::iterator bend =
439  result->m_blocks.end();
440  typename vector<Data_T>::iterator i = emptyValue.begin();
441  // We're assuming there are as many blocks in isAllocated as in the field.
442  for (; b != bend; ++b, ++i) {
443  b->emptyValue = *i;
444  }
445  }
446 
447  // Read the data ---
448 
449  if (occupiedBlocks > 0) {
450 
451  if (dynamicLoading) {
452 
453  result->setupReferenceBlocks();
454 
455  } else {
456 
457  typename vector<SparseBlock<Data_T> >::iterator b =
458  result->m_blocks.begin();
459  typename vector<SparseBlock<Data_T> >::iterator bend =
460  result->m_blocks.end();
461 
462  SparseDataReader<Data_T> reader(location, valuesPerBlock, occupiedBlocks);
463 
464  // We'll read at most 50meg at a time
465  static const long maxMemPerPass = 50*1024*1024;
466 
467  for (int nextBlockIdx = 0;;) {
468 
469  long mem = 0;
470  std::vector<Data_T*> memoryList;
471 
472  for (; b != bend && mem < maxMemPerPass; ++b) {
473  if (b->isAllocated) {
474  mem += sizeof(Data_T)*valuesPerBlock;
475  memoryList.push_back(&b->data[0]);
476  }
477  }
478 
479  // all done.
480  if (!memoryList.size()) {
481  break;
482  }
483 
484  reader.readBlockList(nextBlockIdx, memoryList);
485  nextBlockIdx += memoryList.size();
486  }
487 
488  }
489 
490  } // if occupiedBlocks > 0
491 
492  return true;
493 
494 }
495 
496 //----------------------------------------------------------------------------//
497 
499 
500 //----------------------------------------------------------------------------//
501 
502 #endif