28 #include "splash/basetypes/basetypes.hpp" 29 #include "splash/DomainCollector.hpp" 30 #include "splash/core/DCDataSet.hpp" 31 #include "splash/core/DCGroup.hpp" 32 #include "splash/core/DCAttribute.hpp" 33 #include "splash/core/logging.hpp" 51 if ((fileStatus != FST_MERGING) && (fileStatus != FST_READING))
52 throw DCException(
"DomainCollector::getGlobalDomain: this access is not permitted");
55 if (fileStatus == FST_MERGING)
56 mpi_size.
set(mpiTopology);
58 mpi_size.
set(1, 1, 1);
62 Dimensions mpi_position(mpi_size[0] - 1, mpi_size[1] - 1, mpi_size[2] - 1);
67 readGlobalSizeFallback(
id, name, global_domain_size.
getPointer(), &mpi_position);
68 readGlobalOffsetFallback(
id, name, global_domain_offset.
getPointer(), &mpi_position);
70 return Domain(global_domain_offset, global_domain_size);
76 if ((fileStatus != FST_MERGING) && (fileStatus != FST_READING))
77 throw DCException(
"DomainCollector::getLocalDomain: this access is not permitted");
88 return Domain(domain_offset, domain_size);
91 void DomainCollector::writeDomainAttributes(
102 hid_t dset_handle = openDatasetHandle(
id, name, NULL);
104 DCAttribute::writeAttribute(DOMCOL_ATTR_CLASS, int_t.getDataType(),
105 dset_handle, &dataClass);
106 DCAttribute::writeAttribute(DOMCOL_ATTR_SIZE, dim_t.
getDataType(),
108 DCAttribute::writeAttribute(DOMCOL_ATTR_OFFSET, dim_t.
getDataType(),
110 DCAttribute::writeAttribute(DOMCOL_ATTR_GLOBAL_SIZE, dim_t.
getDataType(),
112 DCAttribute::writeAttribute(DOMCOL_ATTR_GLOBAL_OFFSET, dim_t.
getDataType(),
115 closeDatasetHandle(dset_handle);
118 bool DomainCollector::readDomainInfoForRank(
122 const Domain requestDomain,
127 hid_t dset_handle = openDatasetHandle(
id, name, &mpiPosition);
130 DCAttribute::readAttribute(DOMCOL_ATTR_OFFSET, dset_handle,
133 DCAttribute::readAttribute(DOMCOL_ATTR_SIZE, dset_handle,
136 DCAttribute::readAttribute(DOMCOL_ATTR_GLOBAL_OFFSET, dset_handle,
139 fileDomain.
getOffset() += global_domain_offset;
141 closeDatasetHandle(dset_handle);
151 void DomainCollector::readGridInternal(
156 const Domain &clientDomain,
157 const Domain &requestDomain
161 log_msg(3,
"dataclass = Grid");
167 std::stringstream group_id_name;
168 group_id_name << SDC_GROUP_DATA <<
"/" << id;
169 std::string group_id_string = group_id_name.str();
172 group.open(handles.get(mpiPosition), group_id_string);
174 size_t datatype_size = 0;
179 DCDataSet tmp_dataset(name);
180 tmp_dataset.open(group.getHandle());
182 datatype_size = tmp_dataset.getDataTypeSize();
183 dc_datatype = tmp_dataset.getDCDataType();
194 requestDomain, requestDomain.
getSize(),
195 datatype_size, dc_datatype);
197 dataContainer->
add(target_data);
209 size_t ndims = getNDims(handles.get(mpiPosition), id, name);
213 for (uint32_t i = 0; i < ndims; ++i)
215 dst_offset[i] = std::max((int64_t) clientDomain.
getOffset()[i] - (int64_t) request_offset[i], (int64_t) 0);
217 dst_offset[i] = std::max((int64_t) clientDomain.
getOffset()[i] -
218 (int64_t) request_offset[i], (int64_t) 0);
220 if (request_offset[i] <= client_start[i])
225 if (request_offset[i] + request_size[i] >= client_start[i] + client_size[i])
227 src_size[i] = client_size[i];
230 src_size[i] = request_offset[i] + request_size[i] - client_start[i];
234 src_offset[i] = request_offset[i] - client_start[i];
236 if (request_offset[i] + request_size[i] >= client_start[i] + client_size[i])
238 src_size[i] = client_size[i] - src_offset[i];
241 src_size[i] = request_offset[i] + request_size[i] -
242 (client_start[i] + src_offset[i]);
247 "clientDomain.getSize() = %s\n" 256 assert(src_size[0] <= request_size[0]);
257 assert(src_size[1] <= request_size[1]);
258 assert(src_size[2] <= request_size[2]);
262 uint32_t src_dims = 0;
263 readDataSet(handles.get(mpiPosition), id, name,
272 if (!(elements_read == src_size))
273 throw DCException(
"DomainCollector::readGridInternal: Sizes are not equal but should be (2).");
276 void DomainCollector::readPolyInternal(
282 const Domain &clientDomain,
287 log_msg(3,
"dataclass = Poly");
291 std::stringstream group_id_name;
292 group_id_name << SDC_GROUP_DATA <<
"/" << id;
293 std::string group_id_string = group_id_name.str();
296 group.open(handles.get(mpiPosition), group_id_string);
298 size_t datatype_size = 0;
303 DCDataSet tmp_dataset(name);
304 tmp_dataset.open(group.getHandle());
306 datatype_size = tmp_dataset.getDataTypeSize();
307 dc_datatype = tmp_dataset.getDCDataType();
318 dataSize, datatype_size, dc_datatype);
323 handles.get(mpiPosition), id, name,
331 uint32_t src_ndims = 0;
332 readCompleteDataSet(handles.get(mpiPosition), id, name,
340 if (!(size_read == dataSize))
341 throw DCException(
"DomainCollector::readPolyInternal: Sizes are not equal but should be (1).");
344 dataContainer->
add(client_data);
347 log_msg(3,
"skipping entry with 0 elements");
351 bool DomainCollector::readDomainDataForRank(
357 const Domain requestDomain,
361 log_msg(3,
"loading from mpi_position %s", mpiPosition.
toString().c_str());
363 bool readResult =
false;
364 Domain local_client_domain, global_client_domain;
369 hid_t dset_handle = openDatasetHandle(
id, name, &mpiPosition);
371 DCAttribute::readAttribute(DOMCOL_ATTR_OFFSET, dset_handle,
374 DCAttribute::readAttribute(DOMCOL_ATTR_SIZE, dset_handle,
377 DCAttribute::readAttribute(DOMCOL_ATTR_GLOBAL_OFFSET, dset_handle,
380 DCAttribute::readAttribute(DOMCOL_ATTR_GLOBAL_SIZE, dset_handle,
383 DCAttribute::readAttribute(DOMCOL_ATTR_CLASS, dset_handle,
386 closeDatasetHandle(dset_handle);
391 local_client_domain.
getSize());
393 readSizeInternal(handles.get(mpiPosition), id, name, data_size);
399 client_domain.toString().c_str(),
404 client_domain.getSize().getScalarSize() == 0);
406 if (tmp_data_class == GridType && data_size != client_domain.
getSize() &&
409 std::cout << data_size.
toString() <<
", " << client_domain.getSize().toString() << std::endl;
410 throw DCException(
"DomainCollector::readDomain: Size of data must match domain size for Grid data.");
413 if (*dataClass == UndefinedType)
415 *dataClass = tmp_data_class;
417 if (tmp_data_class != *dataClass)
419 throw DCException(
"DomainCollector::readDomain: Data classes in files are inconsistent!");
432 readPolyInternal(dataContainer, mpiPosition,
id, name,
433 data_size, client_domain, lazyLoad);
438 readGridInternal(dataContainer, mpiPosition,
id, name,
439 client_domain, requestDomain);
446 log_msg(3,
"no loading from this MPI position required");
459 if ((fileStatus != FST_MERGING) && (fileStatus != FST_READING))
460 throw DCException(
"DomainCollector::readDomain: this access is not permitted");
466 "requestDomain = %s ",
472 if (fileStatus == FST_MERGING)
474 mpi_size.
set(mpiTopology);
476 mpi_size.
set(1, 1, 1);
487 Dimensions delta(0, 0, 0);
488 Dimensions belowZero(min_dims);
489 Dimensions aboveZero(max_dims);
494 log_msg(4,
"find zero: belowZero = %s, aboveZero = %s",
499 readDomainInfoForRank(belowZero,
id, name,
500 Domain(Dimensions(0, 0, 0), Dimensions(0, 0, 0)), minDom);
501 readDomainInfoForRank(aboveZero,
id, name,
502 Domain(Dimensions(0, 0, 0), Dimensions(0, 0, 0)), maxDom);
504 log_msg(4,
"find zero: minDom.getOffset() = %s, maxDom.getOffset() = %s",
506 maxDom.getOffset().toString().c_str());
508 for (
size_t i = 0; i < DSP_DIM_MAX; ++i)
511 if (minDom.
getOffset()[i] > maxDom.getOffset()[i])
513 delta[i] = ceil(((
double) aboveZero[i] - (
double) belowZero[i]) / 2.0);
514 belowZero[i] += delta[i];
521 belowZero[i] -= delta[i];
522 aboveZero[i] -= delta[i];
526 }
while (delta != Dimensions(0, 0, 0));
533 readDomainInfoForRank(mpi_size - Dimensions(1, 1, 1),
id, name,
534 Domain(Dimensions(0, 0, 0), Dimensions(0, 0, 0)), lastDom);
535 for (
size_t i = 0; i < DSP_DIM_MAX; ++i)
537 if (request_offset[i] <= lastDom.
getBack()[i])
539 min_dims[i] = belowZero[i];
540 max_dims[i] = mpi_size[i] - 1;
544 belowZero[i] > 1 ? max_dims[i] = belowZero[i] - 1
550 Dimensions current_mpi_pos(min_dims);
551 Dimensions point_dim(1, 1, 1);
556 Dimensions last_mpi_pos(current_mpi_pos);
557 bool found_start =
false;
560 log_msg(4,
"find top-left: min_dims = %s, max_dims = %s",
565 last_mpi_pos = current_mpi_pos;
568 for (
size_t i = 0; i < DSP_DIM_MAX; ++i)
570 current_mpi_pos[i] = min_dims[i] +
571 ceil(((
double) max_dims[i] - (
double) min_dims[i]) / 2.0);
574 if (readDomainInfoForRank(current_mpi_pos,
id, name,
575 Domain(request_offset, point_dim), file_domain))
581 for (
size_t i = 0; i < DSP_DIM_MAX; ++i)
583 if (request_offset[i] >= file_domain.getOffset()[i])
584 min_dims[i] = current_mpi_pos[i];
586 if (request_offset[i] < file_domain.getOffset()[i])
587 max_dims[i] = current_mpi_pos[i] - 1;
589 }
while (last_mpi_pos != current_mpi_pos);
593 log_msg(2,
"readDomain: no data found");
594 return data_container;
603 for (
size_t i = 0; i < DSP_DIM_MAX; ++i)
604 max_dims[i] = (current_mpi_pos[i] + mpi_size[i] - 1) % mpi_size[i];
605 min_dims = current_mpi_pos;
607 log_msg(3,
"readDomain: Looking for matching domain data in range " 613 bool found_last_entry =
false;
616 size_t z, z_lin = min_dims[2];
619 z = z_lin % mpi_size[2];
620 size_t y, y_lin = min_dims[1];
623 y = y_lin % mpi_size[1];
624 size_t x, x_lin = min_dims[0];
627 x = x_lin % mpi_size[0];
628 Dimensions mpi_position(x, y, z);
630 if (!readDomainDataForRank(data_container,
642 if (z == min_dims[2])
644 if (y == min_dims[1])
646 max_dims[0] = (x + mpi_size[0] - 1) % mpi_size[0];
650 max_dims[1] = (y + mpi_size[1] - 1) % mpi_size[1];
655 found_last_entry =
true;
661 }
while (x != max_dims[0]);
663 if (found_last_entry)
667 }
while (y != max_dims[1]);
669 if (found_last_entry)
673 }
while (z != max_dims[2]);
675 if (dataClass != NULL)
676 *dataClass = data_class;
678 return data_container;
684 if (domainData == NULL)
686 throw DCException(
"DomainCollector::readDomainLazy: Invalid parameter, DomainData must not be NULL");
689 DomainH5Ref *loadingRef = domainData->getLoadingReference();
690 if (loadingRef == NULL)
692 throw DCException(
"DomainCollector::readDomainLazy: This DomainData does not allow lazy loading");
695 if (loadingRef->dataClass == UndefinedType)
697 throw DCException(
"DomainCollector::readDomainLazy: DomainData has invalid data class");
700 if (loadingRef->dataClass == PolyType)
703 uint32_t src_dims = 0;
704 readDataSet(loadingRef->handle,
706 loadingRef->name.c_str(),
707 loadingRef->dstBuffer,
708 loadingRef->dstOffset,
710 loadingRef->srcOffset,
713 domainData->getData());
715 if (!(elements_read == loadingRef->dstBuffer))
716 throw DCException(
"DomainCollector::readDomainLazy: Sizes are not equal but should be (1).");
721 throw DCException(
"DomainCollector::readDomainLazy: data class not supported");
731 const Domain globalDomain,
736 write(
id, type, ndims, select, name, buf);
737 writeDomainAttributes(
id, name, dataClass, localDomain, globalDomain);
745 const Domain globalDomain,
749 appendDomain(
id, type, count, 0, 1, name, localDomain, globalDomain, buf);
759 const Domain globalDomain,
766 FileStatusType old_file_status = fileStatus;
767 fileStatus = FST_READING;
772 readSizeInternal(handles.get(0), id, name, elements);
776 elements.
set(0, 1, 1);
779 fileStatus = old_file_status;
781 elements[0] = elements[0] + count;
783 append(
id, type, count, offset, striding, name, buf);
784 writeDomainAttributes(
id, name, PolyType, localDomain, globalDomain);
787 void DomainCollector::readGlobalSizeFallback(int32_t
id,
788 const char *dataName,
795 readAttributeInfo(
id, dataName, DOMCOL_ATTR_GLOBAL_SIZE, mpiPosition).
read(data,
sizeof(*data) * DSP_DIM_MAX);
798 hsize_t local_size[DSP_DIM_MAX];
801 for (
int i = 0; i < DSP_DIM_MAX; ++i)
802 data[i] = mpiTopology[i] * local_size[i];
806 void DomainCollector::readGlobalOffsetFallback(int32_t
id,
807 const char *dataName,
814 readAttributeInfo(
id, dataName, DOMCOL_ATTR_GLOBAL_OFFSET, mpiPosition).
read(data,
sizeof(*data) * DSP_DIM_MAX);
817 for (
int i = 0; i < DSP_DIM_MAX; ++i)
Domain getGlobalDomain(int32_t id, const char *name)
virtual ~DomainCollector()
std::string toString() const
void writeDomain(int32_t id, const CollectionType &type, uint32_t ndims, const Selection select, const char *name, const Domain localDomain, const Domain globalDomain, DomDataClass dataClass, const void *buf)
std::string toString() const
DataContainer * readDomain(int32_t id, const char *name, Domain domain, DomDataClass *dataClass, bool lazyLoad=false)
void append(int32_t id, const CollectionType &type, size_t count, const char *name, const void *data)
void readDomainLazy(DomainData *domainData)
Domain getLocalDomain(int32_t id, const char *name)
void set(hsize_t x, hsize_t y, hsize_t z)
Dimensions getBack() const
void setLoadingReference(int dataClass, hid_t handle, int32_t id, const char *name, const Dimensions dstBuffer, const Dimensions dstOffset, const Dimensions srcSize, const Dimensions srcOffset)
size_t getScalarSize() const
DomainData * getIndex(size_t index)
void write(int32_t id, const CollectionType &type, uint32_t ndims, const Selection select, const char *name, const void *data)
void add(DomainData *entry)
DomainCollector(uint32_t maxFileHandles)
static bool testIntersection(const Domain &d1, const Domain &d2)
const H5DataType & getDataType() const
EXTERN void log_msg(int level, const char *fmt,...)
void appendDomain(int32_t id, const CollectionType &type, size_t count, const char *name, const Domain localDomain, const Domain globalDomain, const void *buf)
void read(const CollectionType &colType, void *buf)
AttributeInfo readAttributeInfo(int32_t id, const char *dataName, const char *attrName, Dimensions *mpiPosition=NULL)
size_t getNumSubdomains()