#include <ParallelDataCollector.hpp>

Inheritance diagram for splash::ParallelDataCollector:

Collaboration diagram for splash::ParallelDataCollector:

Public Member Functions
	ParallelDataCollector (MPI_Comm comm, MPI_Info info, const Dimensions topology, uint32_t maxFileHandles)

virtual	~ParallelDataCollector ()

void	open (const char *filename, FileCreationAttr &attr) throw (DCException)

void	close ()

int32_t	getMaxID ()

void	getMPISize (Dimensions &mpiSize)

void	getEntryIDs (int32_t ids, size_t count) throw (DCException)

void	getEntriesForID (int32_t id, DCEntry entries, size_t count) throw (DCException)

void	write (int32_t id, const CollectionType &type, uint32_t rank, const Selection select, const char name, const void buf) throw (DCException)

void	write (int32_t id, const Dimensions globalSize, const Dimensions globalOffset, const CollectionType &type, uint32_t rank, const Selection select, const char name, const void buf)

void	reserve (int32_t id, const Dimensions globalSize, uint32_t rank, const CollectionType &type, const char *name) throw (DCException)

void	reserve (int32_t id, const Dimensions size, Dimensions globalSize, Dimensions globalOffset, uint32_t rank, const CollectionType &type, const char *name) throw (DCException)

void	append (int32_t id, const Dimensions size, uint32_t rank, const Dimensions globalOffset, const char name, const void buf)

void	remove (int32_t id) throw (DCException)

void	remove (int32_t id, const char *name) throw (DCException)

void	createReference (int32_t srcID, const char srcName, int32_t dstID, const char dstName) throw (DCException)

AttributeInfo	readGlobalAttributeInfo (int32_t id, const char name, Dimensions mpiPosition=NULL) throw (DCException)

void	readGlobalAttribute (int32_t id, const char name, void buf) throw (DCException)

void	writeGlobalAttribute (int32_t id, const CollectionType &type, const char name, const void buf) throw (DCException)

void	writeGlobalAttribute (int32_t id, const CollectionType &type, const char name, uint32_t ndims, const Dimensions dims, const void buf) throw (DCException)

AttributeInfo	readAttributeInfo (int32_t id, const char dataName, const char attrName, Dimensions *mpiPosition=NULL) throw (DCException)

void	readAttribute (int32_t id, const char dataName, const char attrName, void buf, Dimensions mpiPosition=NULL) throw (DCException)

void	writeAttribute (int32_t id, const CollectionType &type, const char dataName, const char attrName, const void *buf) throw (DCException)

void	writeAttribute (int32_t id, const CollectionType &type, const char dataName, const char attrName, uint32_t ndims, const Dimensions dims, const void *buf) throw (DCException)

void	read (int32_t id, const char name, Dimensions &sizeRead, void buf) throw (DCException)

void	read (int32_t id, const char name, const Dimensions dstBuffer, const Dimensions dstOffset, Dimensions &sizeRead, void buf) throw (DCException)

CollectionType *	readMeta (int32_t id, const char *name, const Dimensions dstBuffer, const Dimensions dstOffset, Dimensions &sizeRead) throw (DCException)

void	read (int32_t id, const Dimensions localSize, const Dimensions globalOffset, const char name, Dimensions &sizeRead, void buf) throw (DCException)

void	read (int32_t id, const Dimensions localSize, const Dimensions globalOffset, const char name, const Dimensions dstBuffer, const Dimensions dstOffset, Dimensions &sizeRead, void buf) throw (DCException)

void	finalize (void)

Public Member Functions inherited from splash::DataCollector
virtual	~DataCollector ()

Additional Inherited Members
Public Types inherited from splash::DataCollector
enum	FileAccType { FAT_CREATE, FAT_READ, FAT_READ_MERGED, FAT_WRITE }

typedef struct splash::DataCollector::_FileCreationAttr	FileCreationAttr

typedef struct splash::DataCollector::_DCEntry	DCEntry

Static Public Member Functions inherited from splash::DataCollector
static void	initFileCreationAttr (FileCreationAttr &attr)

Detailed Description

Realizes an IParallelDataCollector which creates a single HDF5 file per iteration for all MPI processes and accesses the file using collective MPI I/O.

Definition at line 49 of file ParallelDataCollector.hpp.

Constructor & Destructor Documentation

◆ ParallelDataCollector()

splash::ParallelDataCollector::ParallelDataCollector	(	MPI_Comm	comm,
		MPI_Info	info,
		const Dimensions	topology,
		uint32_t	maxFileHandles
	)

Constructor

Parameters

comm	The communicator. All processes in this communicator must participate in accessing data.
info	The MPI_Info object.
topology	Number of MPI processes in each dimension.
maxFileHandles	Maximum number of concurrently opened file handles (0=infinite).

Definition at line 156 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

◆ ~ParallelDataCollector()

splash::ParallelDataCollector::~ParallelDataCollector ( )

virtual

Destructor

Definition at line 196 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

Member Function Documentation

◆ append()

void splash::ParallelDataCollector::append	(	int32_t	id,
		const Dimensions	size,
		uint32_t	rank,
		const Dimensions	globalOffset,
		const char *	name,
		const void *	buf
	)

virtual

Appends (1-3) dimensional data to a dataset created with IParallelDataCollector::reserve.

Parameters

id	ID for iteration.
size	Size of the data to be appended.
rank	Number of dimensions (1-3),
globalOffset	Offset in destination dataset to append at.
name	Name for the dataset to append to.
buf	Buffer to append.

Implements splash::IParallelDataCollector.

Definition at line 716 of file ParallelDataCollector.cpp.

Here is the caller graph for this function:

◆ close()

void splash::ParallelDataCollector::close ( )

virtual

Closes open files and releases internal buffers.

Must be called by user when finished writing/reading data.

Implements splash::DataCollector.

Definition at line 241 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ createReference()

void splash::ParallelDataCollector::createReference	(	int32_t	srcID,
		const char *	srcName,
		int32_t	dstID,
		const char *	dstName
	)
throw	(	DCException
	)

virtual

Creates an object reference to an existing dataset in the same HDF5 file.

Parameters

srcID	ID of the iteration holding the source dataset.
srcName	Name of the existing source dataset.
dstID	ID of the simulation iteration holding the created reference dataset. If this group does not exist, it is created.
dstName	Name of the created reference.

Implements splash::DataCollector.

Definition at line 793 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

◆ finalize()

void splash::ParallelDataCollector::finalize ( void )

virtual

Finalizes by freeing all MPI resources. Must be called before MPI_Finalize.

Implements splash::IParallelDataCollector.

Definition at line 203 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ getEntriesForID()

void splash::ParallelDataCollector::getEntriesForID	(	int32_t	id,
		DCEntry *	entries,
		size_t *	count
	)
throw	(	DCException
	)

virtual

Returns all datasets for an ID. The caller must ensure that there is enough space in entries to hold count elements.

Parameters

id	ID for iteration.
entries	Pointer to an array with at least `count` elements, can be NULL.
count	Returns the number of elements in `entries`, can be NULL.

Implements splash::DataCollector.

Definition at line 293 of file ParallelDataCollector.cpp.

Here is the caller graph for this function:

◆ getEntryIDs()

void splash::ParallelDataCollector::getEntryIDs	(	int32_t *	ids,
		size_t *	count
	)
throw	(	DCException
	)

virtual

Returns all IDs in an opened file. The caller must ensure that there is enough space in ids to hold count elements.

Parameters

ids	Pointer to an array with at least `count` entries, can be NULL.
count	Returns the number of entries in `ids`, can be NULL.

Implements splash::DataCollector.

Definition at line 272 of file ParallelDataCollector.cpp.

◆ getMaxID()

int32_t splash::ParallelDataCollector::getMaxID ( )

virtual

Returns: Returns highest iteration ID.

Implements splash::DataCollector.

Definition at line 256 of file ParallelDataCollector.cpp.

Here is the caller graph for this function:

◆ getMPISize()

void splash::ParallelDataCollector::getMPISize ( Dimensions & mpiSize )

virtual

Returns the MPI topology used for creating files.

Parameters

mpiSize Contains MPI topology.

Implements splash::DataCollector.

Definition at line 267 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

◆ open()

void splash::ParallelDataCollector::open	(	const char *	filename,
		FileCreationAttr &	attr
	)
throw	(	DCException
	)

virtual

Opens one or multiple files.

Parameters

filename	Name of the file(s) to open (the common part without index identifiers).
attr	Struct passing several parameters on how to access files.

Implements splash::DataCollector.

Definition at line 213 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

◆ read() [1/4]

void splash::ParallelDataCollector::read	(	int32_t	id,
		const char *	name,
		Dimensions &	sizeRead,
		void *	buf
	)
throw	(	DCException
	)

virtual

Reads data from HDF5 file. If data is to be read (instead of only its size in the file), the destination buffer (buf) must be allocated already.

Parameters

id	ID for iteration.
name	Name for the dataset.
sizeRead	Returns the size of the data in the file.
buf	Buffer to read from file, can be NULL.

Implements splash::DataCollector.

Definition at line 565 of file ParallelDataCollector.cpp.

Here is the caller graph for this function:

◆ read() [2/4]

void splash::ParallelDataCollector::read	(	int32_t	id,
		const char *	name,
		const Dimensions	dstBuffer,
		const Dimensions	dstOffset,
		Dimensions &	sizeRead,
		void *	buf
	)
throw	(	DCException
	)

virtual

Reads data from HDF5 file. If data is to be read (instead of only its size in the file), the destination buffer (buf) must be allocated already.

Parameters

id	ID for iteration.
name	Name for the dataset.
dstBuffer	Size of the buffer `buf` to read to.
dstOffset	Offset in destination buffer to read to.
sizeRead	Returns the size of the data in the file.
buf	Buffer to read from file, can be NULL.

Implements splash::DataCollector.

Definition at line 574 of file ParallelDataCollector.cpp.

◆ read() [3/4]

void splash::ParallelDataCollector::read	(	int32_t	id,
		const Dimensions	localSize,
		const Dimensions	globalOffset,
		const char *	name,
		Dimensions &	sizeRead,
		void *	buf
	)
throw	(	DCException
	)

Reads data from HDF5 file. If data is to be read (instead of only its size in the file), the destination buffer (buf) must be allocated already.

Parameters

id	ID for iteration.
localSize	Size of data to be read, starting at `globalOffset`.
globalOffset	Global offset in source data to start reading from.
name	Name for the dataset.
sizeRead	Returns the size of the data in the file.
buf	Buffer to read from file, can be NULL.

Definition at line 590 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

◆ read() [4/4]

void splash::ParallelDataCollector::read	(	int32_t	id,
		const Dimensions	localSize,
		const Dimensions	globalOffset,
		const char *	name,
		const Dimensions	dstBuffer,
		const Dimensions	dstOffset,
		Dimensions &	sizeRead,
		void *	buf
	)
throw	(	DCException
	)

Reads data from HDF5 file. If data is to be read (instead of only its size in the file), the destination buffer (buf) must be allocated already.

Parameters

id	ID for iteration.
localSize	Size of data to be read, starting at `globalOffset`.
globalOffset	Global offset in source data to start reading from.
name	Name for the dataset.
dstBuffer	Size of destination buffer.
dstOffset	Offset in destination buffer to read to.
sizeRead	Returns the size of the data in the file.
buf	Buffer to read from file, can be NULL.

Definition at line 601 of file ParallelDataCollector.cpp.

◆ readAttribute()

void splash::ParallelDataCollector::readAttribute	(	int32_t	id,
		const char *	dataName,
		const char *	attrName,
		void *	buf,
		Dimensions *	mpiPosition = `NULL`
	)
throw	(	DCException
	)

virtual

Reads an attribute from a single dataset.

Parameters

id	ID for iteration.
dataName	Name of the dataset in group `id` to read attribute from. If dataName is NULL, the attribute is read from the iteration group.
attrName	Name of the attribute.
buf	Buffer to read attribute to.
mpiPosition	Pointer to Dimensions class. Identifies MPI-position-specific file while in FAT_READ_MERGED mode. Use NULL to read from default file index.

Implements splash::DataCollector.

Definition at line 457 of file ParallelDataCollector.cpp.

◆ readAttributeInfo()

AttributeInfo splash::ParallelDataCollector::readAttributeInfo	(	int32_t	id,
		const char *	dataName,
		const char *	attrName,
		Dimensions *	mpiPosition = `NULL`
	)
throw	(	DCException
	)

virtual

Reads attribute meta data from a single dataset.

Parameters

id	ID for iteration.
dataName	Name of the dataset in group `id` to read attribute from. If dataName is NULL, the attribute is read from the iteration group.
attrName	Name of the attribute.
mpiPosition	Pointer to Dimensions class. Identifies MPI-position-specific file while in FAT_READ_MERGED mode. Use NULL to read from default file index.

Returns: Pointer to heap allocated DCAttributeInfo instance or NULL if not found. Must be freed by the caller.

Implements splash::DataCollector.

Definition at line 435 of file ParallelDataCollector.cpp.

Here is the caller graph for this function:

◆ readGlobalAttribute()

void splash::ParallelDataCollector::readGlobalAttribute	(	int32_t	id,
		const char *	name,
		void *	buf
	)
throw	(	DCException
	)

virtual

Reads global attribute from HDF5 file.

Parameters

id	ID for iteration.
name	Name of the attribute.
buf	Destination buffer for attribute.

Implements splash::IParallelDataCollector.

Definition at line 339 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ readGlobalAttributeInfo()

AttributeInfo splash::ParallelDataCollector::readGlobalAttributeInfo	(	int32_t	id,
		const char *	name,
		Dimensions *	mpiPosition = `NULL`
	)
throw	(	DCException
	)

virtual

Reads global attribute meta information from HDF5 file.

Parameters

id	ID for iteration.
name	Name for the attribute.
mpiPosition	Pointer to Dimensions class. Identifies MPI-position-specific custom group. Use NULL to read from default group.

Returns: Pointer to heap allocated DCAttributeInfo instance or NULL if not found. Must be freed by the caller.

Implements splash::DataCollector.

Definition at line 314 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

◆ readMeta()

CollectionType * splash::ParallelDataCollector::readMeta	(	int32_t	id,
		const char *	name,
		const Dimensions	dstBuffer,
		const Dimensions	dstOffset,
		Dimensions &	sizeRead
	)
throw	(	DCException
	)

virtual

Reads meta data from HDF5 file.

Parameters

id	ID of iteration.
name	Name of the dataset.
dstBuffer	Size of the dataset.
dstOffset	Offset in destination dataset.
sizeRead	Returns the size of the data in the file.

Returns: The CollectionType of the dataset as a heap allocated object. The object must be freed by the caller at the end of the object's lifetime.

Implements splash::DataCollector.

Definition at line 618 of file ParallelDataCollector.cpp.

◆ remove() [1/2]

void splash::ParallelDataCollector::remove	(	int32_t	id	)
throw	(	DCException
	)

virtual

Removes a simulation iteration from the HDF5 file.

Recursively removes all datasets and attributes within the iteration.

Parameters

id	ID to remove.

Implements splash::DataCollector.

Definition at line 753 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

◆ remove() [2/2]

void splash::ParallelDataCollector::remove	(	int32_t	id,
		const char *	name
	)
throw	(	DCException
	)

virtual

Removes a dataset from a HDF5 file.

Parameters

id	ID holding the dataset to be removed.
name	Name of the dataset to be removed.

Implements splash::DataCollector.

Definition at line 770 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

◆ reserve() [1/2]

void splash::ParallelDataCollector::reserve	(	int32_t	id,
		const Dimensions	globalSize,
		uint32_t	rank,
		const CollectionType &	type,
		const char *	name
	)
throw	(	DCException
	)

virtual

Reserves a dataset for parallel access.

Parameters

id	ID for iteration.
globalSize	Size to reserve for global data.
rank	Number of dimensions (1-3).
type	Type information for data.
name	Name for the dataset.

Implements splash::IParallelDataCollector.

Definition at line 669 of file ParallelDataCollector.cpp.

Here is the caller graph for this function:

◆ reserve() [2/2]

void splash::ParallelDataCollector::reserve	(	int32_t	id,
		const Dimensions	size,
		Dimensions *	globalSize,
		Dimensions *	globalOffset,
		uint32_t	rank,
		const CollectionType &	type,
		const char *	name
	)
throw	(	DCException
	)

virtual

Reserves a dataset for parallel access.

Parameters

id	ID for iteration.
size	Size to reserve for local data.
globalSize	Returns the global size of the dataset, can be NULL.
globalOffset	Returns the global offset for the calling process, can be NULL.
rank	Number of dimensions (1-3).
type	Type information for data.
name	Name for the dataset.

Implements splash::IParallelDataCollector.

Definition at line 687 of file ParallelDataCollector.cpp.

◆ write() [1/2]

void splash::ParallelDataCollector::write	(	int32_t	id,
		const CollectionType &	type,
		uint32_t	rank,
		const Selection	select,
		const char *	name,
		const void *	buf
	)
throw	(	DCException
	)

virtual

Implements DataCollector::write The global size and the write offset for the calling process are determined automatically via MPI among all participating processes. Note: This is not possible when writing 2D data with a 3D MPI topology.

Implements splash::IParallelDataCollector.

Definition at line 632 of file ParallelDataCollector.cpp.

Here is the caller graph for this function:

◆ write() [2/2]

void splash::ParallelDataCollector::write	(	int32_t	id,
		const Dimensions	globalSize,
		const Dimensions	globalOffset,
		const CollectionType &	type,
		uint32_t	rank,
		const Selection	select,
		const char *	name,
		const void *	buf
	)

virtual

Writes data to HDF5 file.

Parameters

id	ID for iteration
globalSize	Size of global collective write buffer.
globalOffset	Offset in `globalSize` buffer where this process writes to.
type	Type information for data.
rank	Number of dimensions (1-3).
select	Selection in src buffer.
name	Name for the dataset.
buf	Local buffer for writing.

Implements splash::IParallelDataCollector.

Definition at line 643 of file ParallelDataCollector.cpp.

◆ writeAttribute() [1/2]

void splash::ParallelDataCollector::writeAttribute	(	int32_t	id,
		const CollectionType &	type,
		const char *	dataName,
		const char *	attrName,
		const void *	buf
	)
throw	(	DCException
	)

virtual

Writes an attribute to a single dataset.

Parameters

id	ID for iteration.
type	Type information for data.
dataName	Name of the dataset in group `id` to write attribute to. If dataName is NULL, the attribute is written for the iteration group. If the path dataName does not yet exist, missing groups will be created.
attrName	Name of the attribute.
buf	Buffer to be written as attribute.

Implements splash::DataCollector.

Definition at line 480 of file ParallelDataCollector.cpp.

◆ writeAttribute() [2/2]

void splash::ParallelDataCollector::writeAttribute	(	int32_t	id,
		const CollectionType &	type,
		const char *	dataName,
		const char *	attrName,
		uint32_t	ndims,
		const Dimensions	dims,
		const void *	buf
	)
throw	(	DCException
	)

virtual

Writes an attribute to a single dataset.

Parameters

id	ID for iteration.
type	Type information for data.
dataName	Name of the dataset in group `id` to write attribute to. If dataName is NULL, the attribute is written for the iteration group.
attrName	Name of the attribute.
ndims	Number of dimensions (1-3)
dims	Number of elements
buf	Buffer to be written as attribute.

Implements splash::DataCollector.

Definition at line 492 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

◆ writeGlobalAttribute()

void splash::ParallelDataCollector::writeGlobalAttribute	(	int32_t	id,
		const CollectionType &	type,
		const char *	name,
		const void *	buf
	)
throw	(	DCException
	)

virtual

Writes global attribute to HDF5 file (default group).

Parameters

id	ID for iteration.
type	Type information for data.
name	Name of the attribute.
buf	Source buffer for attribute.

Implements splash::IParallelDataCollector.

Definition at line 363 of file ParallelDataCollector.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

The documentation for this class was generated from the following files:

libSplash/src/include/splash/ParallelDataCollector.hpp
libSplash/src/ParallelDataCollector.cpp

Public Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ ParallelDataCollector()

◆ ~ParallelDataCollector()

Member Function Documentation

◆ append()

◆ close()

◆ createReference()

◆ finalize()

◆ getEntriesForID()

◆ getEntryIDs()

◆ getMaxID()

◆ getMPISize()

◆ open()

◆ read() [1/4]

◆ read() [2/4]

◆ read() [3/4]

◆ read() [4/4]

◆ readAttribute()

◆ readAttributeInfo()

◆ readGlobalAttribute()

◆ readGlobalAttributeInfo()

◆ readMeta()

◆ remove() [1/2]

◆ remove() [2/2]

◆ reserve() [1/2]

◆ reserve() [2/2]

◆ write() [1/2]

◆ write() [2/2]

◆ writeAttribute() [1/2]

◆ writeAttribute() [2/2]

◆ writeGlobalAttribute()