You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
385 lines
17 KiB
C
385 lines
17 KiB
C
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
|
|
* Copyright by The HDF Group. *
|
|
* All rights reserved. *
|
|
* *
|
|
* This file is part of HDF5. The full HDF5 copyright notice, including *
|
|
* terms governing use, modification, and redistribution, is contained in *
|
|
* the COPYING file, which can be found at the root of the source code *
|
|
* distribution tree, or in https://www.hdfgroup.org/licenses. *
|
|
* If you do not have access to either file, you may request a copy from *
|
|
* help@hdfgroup.org. *
|
|
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
|
|
|
|
/* Purpose: The public header file for the subfiling driver. */
|
|
#ifndef H5FDsubfiling_H
|
|
#define H5FDsubfiling_H
|
|
|
|
#ifdef H5_HAVE_SUBFILING_VFD
|
|
/**
|
|
* \def H5FD_SUBFILING
|
|
* Macro that returns the identifier for the #H5FD_SUBFILING driver. \hid_t{file driver}
|
|
*/
|
|
#define H5FD_SUBFILING (H5FDperform_init(H5FD_subfiling_init))
|
|
#else
|
|
#define H5FD_SUBFILING (H5I_INVALID_HID)
|
|
#endif
|
|
|
|
/**
|
|
* \def H5FD_SUBFILING_NAME
|
|
* The canonical name for the #H5FD_SUBFILING driver
|
|
*/
|
|
#define H5FD_SUBFILING_NAME "subfiling"
|
|
|
|
#ifdef H5_HAVE_SUBFILING_VFD
|
|
|
|
#ifndef H5FD_SUBFILING_FAPL_MAGIC
|
|
/**
|
|
* \def H5FD_SUBFILING_CURR_FAPL_VERSION
|
|
* The version number of the H5FD_subfiling_config_t configuration
|
|
* structure for the #H5FD_SUBFILING driver
|
|
*/
|
|
#define H5FD_SUBFILING_CURR_FAPL_VERSION 1
|
|
/**
|
|
* \def H5FD_SUBFILING_FAPL_MAGIC
|
|
* Unique number used to distinguish the #H5FD_SUBFILING driver from other HDF5 file drivers
|
|
*/
|
|
#define H5FD_SUBFILING_FAPL_MAGIC 0xFED01331
|
|
#endif
|
|
|
|
/**
|
|
* \def H5FD_SUBFILING_DEFAULT_STRIPE_SIZE
|
|
* The default stripe size (in bytes) for data stripes in subfiles
|
|
*/
|
|
#define H5FD_SUBFILING_DEFAULT_STRIPE_SIZE (32 * 1024 * 1024)
|
|
|
|
/**
|
|
* \def H5FD_SUBFILING_DEFAULT_STRIPE_COUNT
|
|
* Macro for the default Subfiling stripe count value. The default
|
|
* is currently to use one subfile per node.
|
|
*/
|
|
#define H5FD_SUBFILING_DEFAULT_STRIPE_COUNT -1
|
|
|
|
/**
|
|
* \def H5FD_SUBFILING_FILENAME_TEMPLATE
|
|
* The basic template for a subfile filename. The format specifiers
|
|
* correspond to:
|
|
*
|
|
* %s -> base filename, e.g. "file.h5"
|
|
* %PRIu64 -> file inode, e.g. 11273556
|
|
* %0*d -> number (starting at 1) signifying the Nth (out of total
|
|
* number of subfiles) subfile. Zero-padded according
|
|
* to the number of digits in the number of subfiles
|
|
* (calculated by log10(num_subfiles) + 1)
|
|
* %d -> number of subfiles
|
|
*
|
|
* yielding filenames such as:
|
|
*
|
|
* file.h5.subfile_11273556_01_of_10
|
|
* file.h5.subfile_11273556_02_of_10
|
|
* file.h5.subfile_11273556_10_of_10
|
|
*/
|
|
#define H5FD_SUBFILING_FILENAME_TEMPLATE "%s.subfile_%" PRIu64 "_%0*d_of_%d"
|
|
|
|
/**
|
|
* \def H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE
|
|
* The basic template for a #H5FD_SUBFILING driver configuration filename.
|
|
* The format specifiers correspond to:
|
|
*
|
|
* %s -> base filename, e.g. "file.h5"
|
|
* %PRIu64 -> file inode, e.g. 11273556
|
|
*
|
|
* yielding a filename such as:
|
|
*
|
|
* file.h5.subfile_11273556.config
|
|
*/
|
|
#define H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE "%s.subfile_%" PRIu64 ".config"
|
|
|
|
/*
|
|
* Environment variables interpreted by the HDF5 Subfiling feature
|
|
*/
|
|
|
|
/**
|
|
* \def H5FD_SUBFILING_STRIPE_SIZE
|
|
* Macro for name of the environment variable that specifies the size
|
|
* (in bytes) for data stripes in subfiles
|
|
*
|
|
* The value set for this environment variable is interpreted as a
|
|
* long long value and must be > 0.
|
|
*/
|
|
#define H5FD_SUBFILING_STRIPE_SIZE "H5FD_SUBFILING_STRIPE_SIZE"
|
|
/**
|
|
* \def H5FD_SUBFILING_IOC_PER_NODE
|
|
* Macro for name of the environment variable that specifies the number
|
|
* of MPI ranks per node to use as I/O concentrators
|
|
*
|
|
* The value set for this environment variable is interpreted as a
|
|
* long value and must be > 0.
|
|
*/
|
|
#define H5FD_SUBFILING_IOC_PER_NODE "H5FD_SUBFILING_IOC_PER_NODE"
|
|
/**
|
|
* \def H5FD_SUBFILING_IOC_SELECTION_CRITERIA
|
|
* Macro for name of the environment variable that provides information
|
|
* for selection MPI ranks as I/O concentrators
|
|
*
|
|
* The value set for this environment variable is interpreted differently,
|
|
* depending on the IOC selection method chosen.
|
|
*
|
|
* For #SELECT_IOC_ONE_PER_NODE, this value is ignored.
|
|
*
|
|
* For #SELECT_IOC_EVERY_NTH_RANK, this value is interpreted as a
|
|
* long value and must be > 0. The value will correspond to the
|
|
* `N` value when selecting every `N`-th MPI rank as an I/O
|
|
* concentrator.
|
|
*
|
|
* For #SELECT_IOC_WITH_CONFIG, this value is ignored as that particular
|
|
* IOC selection method is not currently supported.
|
|
*
|
|
* For #SELECT_IOC_TOTAL, this value is interpreted as a long value
|
|
* and must be > 0. The value will correspond to the total number
|
|
* of I/O concentrators to be used.
|
|
*/
|
|
#define H5FD_SUBFILING_IOC_SELECTION_CRITERIA "H5FD_SUBFILING_IOC_SELECTION_CRITERIA"
|
|
/**
|
|
* \def H5FD_SUBFILING_SUBFILE_PREFIX
|
|
* Macro for name of the environment variable that specifies a prefix
|
|
* to apply to the filenames generated for subfiles
|
|
*
|
|
* The value set for this environment variable is interpreted as a
|
|
* pathname.
|
|
*/
|
|
#define H5FD_SUBFILING_SUBFILE_PREFIX "H5FD_SUBFILING_SUBFILE_PREFIX"
|
|
|
|
/**
|
|
* \enum H5FD_subfiling_ioc_select_t
|
|
* This enum defines the various constants to allow different
|
|
* allocations of MPI ranks as I/O concentrators.
|
|
*
|
|
* \var SELECT_IOC_ONE_PER_NODE
|
|
* Default selection method. One MPI rank per node is used as an
|
|
* I/O concentrator. If this selection method is used, the number
|
|
* of I/O concentrators per node can be adjusted with the
|
|
* #H5FD_SUBFILING_IOC_PER_NODE environment variable.
|
|
*
|
|
* \var SELECT_IOC_EVERY_NTH_RANK
|
|
* Starting with MPI rank 0, a stride of 'N' is applied to the MPI
|
|
* rank values to determine the next I/O concentrator. The
|
|
* #H5FD_SUBFILING_IOC_SELECTION_CRITERIA environment variable must
|
|
* be set to the value desired for 'N'.
|
|
*
|
|
* \var SELECT_IOC_WITH_CONFIG
|
|
* Currently unsupported. Use a configuration file to determine
|
|
* the mapping from MPI ranks to I/O concentrators. The
|
|
* #H5FD_SUBFILING_IOC_SELECTION_CRITERIA environment variable must
|
|
* be set to the path to the configuration file.
|
|
*
|
|
* \var SELECT_IOC_TOTAL
|
|
* Specifies that a total of 'N' I/O concentrators should be used.
|
|
* Starting with MPI rank 0, a stride of 'MPI comm size' / 'N' is
|
|
* applied to the MPI rank values to determine the next I/O
|
|
* concentrator. The #H5FD_SUBFILING_IOC_SELECTION_CRITERIA
|
|
* environment variable must be set to the value desired for 'N'.
|
|
*
|
|
* \var ioc_selection_options
|
|
* Unused. Sentinel value
|
|
*/
|
|
typedef enum {
|
|
SELECT_IOC_ONE_PER_NODE = 0, /* Default */
|
|
SELECT_IOC_EVERY_NTH_RANK, /* Starting at rank 0, select-next += N */
|
|
SELECT_IOC_WITH_CONFIG, /* NOT IMPLEMENTED: Read-from-file */
|
|
SELECT_IOC_TOTAL, /* Starting at rank 0, mpi_size / total */
|
|
ioc_selection_options /* Sentinel value */
|
|
/* NOTE: Add to the Fortran constants (H5f90global.F90) when adding new entries */
|
|
} H5FD_subfiling_ioc_select_t;
|
|
|
|
/**
|
|
* \struct H5FD_subfiling_params_t
|
|
* \brief Subfiling parameter structure that is shared between the #H5FD_SUBFILING
|
|
* and #H5FD_IOC drivers
|
|
*
|
|
* \var H5FD_subfiling_ioc_select_t H5FD_subfiling_params_t::ioc_selection
|
|
* The method to use for selecting MPI ranks to be I/O concentrators. The
|
|
* current default is to select one MPI rank per node to be an I/O concentrator.
|
|
* Refer to #H5FD_subfiling_ioc_select_t for a description of the algorithms
|
|
* available for use.
|
|
*
|
|
* \var int64_t H5FD_subfiling_params_t::stripe_size
|
|
* The stripe size defines the size (in bytes) of the data stripes in the
|
|
* subfiles for the logical HDF5 file. Data is striped across the subfiles
|
|
* in a round-robin wrap-around fashion in segments equal to the stripe size.
|
|
*
|
|
* For example, in an HDF5 file consisting of four subfiles with a 1MiB stripe
|
|
* size, the first and fifth 1MiB of data would reside in the first subfile,
|
|
* the second and sixth 1MiB of data would reside in the second subfile and so
|
|
* on.
|
|
*
|
|
* This value can also be set or adjusted with the #H5FD_SUBFILING_STRIPE_SIZE
|
|
* environment variable.
|
|
*
|
|
* \var int32_t H5FD_subfiling_params_t::stripe_count
|
|
* The target number of subfiles to use for the logical HDF5 file. The current
|
|
* default is to use one subfile per node, but it can be useful to set a
|
|
* different target number of subfiles, especially if the HDF5 application will
|
|
* pre-create the HDF5 file on a single MPI rank. In that particular case, the
|
|
* single rank will need to know how many subfiles the logical HDF5 file will
|
|
* consist of in order to properly pre-create the file.
|
|
*
|
|
* This value is used in conjunction with the IOC selection method to determine
|
|
* which MPI ranks will be assigned as I/O concentrators. Alternatively, the
|
|
* mapping between MPI ranks and I/O concentrators can be set or adjusted with a
|
|
* combination of the #ioc_selection field and the #H5FD_SUBFILING_IOC_PER_NODE
|
|
* and #H5FD_SUBFILING_IOC_SELECTION_CRITERIA environment variables.
|
|
*/
|
|
typedef struct H5FD_subfiling_params_t {
|
|
H5FD_subfiling_ioc_select_t ioc_selection; /* Method to select I/O concentrators */
|
|
int64_t stripe_size; /* Size (in bytes) of data stripes in subfiles */
|
|
int32_t stripe_count; /* Target number of subfiles to use */
|
|
} H5FD_subfiling_params_t;
|
|
|
|
//! <!-- [H5FD_subfiling_config_t_snip] -->
|
|
/**
|
|
* \struct H5FD_subfiling_config_t
|
|
* \brief Configuration structure for H5Pset_fapl_subfiling() / H5Pget_fapl_subfiling()
|
|
*
|
|
* \details H5FD_subfiling_config_t is a public structure that is used to pass
|
|
* subfiling configuration data to the #H5FD_SUBFILING driver via
|
|
* a File Access Property List. A pointer to an instance of this structure
|
|
* is a parameter to H5Pset_fapl_subfiling() and H5Pget_fapl_subfiling().
|
|
*
|
|
* \var uint32_t H5FD_subfiling_config_t::magic
|
|
* A somewhat unique number which distinguishes the #H5FD_SUBFILING driver
|
|
* from other drivers. Used in combination with a version number, it can
|
|
* help to validate a user-generated File Access Property List. This field
|
|
* should be set to #H5FD_SUBFILING_FAPL_MAGIC.
|
|
*
|
|
* \var uint32_t H5FD_subfiling_config_t::version
|
|
* Version number of the H5FD_subfiling_config_t structure. Any instance
|
|
* passed to H5Pset_fapl_subfiling() / H5Pget_fapl_subfiling() must have
|
|
* a recognized version number or an error will be raised. Currently, this
|
|
* field should be set to #H5FD_SUBFILING_CURR_FAPL_VERSION.
|
|
*
|
|
* \var hid_t H5FD_subfiling_config_t::ioc_fapl_id
|
|
* The File Access Property List which is setup with the file driver that
|
|
* the #H5FD_SUBFILING driver will use for servicing I/O requests to the
|
|
* subfiles. Currently, the File Access Property List must be setup with
|
|
* the #H5FD_IOC driver by calling H5Pset_fapl_ioc(), but future development
|
|
* may allow other file drivers to be used.
|
|
*
|
|
* \var hbool_t H5FD_subfiling_config_t::require_ioc
|
|
* A boolean flag which indicates whether the #H5FD_SUBFILING driver should
|
|
* use the #H5FD_IOC driver for its I/O operations. This field should currently
|
|
* always be set to TRUE.
|
|
*
|
|
* \var H5FD_subfiling_params_t H5FD_subfiling_config_t::shared_cfg
|
|
* A structure which contains the subfiling parameters that are shared between
|
|
* the #H5FD_SUBFILING and #H5FD_IOC drivers. This includes the subfile stripe
|
|
* size, stripe count, IOC selection method, etc.
|
|
*
|
|
*/
|
|
typedef struct H5FD_subfiling_config_t {
|
|
uint32_t magic; /* Must be set to H5FD_SUBFILING_FAPL_MAGIC */
|
|
uint32_t version; /* Must be set to H5FD_SUBFILING_CURR_FAPL_VERSION */
|
|
hid_t ioc_fapl_id; /* The FAPL setup with the stacked VFD to use for I/O concentrators */
|
|
hbool_t require_ioc; /* Whether to use the IOC VFD (currently must always be TRUE) */
|
|
H5FD_subfiling_params_t shared_cfg; /* Subfiling/IOC parameters (stripe size, stripe count, etc.) */
|
|
} H5FD_subfiling_config_t;
|
|
//! <!-- [H5FD_subfiling_config_t_snip] -->
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
/**
|
|
* \brief Internal routine to initialize #H5FD_SUBFILING driver. Not meant to be
|
|
* called directly by an HDF5 application
|
|
*/
|
|
H5_DLL hid_t H5FD_subfiling_init(void);
|
|
/**
|
|
* \ingroup FAPL
|
|
*
|
|
* \brief Modifies the specified File Access Property List to use the #H5FD_SUBFILING driver
|
|
*
|
|
* \fapl_id
|
|
* \param[in] vfd_config Pointer to #H5FD_SUBFILING driver configuration structure. May be NULL.
|
|
* \returns \herr_t
|
|
*
|
|
* \details H5Pset_fapl_subfiling() modifies the File Access Property List to use the
|
|
* #H5FD_SUBFILING driver.
|
|
*
|
|
* The #H5FD_SUBFILING driver is an MPI-based file driver that allows an
|
|
* HDF5 application to distribute a logical HDF5 file across a collection
|
|
* of "subfiles" in equal-sized data segment "stripes". I/O to the logical
|
|
* HDF5 file is then directed to the appropriate "subfile" according to the
|
|
* #H5FD_SUBFILING configuration and a system of I/O concentrators, which
|
|
* are MPI ranks operating worker threads.
|
|
*
|
|
* By allowing a configurable stripe size, number of I/O concentrators and
|
|
* method for selecting MPI ranks as I/O concentrators, the #H5FD_SUBFILING
|
|
* driver aims to enable an HDF5 application to find a middle ground between
|
|
* the single shared file and file-per-process approaches to parallel file I/O
|
|
* for the particular machine the application is running on. In general, the
|
|
* goal is to avoid some of the complexity of the file-per-process approach
|
|
* while also minimizing the locking issues of the single shared file approach
|
|
* on a parallel file system.
|
|
*
|
|
* \note Since the #H5FD_SUBFILING driver is an MPI-based file driver, the HDF5
|
|
* application should ensure that H5Pset_mpi_params() is called before this
|
|
* routine so that the appropriate MPI communicator and info objects will be
|
|
* setup for use by the #H5FD_SUBFILING and #H5FD_IOC drivers.
|
|
*
|
|
* \note The current architecture of the #H5FD_SUBFILING driver requires that the
|
|
* HDF5 application must have been initialized with MPI_Init_thread() using
|
|
* a value of MPI_THREAD_MULTIPLE for the thread support level.
|
|
*
|
|
* \note The \p vfd_config parameter may be NULL. In this case, the reference
|
|
* implementation I/O concentrator VFD will be used with the default settings
|
|
* of one I/O concentrator per node and a stripe size of 32MiB. Refer to the
|
|
* H5FD_subfiling_config_t documentation for information about configuration
|
|
* for the #H5FD_SUBFILING driver.
|
|
*
|
|
* \since 1.14.0
|
|
*
|
|
*/
|
|
H5_DLL herr_t H5Pset_fapl_subfiling(hid_t fapl_id, const H5FD_subfiling_config_t *vfd_config);
|
|
/**
|
|
* \ingroup FAPL
|
|
*
|
|
* \brief Queries a File Access Property List for #H5FD_SUBFILING file driver properties
|
|
*
|
|
* \fapl_id
|
|
* \param[out] config_out Pointer to H5FD_subfiling_config_t structure through which the
|
|
* #H5FD_SUBFILING file driver properties will be returned.
|
|
*
|
|
* \returns \herr_t
|
|
*
|
|
* \details H5Pget_fapl_subfiling() queries the specified File Access Property List for
|
|
* #H5FD_SUBFILING driver properties as set by H5Pset_fapl_subfiling(). If the
|
|
* #H5FD_SUBFILING driver has not been set on the File Access Property List, a
|
|
* default configuration is returned. An HDF5 application may use this
|
|
* functionality to manually configure the #H5FD_SUBFILING driver by calling
|
|
* H5Pget_fapl_subfiling() on a newly-created File Access Property List, adjusting
|
|
* the default values and then calling H5Pset_fapl_subfiling() with the configured
|
|
* H5FD_subfiling_config_t structure.
|
|
*
|
|
* \note H5Pget_fapl_subfiling() returns the #H5FD_SUBFILING driver properties as they
|
|
* were initially set for the File Access Property List using H5Pset_fapl_subfiling().
|
|
* Alternatively, the driver properties can be modified at runtime according to values
|
|
* set for the #H5FD_SUBFILING_STRIPE_SIZE, #H5FD_SUBFILING_IOC_PER_NODE and
|
|
* #H5FD_SUBFILING_IOC_SELECTION_CRITERIA environment variables. However, driver
|
|
* properties set through environment variables will not be reflected in what is
|
|
* returned by H5Pget_fapl_subfiling(), so an application may need to check those
|
|
* environment variables to get accurate values for the #H5FD_SUBFILING driver
|
|
* properties.
|
|
*
|
|
* \since 1.14.0
|
|
*
|
|
*/
|
|
H5_DLL herr_t H5Pget_fapl_subfiling(hid_t fapl_id, H5FD_subfiling_config_t *config_out);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif /* H5_HAVE_SUBFILING_VFD */
|
|
|
|
#endif /* H5FDsubfiling_H */
|