Merge pull request #1657 from NOAA-GSD/ejh_fix_master

Fix largefile issue, szip parallel test, and some documentation updates

Author: WardF

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.7.4 - TBD
 
+* [Bug Fix] Attempts to set chunking on scalar vars will now return NC_EINVAL. Scalar vars cannot be chunked. Previously the library ignored these attempts and scalar vars were always stored as contiguous. See [https://github.com/Unidata/netcdf-c/issues/1644].
 * [Enhancement] Support has been added for multiple filters per variable.  See [https://github.com/Unidata/netcdf-c/issues/1584].
 * [Enhancement] Now nc_inq_var_szip retuns 0 for parameter values if szip is not in use for var. See [https://github.com/Unidata/netcdf-c/issues/1618].
 * [Enhancement] Now allow parallel I/O with filters, for HDF5-1.10.3 and later. See [https://github.com/Unidata/netcdf-c/issues/1473].

h5_test/tst_h_files.c

@@ -246,7 +246,6 @@ main()
         char file_name[STR_LEN + 1];
         char dimscale_wo_var[STR_LEN];
         void *bufr;
-        hsize_t num_obj;
         void *fillp = NULL;
 
         sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME);
@@ -361,12 +360,6 @@ main()
         if (H5Gclose(grpid) < 0) ERR;
         if (H5Fclose(fileid) < 0) ERR;
 
-        /* Reopen the file and check it. */
-        if ((fileid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) ERR;
-        if (H5Gget_num_objs(fileid, &num_obj) < 0) ERR;
-        if (num_obj) ERR;
-        if (H5Fclose(fileid) < 0) ERR;
-
         /* Delete the huge data file we created. */
         (void) remove(file_name);
     }

libdispatch/dparallel.c

@@ -289,50 +289,58 @@ nc_open_par_fortran(const char *path, int omode, int comm,
 #endif
 }
 
-/**\ingroup datasets
+/**
+   @ingroup datasets
 
-This function will change the parallel access of a variable from independent to
-collective and vice versa. Note when file is opened/created to use PnetCDF
-library to perform parallel I/O underneath, argument varid is ignored and the
-mode changed by this function applies to all variables. This is because PnetCDF
-does not support access mode change for individual variables. In this case,
-users may use NC_GLOBAL in varid argument for better program readability. This
-function is collective, i.e. must be called by all MPI processes defined in the
-MPI communicator used in nc_create_par() or nc_open_par(). In addition, values
-of arguments of this function must be the same among all MPI processes.
+   This function will change the parallel access of a variable from
+   independent to collective and vice versa.
 
-To obtain a good I/O performance, users are recommended to use collective mode.
-In addition, switching between collective and independent I/O mode can be
-expensive.
+   This function is collective, i.e. must be called by all MPI
+   processes defined in the MPI communicator used in nc_create_par()
+   or nc_open_par(). In addition, values of arguments of this function
+   must be the same among all MPI processes.
 
-In netcdf-c-4.7.4 or later, using hdf5-1.10.2 or later, the zlib and
-fletcher32 filters may be used when writing data with parallel
-I/O. The use of these filters require collective access. Turning on
-the zlib (deflate) or fletcher32 filter for a variable will
-automatically set its access to collective. Attempts to set access to
-independent will return ::NC_EINVAL.
+   To obtain a good I/O performance, users are recommended to use
+   collective mode. In addition, switching between collective and
+   independent I/O mode can be expensive.
 
-\param ncid NetCDF or group ID, from a previous call to nc_open_par(),
-nc_create_par(), nc_def_grp(), or associated inquiry functions such as
-nc_inq_ncid().
+   In netcdf-c-4.7.4 or later, using hdf5-1.10.2 or later, the zlib,
+   szip, fletcher32, and other filters may be used when writing data
+   with parallel I/O. The use of these filters require collective
+   access. Turning on the zlib (deflate) or fletcher32 filter for a
+   variable will automatically set its access to collective if the
+   file has been opened for parallel I/O. Attempts to set access to
+   independent will return ::NC_EINVAL.
 
-\param varid Variable ID
+   @note When the library is build with --enable-pnetcdf, and when
+   file is opened/created to use PnetCDF library to perform parallel
+   I/O underneath, argument varid is ignored and the mode changed by
+   this function applies to all variables. This is because PnetCDF
+   does not support access mode change for individual variables. In
+   this case, users may use NC_GLOBAL in varid argument for better
+   program readability.
 
-\param par_access NC_COLLECTIVE or NC_INDEPENDENT.
+   @param ncid NetCDF or group ID, from a previous call to
+   nc_open_par(), nc_create_par(), nc_def_grp(), or associated inquiry
+   functions such as nc_inq_ncid().
 
-\returns ::NC_NOERR No error.
-\returns ::NC_EBADID Invalid ncid passed.
-\returns ::NC_ENOTVAR Invalid varid passed.
-\returns ::NC_ENOPAR File was not opened with nc_open_par/nc_create_var.
-\returns ::NC_EINVAL Invalid par_access specified, or attempt to set
-filtered variable to independent access.
+   @param varid Variable ID
 
-<h1>Example</h1>
+   @param par_access NC_COLLECTIVE or NC_INDEPENDENT.
 
-Here is an example from examples/C/parallel_vara.c which changes the
-parallel access of a variable and then writes to it.
+   @return ::NC_NOERR No error.
+   @return ::NC_EBADID Invalid ncid passed.
+   @return ::NC_ENOTVAR Invalid varid passed.
+   @return ::NC_ENOPAR File was not opened with nc_open_par/nc_create_var.
+   @return ::NC_EINVAL Invalid par_access specified, or attempt to set
+   filtered variable to independent access.
 
-\code
+   <h1>Example</h1>
+
+   Here is an example from examples/C/parallel_vara.c which changes
+   the parallel access of a variable and then writes to it.
+
+@code
 #define NY 10
 #define NX 4
 
@@ -366,10 +374,9 @@ parallel access of a variable and then writes to it.
 
     err = nc_put_vara_int(ncid, varid, start, count, &buf[0][0]); ERR
 
-\endcode
-\author Ed Hartnett, Dennis Heimbigner
-\ingroup datasets
- */
+@endcode
+@author Ed Hartnett, Dennis Heimbigner
+*/
 int
 nc_var_par_access(int ncid, int varid, int par_access)
 {

libdispatch/dvar.c

@@ -470,10 +470,8 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
 }
 
 /**
-   Define chunking parameters for a variable
-
-   The function nc_def_var_chunking sets the storage and, optionally,
-   the chunking parameters for a variable in a netCDF-4 file.
+   Define storage and, if chunked storage is used, chunking parameters
+   for a variable
 
    The storage may be set to NC_CONTIGUOUS, NC_COMPACT, or NC_CHUNKED.
 
@@ -490,7 +488,7 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
    Chunked storage means the data are stored as chunks, of
    user-configurable size. Chunked storage is required for variable
    with one or more unlimted dimensions, or variable which use
-   compression.
+   compression, or any other filter.
 
    The total size of a chunk must be less than 4 GiB. That is, the
    product of all chunksizes and the size of the data (or the size of
@@ -500,16 +498,14 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
    before nc_enddef is called. Once the chunking parameters are set for a
    variable, they cannot be changed.
 
-   Note scalar variables may have a storage of NC_CONTIGUOUS or
-   NC_COMPACT. Attempts to set chunking on a scalare variable will be
-   ignored by the library (but no error code is returned). Only
-   non-scalar variables can have chunking.
+   @note Scalar variables may have a storage of NC_CONTIGUOUS or
+   NC_COMPACT. Attempts to set chunking on a scalare variable will
+   cause ::NC_EINVEL to be returned. Only non-scalar variables can
+   have chunking.
 
    @param ncid NetCDF ID, from a previous call to nc_open() or
    nc_create().
-
    @param varid Variable ID.
-
    @param storage If ::NC_CONTIGUOUS or ::NC_COMPACT, then contiguous
    or compact storage is used for this variable. Variables with one or
    more unlimited dimensions cannot use contiguous or compact
@@ -518,20 +514,19 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
    storage is used for this variable. Chunk sizes may be specified
    with the chunksizes parameter or default sizes will be used if that
    parameter is NULL.
-
    @param chunksizesp A pointer to an array list of chunk sizes. The
    array must have one chunksize for each dimension of the variable. If
    ::NC_CONTIGUOUS storage is set, then the chunksizes parameter is
-   ignored.
+   ignored. Ignored if NULL.
 
    @return ::NC_NOERR No error.
    @return ::NC_EBADID Bad ID.
    @return ::NC_ENOTNC4 Not a netCDF-4 file.
-   @return ::NC_ELATEDEF This variable has already been the subject of a
-   nc_enddef call.  In netCDF-4 files nc_enddef will be called
+   @return ::NC_ELATEDEF This variable has already been the subject of
+   a nc_enddef call.  In netCDF-4 files nc_enddef will be called
    automatically for any data read or write. Once nc_enddef has been
-   called after the nc_def_var call for a variable, it is impossible to
-   set the chunking for that variable.
+   called after the nc_def_var call for a variable, it is impossible
+   to set the chunking for that variable.
    @return ::NC_ENOTINDEFINE Not in define mode.  This is returned for
    netCDF classic or 64-bit offset files, or for netCDF-4 files, when
    they wwere created with ::NC_CLASSIC_MODEL flag by nc_create().
@@ -542,7 +537,8 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
    @return ::NC_EVARSIZE Compact storage attempted for variable bigger
    than 64 KB.
    @return ::NC_EINVAL Attempt to set contiguous or compact storage
-   for var with one or more unlimited dimensions.
+   for var with one or more unlimited dimensions, or chunking for a
+   scalar var.
 
    @section nc_def_var_chunking_example Example
 
@@ -585,8 +581,7 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
    @author Ed Hartnett, Dennis Heimbigner
 */
 int
-nc_def_var_chunking(int ncid, int varid, int storage,
-                    const size_t *chunksizesp)
+nc_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp)
 {
     NC* ncp;
     int stat = NC_check_id(ncid, &ncp);

nc_test4/tst_parallel5.c

@@ -336,7 +336,8 @@ main(int argc, char **argv)
                           &ncid)) ERR;
         if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN, &dimid)) ERR;
         if (nc_def_var(ncid, SZIP_VAR_NAME, NC_FLOAT, NDIMS1, &dimid, &varid)) ERR;
-        if (nc_def_var_szip(ncid, varid, NC_SZIP_NN, SZIP_PIXELS_PER_BLOCK)) ERR;
+        if (nc_def_var_szip(ncid, varid, H5_SZIP_NN_OPTION_MASK,
+                            SZIP_PIXELS_PER_BLOCK)) ERR;
         if (nc_enddef(ncid)) ERR;
         start[0] = mpi_rank * elements_per_pe;
         count[0] = elements_per_pe;

unit_test/tst_nclist.c

@@ -112,7 +112,6 @@ main(int argc, char **argv)
     {
         NC *ncp;
         int mode = 0;
-        NCmodel model;
         int max_num_nc = 65535;
         int i;
         int ret;