From a223e11789eb7bcdd62b90ba74163981d11453b3 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Wed, 28 Oct 2026 15:16:47 -0700 Subject: [PATCH 1/5] add constants, min file sizes for CDF-1/2/5 --- src/include/dispatch.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/include/dispatch.h b/src/include/dispatch.h index 873e9fc6ec..2fab86f89f 100644 --- a/src/include/dispatch.h +++ b/src/include/dispatch.h @@ -10,6 +10,14 @@ #include #include +/* + * The minimum file size of a CDF-1 or CDF-2 file is 32 bytes. + * For CDF-5 files, the minimum is 48 bytes. + */ +#define CDF1_MIN_FSIZE 32 +#define CDF2_MIN_FSIZE 32 +#define CDF5_MIN_FSIZE 48 + /* various I/O modes or types */ #define NC_REQ_COLL 0x00000001 /* collective request */ #define NC_REQ_INDEP 0x00000002 /* independent request */ From 9e1a4cb3e902f415f0ae5b4a5478200838db5552 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Wed, 28 Oct 2026 15:25:34 -0700 Subject: [PATCH 2/5] Delete MIN_NC_XSZ and use CDF1_MIN_FSIZE, CDF2_MIN_FSIZE, CDF5_MIN_FSIZE Delete constant MIN_NC_XSZ from ncmpio_NC.h Use the three new constants CDF1_MIN_FSIZE, CDF2_MIN_FSIZE, CDF5_MIN_FSIZE defined in src/include/dispatch.h --- src/drivers/ncmpio/ncmpio_NC.h | 6 ------ src/drivers/ncmpio/ncmpio_util.c | 12 +++++++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/drivers/ncmpio/ncmpio_NC.h b/src/drivers/ncmpio/ncmpio_NC.h index b0fdc0c878..de1bdbe479 100644 --- a/src/drivers/ncmpio/ncmpio_NC.h +++ b/src/drivers/ncmpio/ncmpio_NC.h @@ -105,12 +105,6 @@ typedef struct { * and vice versa */ #define NCMPII_ECHAR(nctype, mpitype) ((((nctype) == NC_CHAR) == ((mpitype) != MPI_CHAR)) ? NC_ECHAR : NC_NOERR) -/* - * The external size of an empty netcdf version 1 file is 32 bytes, which is - * also set as the initial value of ncp->xsz. - */ -#define MIN_NC_XSZ 32 - typedef enum { NC_UNSPECIFIED = 0, /* ABSENT */ NC_DIMENSION = 10, /* \x00 \x00 \x00 \x0A */ diff --git a/src/drivers/ncmpio/ncmpio_util.c b/src/drivers/ncmpio/ncmpio_util.c index bbee25799b..01caf472d0 100644 --- a/src/drivers/ncmpio/ncmpio_util.c +++ b/src/drivers/ncmpio/ncmpio_util.c @@ -135,14 +135,16 @@ void ncmpio_hint_extract(NC *ncp, ncp->hdr_chunk = PNC_HDR_READ_CHUNK_SIZE; else if (llval > NC_MAX_INT) /* limit to NC_MAX_INT */ ncp->hdr_chunk = NC_MAX_INT; - else if (llval < 48) - /* minimum file sizes: 32 bytes for CDF-1 and 2, 48 for CDF5 */ - ncp->hdr_chunk = 48; + else if (llval < CDF5_MIN_FSIZE) + /* minimum file header size for CDF-1 and 2 files is 32 bytes, + * and 48 bytes for CDF-5. + */ + ncp->hdr_chunk = CDF5_MIN_FSIZE; else ncp->hdr_chunk = (int)llval; - /* CDF-5's minimum header size is 4 bytes more than CDF-1 2's */ - ncp->hdr_chunk = PNETCDF_RNDUP(MAX(MIN_NC_XSZ+4, ncp->hdr_chunk), + /* CDF-5's minimum header size is CDF5_MIN_FSIZE, 48 bytes */ + ncp->hdr_chunk = PNETCDF_RNDUP(MAX(CDF5_MIN_FSIZE, ncp->hdr_chunk), X_ALIGN); } } From 893546f3715e0e6bcd85997fde906f98feaacaf6 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Wed, 28 Oct 2026 15:28:50 -0700 Subject: [PATCH 3/5] Report NC_FORMAT_UNKNOWN when file size is smaller than the minimum The minimum file sizes for CDF-1, 2, and 5 are defined as the followings in src/include/dispatch.h #define CDF1_MIN_FSIZE 32 #define CDF2_MIN_FSIZE 32 #define CDF5_MIN_FSIZE 48 --- src/dispatchers/file.c | 65 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/src/dispatchers/file.c b/src/dispatchers/file.c index 420f3c0c1e..c93cb99831 100644 --- a/src/dispatchers/file.c +++ b/src/dispatchers/file.c @@ -2073,13 +2073,65 @@ ncmpi_inq_file_format(const char *filename, close(fd); /* ignore error */ DEBUG_RETURN_ERROR(NC_EFILE) } - if (close(fd) == -1) - DEBUG_RETURN_ERROR(NC_EFILE) if (memcmp(signature, cdf_signature, 3) == 0) { + /* The first 3 characters of the file are "CDF" */ if (signature[3] == 5) *formatp = NC_FORMAT_CDF5; else if (signature[3] == 2) *formatp = NC_FORMAT_CDF2; else if (signature[3] == 1) *formatp = NC_FORMAT_CLASSIC; + + /* Check if file size is at least the minimal sizes of CDF-1/2/5. */ + if (*formatp != NC_FORMAT_UNKNOWN) { + /* CDF-1 and 2's minimum header size is 32 bytes. + * CDF-5's minimum header size is 48 bytes. + * netcdf_file = header data + * header = magic numrecs dim_list gatt_list var_list + * CDF-1/2 + * 4 4 8 8 8 = 32 bytes + * CDF-5 + * 4 8 12 12 12 = 48 bytes + */ + off_t offset = lseek(fd, 0, SEEK_END); + if (*formatp == NC_FORMAT_CDF5 && offset < CDF5_MIN_FSIZE) { + *formatp = NC_FORMAT_UNKNOWN; + char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); + if (_env_str != NULL && *_env_str != '0') { + int _rank; + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + fprintf(stderr,"Rank %d: error at line %d of %s in %s\n", + _rank,__LINE__,__func__,__FILE__); + fprintf(stderr,"\tfile size (%lld) is less than CDF-5 minimum (%d)!\n",(long long)offset, CDF5_MIN_FSIZE); + } + } + else if (*formatp == NC_FORMAT_CDF2 && offset < CDF2_MIN_FSIZE) { + *formatp = NC_FORMAT_UNKNOWN; + char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); + if (_env_str != NULL && *_env_str != '0') { + int _rank; + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + fprintf(stderr,"Rank %d: error at line %d of %s in %s\n", + _rank,__LINE__,__func__,__FILE__); + fprintf(stderr,"\tfile size (%lld) is less than CDF-2 minimum (%d)!\n",(long long)offset, CDF2_MIN_FSIZE); + } + } + else if (*formatp == NC_FORMAT_CLASSIC && offset < CDF1_MIN_FSIZE) { + *formatp = NC_FORMAT_UNKNOWN; + char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); + if (_env_str != NULL && *_env_str != '0') { + int _rank; + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + fprintf(stderr,"Rank %d: error at line %d of %s in %s\n", + _rank,__LINE__,__func__,__FILE__); + fprintf(stderr,"\tfile size (%lld) is less than CDF-1 minimum (%d)!\n",(long long)offset, CDF1_MIN_FSIZE); + } + } + } + + /* close the file */ + if (close(fd) == -1) + DEBUG_RETURN_ERROR(NC_EFILE) + + return NC_NOERR; } /* check if the file is an HDF5. */ @@ -2093,7 +2145,9 @@ ncmpi_inq_file_format(const char *filename, */ off_t offset=0; - fd = open(path, O_RDONLY, 00400); /* error check already done */ + /* reset file pointer to the beginning of the file */ + lseek(fd, 0, SEEK_SET); + /* get first 8 bytes of file */ rlen = read(fd, signature, 8); /* error check already done */ @@ -2102,7 +2156,10 @@ ncmpi_inq_file_format(const char *filename, lseek(fd, offset, SEEK_SET); rlen = read(fd, signature, 8); } - close(fd); /* ignore error */ + + /* close the file */ + if (close(fd) == -1) + DEBUG_RETURN_ERROR(NC_EFILE) if (rlen == 8) { /* HDF5 signature found */ /* TODO: whether the file is NC_FORMAT_NETCDF4_CLASSIC is From 6db82b881d381dd6a4874f190b20d1657d942650 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Wed, 28 Oct 2026 15:32:54 -0700 Subject: [PATCH 4/5] ncvalidator - check and report if a file size is less than minimum Also, add three test netCDF files that contain only the netCDF signature, to be used for fail tests: bad_min.nc1, bad_min.nc2, and bad_min.nc5 --- src/utils/ncvalidator/Makefile.am | 5 ++- src/utils/ncvalidator/bad_min.nc1 | Bin 0 -> 8 bytes src/utils/ncvalidator/bad_min.nc2 | Bin 0 -> 8 bytes src/utils/ncvalidator/bad_min.nc5 | Bin 0 -> 8 bytes src/utils/ncvalidator/ncvalidator.c | 55 +++++++++++++++++++++++----- 5 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 src/utils/ncvalidator/bad_min.nc1 create mode 100644 src/utils/ncvalidator/bad_min.nc2 create mode 100644 src/utils/ncvalidator/bad_min.nc5 diff --git a/src/utils/ncvalidator/Makefile.am b/src/utils/ncvalidator/Makefile.am index 7441f838e0..f15dcf6272 100644 --- a/src/utils/ncvalidator/Makefile.am +++ b/src/utils/ncvalidator/Makefile.am @@ -65,7 +65,10 @@ ENOTNC_FILES = bad_magic.nc \ bad_tag_vattr.nc5 \ bad_tag_variable.nc1 \ bad_tag_variable.nc2 \ - bad_tag_variable.nc5 + bad_tag_variable.nc5 \ + bad_min.nc1 \ + bad_min.nc2 \ + bad_min.nc5 EVARSIZE_FILES = bad_large_fixed_var.nc2 \ bad_large_rec_2_vars.nc2 \ diff --git a/src/utils/ncvalidator/bad_min.nc1 b/src/utils/ncvalidator/bad_min.nc1 new file mode 100644 index 0000000000000000000000000000000000000000..67c45fc6a6592542ea24e6e9ae793e29b3324f16 GIT binary patch literal 8 NcmZ>Eabsiv0ssZ20M7sb literal 0 HcmV?d00001 diff --git a/src/utils/ncvalidator/bad_min.nc2 b/src/utils/ncvalidator/bad_min.nc2 new file mode 100644 index 0000000000000000000000000000000000000000..8a39b12b85214cb811de418629313f6cafebcb06 GIT binary patch literal 8 NcmZ>Eabsct0ssZ70MGyc literal 0 HcmV?d00001 diff --git a/src/utils/ncvalidator/bad_min.nc5 b/src/utils/ncvalidator/bad_min.nc5 new file mode 100644 index 0000000000000000000000000000000000000000..da708a43169c25bd2a67a4dfaac888fc21d7851c GIT binary patch literal 8 NcmZ>Eabslw0ssZM0Mh^f literal 0 HcmV?d00001 diff --git a/src/utils/ncvalidator/ncvalidator.c b/src/utils/ncvalidator/ncvalidator.c index 420e1f6277..a196653090 100644 --- a/src/utils/ncvalidator/ncvalidator.c +++ b/src/utils/ncvalidator/ncvalidator.c @@ -97,7 +97,14 @@ static const char *off_limit = "https://docs.unidata.ucar.edu/nug/current/file_s typedef int nc_type; -#define MIN_NC_XSZ 32 +/* + * The minimum file size of a CDF-1 or CDF-2 file is 32 bytes. + * For CDF-5 files, the minimum is 48 bytes. + */ +#define CDF1_MIN_FSIZE 32 +#define CDF2_MIN_FSIZE 32 +#define CDF5_MIN_FSIZE 48 + #define NC_DEFAULT_CHUNKSIZE 1048576 #define NC_FORMAT_UNKNOWN -1 @@ -2263,14 +2270,19 @@ val_get_NC(int fd, NC *ncp) /* Initialize the get buffer that stores the header read from the file */ getbuf.offset = 0; /* read from start of the file */ - /* CDF-5's minimum header size is 4 bytes more than CDF-1 and CDF-2's */ + /* CDF-1 and 2's minimum header size is 32 bytes. + * CDF-5's minimum header size is 48 bytes. + * netcdf_file = header data + * header = magic numrecs dim_list gatt_list var_list + * CDF-1/2 + * 4 4 8 8 8 = 32 bytes + * CDF-5 + * 4 8 12 12 12 = 48 bytes + */ getbuf.size = NC_DEFAULT_CHUNKSIZE; getbuf.pos = getbuf.base = (void *)malloc(getbuf.size); - /* Fetch the next header chunk. The chunk is 'gbp->size' bytes big - * netcdf_file = header data - * header = magic numrecs dim_list gatt_list var_list - */ + /* Fetch the first header chunk. The chunk is of size 'gbp->size' bytes. */ status = val_fetch(fd, &getbuf); if (status != NC_NOERR) goto fn_exit; @@ -2507,10 +2519,34 @@ check_signature(char *filename) else if (signature[3] == 2) format = NC_FORMAT_CDF2; else if (signature[3] == 1) format = NC_FORMAT_CLASSIC; + /* Check if file size is at least the minimal sizes of CDF-1/2/5. */ if (format != NC_FORMAT_UNKNOWN) { - close(fd); /* ignore error */ - return format; + /* CDF-1 and 2's minimum header size is 32 bytes. + * CDF-5's minimum header size is 48 bytes. + * netcdf_file = header data + * header = magic numrecs dim_list gatt_list var_list + * CDF-1/2 + * 4 4 8 8 8 = 32 bytes + * CDF-5 + * 4 8 12 12 12 = 48 bytes + */ + offset = lseek(fd, 0, SEEK_END); + if (format == NC_FORMAT_CDF5 && offset < CDF5_MIN_FSIZE) { + format = NC_FORMAT_UNKNOWN; + fprintf(stderr,"\tfile size (%lld) is less than CDF-5 minimum (%d)!\n",(long long)offset, CDF5_MIN_FSIZE); + } + else if (format == NC_FORMAT_CDF2 && offset < CDF2_MIN_FSIZE) { + format = NC_FORMAT_UNKNOWN; + fprintf(stderr,"\tfile size (%lld) is less than CDF-2 minimum (%d)!\n",(long long)offset, CDF2_MIN_FSIZE); + } + else if (format == NC_FORMAT_CLASSIC && offset < CDF1_MIN_FSIZE) { + format = NC_FORMAT_UNKNOWN; + fprintf(stderr,"\tfile size (%lld) is less than CDF-1 minimum (%d)!\n",(long long)offset, CDF1_MIN_FSIZE); + } } + + close(fd); /* ignore error */ + return format; } /* The HDF5 superblock is located by searching for the HDF5 format @@ -2631,7 +2667,8 @@ int main(int argc, char **argv) status = NC_EFILE; goto prog_exit; } - if (ncp->numrecs > 0) { + + if (ncp->numrecs > 0) { /* If the file contains record variables */ long long expect_fsize; expect_fsize = ncp->begin_rec + ncp->recsize * ncp->numrecs; if (expect_fsize < ncfilestat.st_size) { From 647b83d8e6fbc60ec972a6483ccc41f2f0530171 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Thu, 29 Oct 2026 12:03:22 -0700 Subject: [PATCH 5/5] sneak_peek.md - add note about PR 247 --- sneak_peek.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sneak_peek.md b/sneak_peek.md index 62efa25e4d..57bf7756c1 100644 --- a/sneak_peek.md +++ b/sneak_peek.md @@ -57,7 +57,11 @@ This is essentially a placeholder for the next release note ... + none * Other updates: - + none + + Check the size of a given netCDF file to be opened against the minimum size + calculated based on the CDF file format specifications. The error code + NC_ENOTNC will be returned if the file size is smaller than the minimum. + The minimum file size for CDF-1 and 2 is 32 bytes, and for CDF-5 48 bytes. + See [PR #247](https://github.com/Parallel-NetCDF/PnetCDF/pull/247). * Bug fixes + Fix PnetCDF package file, pnetcdf.pc.in, by removing GIO library as a