diff --git a/sneak_peek.md b/sneak_peek.md index 62efa25e4..57bf7756c 100644 --- a/sneak_peek.md +++ b/sneak_peek.md @@ -57,7 +57,11 @@ This is essentially a placeholder for the next release note ... + none * Other updates: - + none + + Check the size of a given netCDF file to be opened against the minimum size + calculated based on the CDF file format specifications. The error code + NC_ENOTNC will be returned if the file size is smaller than the minimum. + The minimum file size for CDF-1 and 2 is 32 bytes, and for CDF-5 48 bytes. + See [PR #247](https://github.com/Parallel-NetCDF/PnetCDF/pull/247). * Bug fixes + Fix PnetCDF package file, pnetcdf.pc.in, by removing GIO library as a diff --git a/src/dispatchers/file.c b/src/dispatchers/file.c index 420f3c0c1..c93cb9983 100644 --- a/src/dispatchers/file.c +++ b/src/dispatchers/file.c @@ -2073,13 +2073,65 @@ ncmpi_inq_file_format(const char *filename, close(fd); /* ignore error */ DEBUG_RETURN_ERROR(NC_EFILE) } - if (close(fd) == -1) - DEBUG_RETURN_ERROR(NC_EFILE) if (memcmp(signature, cdf_signature, 3) == 0) { + /* The first 3 characters of the file are "CDF" */ if (signature[3] == 5) *formatp = NC_FORMAT_CDF5; else if (signature[3] == 2) *formatp = NC_FORMAT_CDF2; else if (signature[3] == 1) *formatp = NC_FORMAT_CLASSIC; + + /* Check if file size is at least the minimal sizes of CDF-1/2/5. */ + if (*formatp != NC_FORMAT_UNKNOWN) { + /* CDF-1 and 2's minimum header size is 32 bytes. + * CDF-5's minimum header size is 48 bytes. + * netcdf_file = header data + * header = magic numrecs dim_list gatt_list var_list + * CDF-1/2 + * 4 4 8 8 8 = 32 bytes + * CDF-5 + * 4 8 12 12 12 = 48 bytes + */ + off_t offset = lseek(fd, 0, SEEK_END); + if (*formatp == NC_FORMAT_CDF5 && offset < CDF5_MIN_FSIZE) { + *formatp = NC_FORMAT_UNKNOWN; + char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); + if (_env_str != NULL && *_env_str != '0') { + int _rank; + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + fprintf(stderr,"Rank %d: error at line %d of %s in %s\n", + _rank,__LINE__,__func__,__FILE__); + fprintf(stderr,"\tfile size (%lld) is less than CDF-5 minimum (%d)!\n",(long long)offset, CDF5_MIN_FSIZE); + } + } + else if (*formatp == NC_FORMAT_CDF2 && offset < CDF2_MIN_FSIZE) { + *formatp = NC_FORMAT_UNKNOWN; + char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); + if (_env_str != NULL && *_env_str != '0') { + int _rank; + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + fprintf(stderr,"Rank %d: error at line %d of %s in %s\n", + _rank,__LINE__,__func__,__FILE__); + fprintf(stderr,"\tfile size (%lld) is less than CDF-2 minimum (%d)!\n",(long long)offset, CDF2_MIN_FSIZE); + } + } + else if (*formatp == NC_FORMAT_CLASSIC && offset < CDF1_MIN_FSIZE) { + *formatp = NC_FORMAT_UNKNOWN; + char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE"); + if (_env_str != NULL && *_env_str != '0') { + int _rank; + MPI_Comm_rank(MPI_COMM_WORLD, &_rank); + fprintf(stderr,"Rank %d: error at line %d of %s in %s\n", + _rank,__LINE__,__func__,__FILE__); + fprintf(stderr,"\tfile size (%lld) is less than CDF-1 minimum (%d)!\n",(long long)offset, CDF1_MIN_FSIZE); + } + } + } + + /* close the file */ + if (close(fd) == -1) + DEBUG_RETURN_ERROR(NC_EFILE) + + return NC_NOERR; } /* check if the file is an HDF5. */ @@ -2093,7 +2145,9 @@ ncmpi_inq_file_format(const char *filename, */ off_t offset=0; - fd = open(path, O_RDONLY, 00400); /* error check already done */ + /* reset file pointer to the beginning of the file */ + lseek(fd, 0, SEEK_SET); + /* get first 8 bytes of file */ rlen = read(fd, signature, 8); /* error check already done */ @@ -2102,7 +2156,10 @@ ncmpi_inq_file_format(const char *filename, lseek(fd, offset, SEEK_SET); rlen = read(fd, signature, 8); } - close(fd); /* ignore error */ + + /* close the file */ + if (close(fd) == -1) + DEBUG_RETURN_ERROR(NC_EFILE) if (rlen == 8) { /* HDF5 signature found */ /* TODO: whether the file is NC_FORMAT_NETCDF4_CLASSIC is diff --git a/src/drivers/ncmpio/ncmpio_NC.h b/src/drivers/ncmpio/ncmpio_NC.h index b0fdc0c87..de1bdbe47 100644 --- a/src/drivers/ncmpio/ncmpio_NC.h +++ b/src/drivers/ncmpio/ncmpio_NC.h @@ -105,12 +105,6 @@ typedef struct { * and vice versa */ #define NCMPII_ECHAR(nctype, mpitype) ((((nctype) == NC_CHAR) == ((mpitype) != MPI_CHAR)) ? NC_ECHAR : NC_NOERR) -/* - * The external size of an empty netcdf version 1 file is 32 bytes, which is - * also set as the initial value of ncp->xsz. - */ -#define MIN_NC_XSZ 32 - typedef enum { NC_UNSPECIFIED = 0, /* ABSENT */ NC_DIMENSION = 10, /* \x00 \x00 \x00 \x0A */ diff --git a/src/drivers/ncmpio/ncmpio_util.c b/src/drivers/ncmpio/ncmpio_util.c index bbee25799..01caf472d 100644 --- a/src/drivers/ncmpio/ncmpio_util.c +++ b/src/drivers/ncmpio/ncmpio_util.c @@ -135,14 +135,16 @@ void ncmpio_hint_extract(NC *ncp, ncp->hdr_chunk = PNC_HDR_READ_CHUNK_SIZE; else if (llval > NC_MAX_INT) /* limit to NC_MAX_INT */ ncp->hdr_chunk = NC_MAX_INT; - else if (llval < 48) - /* minimum file sizes: 32 bytes for CDF-1 and 2, 48 for CDF5 */ - ncp->hdr_chunk = 48; + else if (llval < CDF5_MIN_FSIZE) + /* minimum file header size for CDF-1 and 2 files is 32 bytes, + * and 48 bytes for CDF-5. + */ + ncp->hdr_chunk = CDF5_MIN_FSIZE; else ncp->hdr_chunk = (int)llval; - /* CDF-5's minimum header size is 4 bytes more than CDF-1 2's */ - ncp->hdr_chunk = PNETCDF_RNDUP(MAX(MIN_NC_XSZ+4, ncp->hdr_chunk), + /* CDF-5's minimum header size is CDF5_MIN_FSIZE, 48 bytes */ + ncp->hdr_chunk = PNETCDF_RNDUP(MAX(CDF5_MIN_FSIZE, ncp->hdr_chunk), X_ALIGN); } } diff --git a/src/include/dispatch.h b/src/include/dispatch.h index 873e9fc6e..2fab86f89 100644 --- a/src/include/dispatch.h +++ b/src/include/dispatch.h @@ -10,6 +10,14 @@ #include #include +/* + * The minimum file size of a CDF-1 or CDF-2 file is 32 bytes. + * For CDF-5 files, the minimum is 48 bytes. + */ +#define CDF1_MIN_FSIZE 32 +#define CDF2_MIN_FSIZE 32 +#define CDF5_MIN_FSIZE 48 + /* various I/O modes or types */ #define NC_REQ_COLL 0x00000001 /* collective request */ #define NC_REQ_INDEP 0x00000002 /* independent request */ diff --git a/src/utils/ncvalidator/Makefile.am b/src/utils/ncvalidator/Makefile.am index 7441f838e..f15dcf627 100644 --- a/src/utils/ncvalidator/Makefile.am +++ b/src/utils/ncvalidator/Makefile.am @@ -65,7 +65,10 @@ ENOTNC_FILES = bad_magic.nc \ bad_tag_vattr.nc5 \ bad_tag_variable.nc1 \ bad_tag_variable.nc2 \ - bad_tag_variable.nc5 + bad_tag_variable.nc5 \ + bad_min.nc1 \ + bad_min.nc2 \ + bad_min.nc5 EVARSIZE_FILES = bad_large_fixed_var.nc2 \ bad_large_rec_2_vars.nc2 \ diff --git a/src/utils/ncvalidator/bad_min.nc1 b/src/utils/ncvalidator/bad_min.nc1 new file mode 100644 index 000000000..67c45fc6a Binary files /dev/null and b/src/utils/ncvalidator/bad_min.nc1 differ diff --git a/src/utils/ncvalidator/bad_min.nc2 b/src/utils/ncvalidator/bad_min.nc2 new file mode 100644 index 000000000..8a39b12b8 Binary files /dev/null and b/src/utils/ncvalidator/bad_min.nc2 differ diff --git a/src/utils/ncvalidator/bad_min.nc5 b/src/utils/ncvalidator/bad_min.nc5 new file mode 100644 index 000000000..da708a431 Binary files /dev/null and b/src/utils/ncvalidator/bad_min.nc5 differ diff --git a/src/utils/ncvalidator/ncvalidator.c b/src/utils/ncvalidator/ncvalidator.c index 420e1f627..a19665309 100644 --- a/src/utils/ncvalidator/ncvalidator.c +++ b/src/utils/ncvalidator/ncvalidator.c @@ -97,7 +97,14 @@ static const char *off_limit = "https://docs.unidata.ucar.edu/nug/current/file_s typedef int nc_type; -#define MIN_NC_XSZ 32 +/* + * The minimum file size of a CDF-1 or CDF-2 file is 32 bytes. + * For CDF-5 files, the minimum is 48 bytes. + */ +#define CDF1_MIN_FSIZE 32 +#define CDF2_MIN_FSIZE 32 +#define CDF5_MIN_FSIZE 48 + #define NC_DEFAULT_CHUNKSIZE 1048576 #define NC_FORMAT_UNKNOWN -1 @@ -2263,14 +2270,19 @@ val_get_NC(int fd, NC *ncp) /* Initialize the get buffer that stores the header read from the file */ getbuf.offset = 0; /* read from start of the file */ - /* CDF-5's minimum header size is 4 bytes more than CDF-1 and CDF-2's */ + /* CDF-1 and 2's minimum header size is 32 bytes. + * CDF-5's minimum header size is 48 bytes. + * netcdf_file = header data + * header = magic numrecs dim_list gatt_list var_list + * CDF-1/2 + * 4 4 8 8 8 = 32 bytes + * CDF-5 + * 4 8 12 12 12 = 48 bytes + */ getbuf.size = NC_DEFAULT_CHUNKSIZE; getbuf.pos = getbuf.base = (void *)malloc(getbuf.size); - /* Fetch the next header chunk. The chunk is 'gbp->size' bytes big - * netcdf_file = header data - * header = magic numrecs dim_list gatt_list var_list - */ + /* Fetch the first header chunk. The chunk is of size 'gbp->size' bytes. */ status = val_fetch(fd, &getbuf); if (status != NC_NOERR) goto fn_exit; @@ -2507,10 +2519,34 @@ check_signature(char *filename) else if (signature[3] == 2) format = NC_FORMAT_CDF2; else if (signature[3] == 1) format = NC_FORMAT_CLASSIC; + /* Check if file size is at least the minimal sizes of CDF-1/2/5. */ if (format != NC_FORMAT_UNKNOWN) { - close(fd); /* ignore error */ - return format; + /* CDF-1 and 2's minimum header size is 32 bytes. + * CDF-5's minimum header size is 48 bytes. + * netcdf_file = header data + * header = magic numrecs dim_list gatt_list var_list + * CDF-1/2 + * 4 4 8 8 8 = 32 bytes + * CDF-5 + * 4 8 12 12 12 = 48 bytes + */ + offset = lseek(fd, 0, SEEK_END); + if (format == NC_FORMAT_CDF5 && offset < CDF5_MIN_FSIZE) { + format = NC_FORMAT_UNKNOWN; + fprintf(stderr,"\tfile size (%lld) is less than CDF-5 minimum (%d)!\n",(long long)offset, CDF5_MIN_FSIZE); + } + else if (format == NC_FORMAT_CDF2 && offset < CDF2_MIN_FSIZE) { + format = NC_FORMAT_UNKNOWN; + fprintf(stderr,"\tfile size (%lld) is less than CDF-2 minimum (%d)!\n",(long long)offset, CDF2_MIN_FSIZE); + } + else if (format == NC_FORMAT_CLASSIC && offset < CDF1_MIN_FSIZE) { + format = NC_FORMAT_UNKNOWN; + fprintf(stderr,"\tfile size (%lld) is less than CDF-1 minimum (%d)!\n",(long long)offset, CDF1_MIN_FSIZE); + } } + + close(fd); /* ignore error */ + return format; } /* The HDF5 superblock is located by searching for the HDF5 format @@ -2631,7 +2667,8 @@ int main(int argc, char **argv) status = NC_EFILE; goto prog_exit; } - if (ncp->numrecs > 0) { + + if (ncp->numrecs > 0) { /* If the file contains record variables */ long long expect_fsize; expect_fsize = ncp->begin_rec + ncp->recsize * ncp->numrecs; if (expect_fsize < ncfilestat.st_size) {