Check input file size against minimum netCDF file size

sneak_peek.md

@@ -57,7 +57,11 @@ This is essentially a placeholder for the next release note ...
   + none
 
 * Other updates:
-  + none
+  + Check the size of a given netCDF file to be opened against the minimum size
+    calculated based on the CDF file format specifications. The error code
+    NC_ENOTNC will be returned if the file size is smaller than the minimum.
+    The minimum file size for CDF-1 and 2 is 32 bytes, and for CDF-5 48 bytes.
+    See [PR #247](https://github.com/Parallel-NetCDF/PnetCDF/pull/247).
 
 * Bug fixes
   + Fix PnetCDF package file, pnetcdf.pc.in, by removing GIO library as a

src/dispatchers/file.c

@@ -2073,13 +2073,65 @@ ncmpi_inq_file_format(const char *filename,
         close(fd); /* ignore error */
         DEBUG_RETURN_ERROR(NC_EFILE)
     }
-    if (close(fd) == -1)
-        DEBUG_RETURN_ERROR(NC_EFILE)
 
     if (memcmp(signature, cdf_signature, 3) == 0) {
+        /* The first 3 characters of the file are "CDF" */
              if (signature[3] == 5)  *formatp = NC_FORMAT_CDF5;
         else if (signature[3] == 2)  *formatp = NC_FORMAT_CDF2;
         else if (signature[3] == 1)  *formatp = NC_FORMAT_CLASSIC;
+
+        /* Check if file size is at least the minimal sizes of CDF-1/2/5. */
+        if (*formatp != NC_FORMAT_UNKNOWN) {
+            /* CDF-1 and 2's minimum header size is 32 bytes.
+             * CDF-5's minimum header size is 48 bytes.
+             * netcdf_file = header data
+             * header      = magic numrecs dim_list gatt_list var_list
+             * CDF-1/2
+             *               4     4       8        8         8   = 32 bytes
+             * CDF-5
+             *               4     8       12       12        12  = 48 bytes
+             */
+            off_t offset = lseek(fd, 0, SEEK_END);
+            if (*formatp == NC_FORMAT_CDF5 && offset < CDF5_MIN_FSIZE) {
+                *formatp = NC_FORMAT_UNKNOWN;
+                char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE");
+                if (_env_str != NULL && *_env_str != '0') {
+                    int _rank;
+                    MPI_Comm_rank(MPI_COMM_WORLD, &_rank);
+                    fprintf(stderr,"Rank %d: error at line %d of %s in %s\n",
+                    _rank,__LINE__,__func__,__FILE__);
+                    fprintf(stderr,"\tfile size (%lld) is less than CDF-5 minimum (%d)!\n",(long long)offset, CDF5_MIN_FSIZE);
+                }
+            }
+            else if (*formatp == NC_FORMAT_CDF2 && offset < CDF2_MIN_FSIZE) {
+                *formatp = NC_FORMAT_UNKNOWN;
+                char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE");
+                if (_env_str != NULL && *_env_str != '0') {
+                    int _rank;
+                    MPI_Comm_rank(MPI_COMM_WORLD, &_rank);
+                    fprintf(stderr,"Rank %d: error at line %d of %s in %s\n",
+                    _rank,__LINE__,__func__,__FILE__);
+                    fprintf(stderr,"\tfile size (%lld) is less than CDF-2 minimum (%d)!\n",(long long)offset, CDF2_MIN_FSIZE);
+                }
+            }
+            else if (*formatp == NC_FORMAT_CLASSIC && offset < CDF1_MIN_FSIZE) {
+                *formatp = NC_FORMAT_UNKNOWN;
+                char *_env_str = getenv("PNETCDF_VERBOSE_DEBUG_MODE");
+                if (_env_str != NULL && *_env_str != '0') {
+                    int _rank;
+                    MPI_Comm_rank(MPI_COMM_WORLD, &_rank);
+                    fprintf(stderr,"Rank %d: error at line %d of %s in %s\n",
+                    _rank,__LINE__,__func__,__FILE__);
+                    fprintf(stderr,"\tfile size (%lld) is less than CDF-1 minimum (%d)!\n",(long long)offset, CDF1_MIN_FSIZE);
+                }
+            }
+        }
+
+        /* close the file */
+        if (close(fd) == -1)
+            DEBUG_RETURN_ERROR(NC_EFILE)
+
+        return NC_NOERR;
     }
 
     /* check if the file is an HDF5. */
@@ -2093,7 +2145,9 @@ ncmpi_inq_file_format(const char *filename,
          */
         off_t offset=0;
 
-        fd = open(path, O_RDONLY, 00400); /* error check already done */
+        /* reset file pointer to the beginning of the file */
+        lseek(fd, 0, SEEK_SET);
+
         /* get first 8 bytes of file */
         rlen = read(fd, signature, 8); /* error check already done */
 
@@ -2102,7 +2156,10 @@ ncmpi_inq_file_format(const char *filename,
             lseek(fd, offset, SEEK_SET);
             rlen = read(fd, signature, 8);
         }
-        close(fd); /* ignore error */
+
+        /* close the file */
+        if (close(fd) == -1)
+            DEBUG_RETURN_ERROR(NC_EFILE)
 
         if (rlen == 8) { /* HDF5 signature found */
             /* TODO: whether the file is NC_FORMAT_NETCDF4_CLASSIC is

src/drivers/ncmpio/ncmpio_NC.h

@@ -105,12 +105,6 @@ typedef struct {
  * and vice versa */
 #define NCMPII_ECHAR(nctype, mpitype) ((((nctype) == NC_CHAR) == ((mpitype) != MPI_CHAR)) ? NC_ECHAR : NC_NOERR)
 
-/*
- * The external size of an empty netcdf version 1 file is 32 bytes, which is
- * also set as the initial value of ncp->xsz.
- */
-#define MIN_NC_XSZ 32
-
 typedef enum {
     NC_UNSPECIFIED =  0,  /* ABSENT */
     NC_DIMENSION   = 10,  /* \x00 \x00 \x00 \x0A */

src/drivers/ncmpio/ncmpio_util.c

@@ -135,14 +135,16 @@ void ncmpio_hint_extract(NC       *ncp,
                 ncp->hdr_chunk = PNC_HDR_READ_CHUNK_SIZE;
             else if (llval > NC_MAX_INT) /* limit to NC_MAX_INT */
                 ncp->hdr_chunk = NC_MAX_INT;
-            else if (llval < 48)
-                /* minimum file sizes: 32 bytes for CDF-1 and 2, 48 for CDF5 */
-                ncp->hdr_chunk = 48;
+            else if (llval < CDF5_MIN_FSIZE)
+                /* minimum file header size for CDF-1 and 2 files is 32 bytes,
+                 * and 48 bytes for CDF-5.
+                 */
+                ncp->hdr_chunk = CDF5_MIN_FSIZE;
             else
                 ncp->hdr_chunk = (int)llval;
 
-            /* CDF-5's minimum header size is 4 bytes more than CDF-1 2's */
-            ncp->hdr_chunk = PNETCDF_RNDUP(MAX(MIN_NC_XSZ+4, ncp->hdr_chunk),
+            /* CDF-5's minimum header size is CDF5_MIN_FSIZE, 48 bytes */
+            ncp->hdr_chunk = PNETCDF_RNDUP(MAX(CDF5_MIN_FSIZE, ncp->hdr_chunk),
                                            X_ALIGN);
         }
     }

src/include/dispatch.h

@@ -10,6 +10,14 @@
 #include <pnetcdf.h>
 #include <mpi.h>
 
+/*
+ * The minimum file size of a CDF-1 or CDF-2 file is 32 bytes.
+ * For CDF-5 files, the minimum is 48 bytes.
+ */
+#define CDF1_MIN_FSIZE 32
+#define CDF2_MIN_FSIZE 32
+#define CDF5_MIN_FSIZE 48
+
 /* various I/O modes or types */
 #define NC_REQ_COLL    0x00000001  /* collective request */
 #define NC_REQ_INDEP   0x00000002  /* independent request */

src/utils/ncvalidator/Makefile.am

@@ -65,7 +65,10 @@ ENOTNC_FILES =   bad_magic.nc \
                  bad_tag_vattr.nc5 \
                  bad_tag_variable.nc1 \
                  bad_tag_variable.nc2 \
-                 bad_tag_variable.nc5
+                 bad_tag_variable.nc5 \
+                 bad_min.nc1 \
+                 bad_min.nc2 \
+                 bad_min.nc5
 
 EVARSIZE_FILES = bad_large_fixed_var.nc2 \
                  bad_large_rec_2_vars.nc2 \

src/utils/ncvalidator/ncvalidator.c

@@ -97,7 +97,14 @@ static const char *off_limit = "https://docs.unidata.ucar.edu/nug/current/file_s
 
 typedef int nc_type;
 
-#define MIN_NC_XSZ 32
+/*
+ * The minimum file size of a CDF-1 or CDF-2 file is 32 bytes.
+ * For CDF-5 files, the minimum is 48 bytes.
+ */
+#define CDF1_MIN_FSIZE 32
+#define CDF2_MIN_FSIZE 32
+#define CDF5_MIN_FSIZE 48
+
 #define NC_DEFAULT_CHUNKSIZE 1048576
 
 #define NC_FORMAT_UNKNOWN         -1
@@ -2263,14 +2270,19 @@ val_get_NC(int fd, NC *ncp)
     /* Initialize the get buffer that stores the header read from the file */
     getbuf.offset = 0;     /* read from start of the file */
 
-    /* CDF-5's minimum header size is 4 bytes more than CDF-1 and CDF-2's */
+    /* CDF-1 and 2's minimum header size is 32 bytes.
+     * CDF-5's minimum header size is 48 bytes.
+     * netcdf_file = header data
+     * header      = magic numrecs dim_list gatt_list var_list
+     * CDF-1/2
+     *               4     4       8        8         8           = 32 bytes
+     * CDF-5
+     *               4     8       12       12        12          = 48 bytes
+     */
     getbuf.size = NC_DEFAULT_CHUNKSIZE;
     getbuf.pos  = getbuf.base = (void *)malloc(getbuf.size);
 
-    /* Fetch the next header chunk. The chunk is 'gbp->size' bytes big
-     * netcdf_file = header data
-     * header      = magic numrecs dim_list gatt_list var_list
-    */
+    /* Fetch the first header chunk. The chunk is of size 'gbp->size' bytes. */
     status = val_fetch(fd, &getbuf);
     if (status != NC_NOERR) goto fn_exit;
 
@@ -2507,10 +2519,34 @@ check_signature(char *filename)
         else if (signature[3] == 2) format = NC_FORMAT_CDF2;
         else if (signature[3] == 1) format = NC_FORMAT_CLASSIC;
 
+        /* Check if file size is at least the minimal sizes of CDF-1/2/5. */
         if (format != NC_FORMAT_UNKNOWN) {
-            close(fd); /* ignore error */
-            return format;
+            /* CDF-1 and 2's minimum header size is 32 bytes.
+             * CDF-5's minimum header size is 48 bytes.
+             * netcdf_file = header data
+             * header      = magic numrecs dim_list gatt_list var_list
+             * CDF-1/2
+             *               4     4       8        8         8   = 32 bytes
+             * CDF-5
+             *               4     8       12       12        12  = 48 bytes
+             */
+            offset = lseek(fd, 0, SEEK_END);
+            if (format == NC_FORMAT_CDF5 && offset < CDF5_MIN_FSIZE) {
+                format = NC_FORMAT_UNKNOWN;
+                fprintf(stderr,"\tfile size (%lld) is less than CDF-5 minimum (%d)!\n",(long long)offset, CDF5_MIN_FSIZE);
+            }
+            else if (format == NC_FORMAT_CDF2 && offset < CDF2_MIN_FSIZE) {
+                format = NC_FORMAT_UNKNOWN;
+                fprintf(stderr,"\tfile size (%lld) is less than CDF-2 minimum (%d)!\n",(long long)offset, CDF2_MIN_FSIZE);
+            }
+            else if (format == NC_FORMAT_CLASSIC && offset < CDF1_MIN_FSIZE) {
+                format = NC_FORMAT_UNKNOWN;
+                fprintf(stderr,"\tfile size (%lld) is less than CDF-1 minimum (%d)!\n",(long long)offset, CDF1_MIN_FSIZE);
+            }
         }
+
+        close(fd); /* ignore error */
+        return format;
     }
 
     /* The HDF5 superblock is located by searching for the HDF5 format
@@ -2631,7 +2667,8 @@ int main(int argc, char **argv)
         status = NC_EFILE;
         goto prog_exit;
     }
-    if (ncp->numrecs > 0) {
+
+    if (ncp->numrecs > 0) { /* If the file contains record variables */
         long long expect_fsize;
         expect_fsize = ncp->begin_rec + ncp->recsize * ncp->numrecs;
         if (expect_fsize < ncfilestat.st_size) {