Reject chunked datasets with mismatched chunk/dspace rank at open time

release_docs/CHANGELOG.md

@@ -144,6 +144,10 @@ The `h5repack` tool now obtains its default low and high library version bounds
 
 ## Library
 
+### Fixed a crash when reading a chunked dataset whose chunk rank mismatches the dataspace
+
+   The chunk layout's stored dimensionality was validated against the dataspace rank at creation time, but not at open time, so a file whose stored chunk rank disagrees with its dataspace rank would not be caught. The resulting inconsistent selection ranks during chunk I/O caused a divide-by-zero in the hyperslab iterator. The chunk dimensionality is now also validated on open, and such a dataset is rejected with an error instead of crashing.
+
 ### HTTP 403 errors in the ROS3 VFD for object keys with special characters
 
    The ROS3 VFD did not URI-encode the S3 object key when building the HTTP request path, so keys containing characters that AWS Signature Version 4 requires to be percent-encoded — such as the '=' in Hive-style `key=value` partition prefixes, '+', or spaces — produced a signed request whose signature did not match S3's server-side recomputation. S3 rejects such requests with `SignatureDoesNotMatch`, which surfaces as an HTTP 403 error (indistinguishable from a permissions error on a HEAD request), even though tools like the AWS CLI could access the same object. The object key is now percent-encoded exactly once when the request path is built, matching the behavior of other S3 clients. Note that URLs must now be passed to the ROS3 VFD with their object keys unencoded; a key that was pre-encoded as a workaround for this issue will now be double-encoded and fail to resolve.

src/H5Dchunk.c

@@ -1120,6 +1120,13 @@ H5D__chunk_init(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool open_op)
     assert(dset);
     H5D_CHUNK_STORAGE_INDEX_CHK(sc);
 
+    /* When opening an existing dataset, validate that the stored chunk
+     * dimensionality is consistent with the dataspace rank.  H5D__chunk_construct()
+     * performs this check at creation time, but it must also be performed
+     * here to catch malformed files. */
+    if (open_op && dset->shared->layout.u.chunk.ndims != dset->shared->ndims + 1)
+        HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "dimensionality of chunks doesn't match the dataspace");
+
     if (NULL == (dapl = (H5P_genplist_t *)H5I_object(dapl_id)))
         HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for fapl ID");
 

test/CMakeTests.cmake

@@ -115,6 +115,7 @@ endforeach ()
 # --------------------------------------------------------------------
 set (HDF5_REFERENCE_TEST_FILES
     aggr.h5
+    bad_chunk_ndims.h5
     bad_compound.h5
     bad_offset.h5
     be_data.h5
@@ -1165,6 +1166,7 @@ endmacro ()
 
 # generator executables
 set (H5_GENERATORS
+    gen_bad_chunk
     gen_bad_offset
     gen_bad_ohdr
     gen_bogus

test/dsets.c

@@ -8031,6 +8031,64 @@ test_filters_endianess(void)
     return FAIL;
 } /* end test_filters_endianess() */
 
+/*-------------------------------------------------------------------------
+ * Function: test_chunk_dims_mismatch
+ *
+ * Purpose: Test that a malformed file whose stored chunk layout
+ *          dimensionality does not match the dataset's dataspace rank
+ *          is properly rejected at open-time.
+ * 
+ * Return: Success: 0
+ *         Failure: -1
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_chunk_dims_mismatch(void)
+{
+    hid_t       fid       = H5I_INVALID_HID;
+    hid_t       did       = H5I_INVALID_HID;
+    const char *data_file = H5_get_srcdir_filename("bad_chunk_ndims.h5");
+
+    TESTING("rejection of chunk dimensionality that mismatches the dataspace");
+
+    if ((fid = H5Fopen(data_file, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) {
+        printf("    Could not open file %s. Try set $srcdir to point at the "
+               "source directory of test\n",
+               data_file);
+        goto error;
+    }
+
+    /* Opening the dataset must fail cleanly */
+    H5E_BEGIN_TRY
+    {
+        did = H5Dopen2(fid, "dset", H5P_DEFAULT);
+    }
+    H5E_END_TRY
+
+    if (did >= 0) {
+        H5_FAILED();
+        puts("    Opening a dataset with mismatched chunk/dataspace rank should have failed.");
+        goto error;
+    }
+
+    if (H5Fclose(fid) < 0)
+        FAIL_STACK_ERROR;
+
+    PASSED();
+
+    return SUCCEED;
+
+error:
+    H5E_BEGIN_TRY
+    {
+        H5Dclose(did);
+        H5Fclose(fid);
+    }
+    H5E_END_TRY
+    return FAIL;
+} /* end test_chunk_dims_mismatch() */
+
 /*-------------------------------------------------------------------------
  * Function: test_zero_dims
  *
@@ -19064,6 +19122,7 @@ main(void)
 
                 if (driver_is_default_compatible) {
                     nerrors += (test_filters_endianess() < 0 ? 1 : 0);
+                    nerrors += (test_chunk_dims_mismatch() < 0 ? 1 : 0);
                 }
 
                 nerrors += (test_zero_dims(file) < 0 ? 1 : 0);

test/gen_bad_chunk.c

@@ -0,0 +1,164 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the LICENSE file, which can be found at the root of the source code       *
+ * distribution tree, or in https://www.hdfgroup.org/licenses.               *
+ * If you do not have access to either file, you may request a copy from     *
+ * help@hdfgroup.org.                                                        *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Generate an HDF5 file with a chunked dataset whose stored chunk layout
+ * dimensionality does not match the dataset's dataspace rank.
+ *
+ * A valid 3-D chunked dataset of native int is written, then the version-3
+ * chunk layout message's "dimensionality" byte is patched from 4 down to 3.
+ * The stored chunk rank includes an extra element-size dimension, so a valid
+ * 3-D dataset stores 4. Patching it to 3 makes the layout describe a 2-D
+ * chunk over a 3-D dataspace.
+ *
+ */
+
+#include "h5test.h"
+
+#define BAD_CHUNK_FILE "bad_chunk_ndims.h5"
+#define BAD_CHUNK_DSET "dset"
+
+/* Read an entire file into a newly allocated buffer. */
+static unsigned char *
+slurp(const char *name, size_t *len_out)
+{
+    FILE          *f = fopen(name, "rb");
+    long           len;
+    unsigned char *buf;
+
+    if (!f)
+        return NULL;
+    if (fseek(f, 0, SEEK_END) != 0 || (len = ftell(f)) < 0 || fseek(f, 0, SEEK_SET) != 0) {
+        fclose(f);
+        return NULL;
+    }
+    if (NULL == (buf = malloc((size_t)len))) {
+        fclose(f);
+        return NULL;
+    }
+    if (fread(buf, 1, (size_t)len, f) != (size_t)len) {
+        free(buf);
+        fclose(f);
+        return NULL;
+    }
+    fclose(f);
+    *len_out = (size_t)len;
+    return buf;
+}
+
+/* Write a buffer back out to a file. */
+static int
+spew(const char *name, const unsigned char *buf, size_t len)
+{
+    FILE *f = fopen(name, "wb");
+    if (!f)
+        return -1;
+    if (fwrite(buf, 1, len, f) != len) {
+        fclose(f);
+        return -1;
+    }
+    return fclose(f) == 0 ? 0 : -1;
+}
+
+/* Find the unique occurrence of pattern in buf; return offset or (size_t)-1. */
+static size_t
+find_once(const unsigned char *buf, size_t len, const unsigned char *pat, size_t patlen)
+{
+    size_t i, found = (size_t)-1;
+
+    if (patlen == 0 || len < patlen)
+        return (size_t)-1;
+    for (i = 0; i <= len - patlen; i++) {
+        if (memcmp(buf + i, pat, patlen) == 0) {
+            if (found != (size_t)-1)
+                return (size_t)-1; /* not unique */
+            found = i;
+        }
+    }
+    return found;
+}
+
+int
+main(void)
+{
+    hid_t          fapl = H5I_INVALID_HID, file = H5I_INVALID_HID, sid = H5I_INVALID_HID;
+    hid_t          dcpl = H5I_INVALID_HID, dset = H5I_INVALID_HID;
+    hsize_t        dims[3]  = {3, 4, 5};
+    hsize_t        chunk[3] = {2, 2, 4}; /* last chunk size equals sizeof(int) */
+    int            data[3 * 4 * 5];
+    unsigned char *buf = NULL;
+    size_t         len, off;
+    int            i;
+
+    /* Version-3 chunk layout message header immediately followed after the 8-byte b-tree address
+     * by the four chunk sizes {2, 2, 4, sizeof(int)=4} as little-endian
+     * uint32 values. This anchor uniquely locates the layout message. */
+    static const unsigned char layout_dims[] = {2, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0};
+
+    for (i = 0; i < 3 * 4 * 5; i++)
+        data[i] = i;
+
+    if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+        TEST_ERROR;
+    if (H5Pset_libver_bounds(fapl, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0)
+        TEST_ERROR;
+    if ((file = H5Fcreate(BAD_CHUNK_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
+        TEST_ERROR;
+    if ((sid = H5Screate_simple(3, dims, NULL)) < 0)
+        TEST_ERROR;
+
+    if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+        TEST_ERROR;
+    if (H5Pset_chunk(dcpl, 3, chunk) < 0)
+        TEST_ERROR;
+
+    if ((dset = H5Dcreate2(file, BAD_CHUNK_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0)
+        TEST_ERROR;
+    if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0)
+        TEST_ERROR;
+
+    if (H5Dclose(dset) < 0 || H5Pclose(dcpl) < 0 || H5Sclose(sid) < 0 || H5Fclose(file) < 0 ||
+        H5Pclose(fapl) < 0)
+        TEST_ERROR;
+
+    /* Patch the chunk layout's dimensionality byte from 4 to 3. */
+    if (NULL == (buf = slurp(BAD_CHUNK_FILE, &len)))
+        TEST_ERROR;
+    if ((off = find_once(buf, len, layout_dims, sizeof(layout_dims))) == (size_t)-1)
+        TEST_ERROR;
+    /* Layout header is 3 bytes (version, class, ndims) + 8-byte address before
+     * the chunk sizes, so the ndims byte is 11 bytes before the sizes. */
+    if (off < 11 || buf[off - 11] != 3 /* version */ || buf[off - 10] != 2 /* chunked */ ||
+        buf[off - 9] != 4 /* ndims */)
+        TEST_ERROR;
+    buf[off - 9] = 3;
+    if (spew(BAD_CHUNK_FILE, buf, len) < 0)
+        TEST_ERROR;
+
+    free(buf);
+    printf("Generated %s\n", BAD_CHUNK_FILE);
+    return EXIT_SUCCESS;
+
+error:
+    free(buf);
+    H5E_BEGIN_TRY
+    {
+        H5Dclose(dset);
+        H5Pclose(dcpl);
+        H5Sclose(sid);
+        H5Fclose(file);
+        H5Pclose(fapl);
+    }
+    H5E_END_TRY
+    fprintf(stderr, "failed to generate %s\n", BAD_CHUNK_FILE);
+    return EXIT_FAILURE;
+}