diff --git a/release_docs/CHANGELOG.md b/release_docs/CHANGELOG.md index 49d4d6c4501..c70c04d53f0 100644 --- a/release_docs/CHANGELOG.md +++ b/release_docs/CHANGELOG.md @@ -144,6 +144,10 @@ The `h5repack` tool now obtains its default low and high library version bounds ## Library +### Fixed a crash when reading a chunked dataset whose chunk rank mismatches the dataspace + + The chunk layout's stored dimensionality was validated against the dataspace rank at creation time, but not at open time, so a file whose stored chunk rank disagrees with its dataspace rank would not be caught. The resulting inconsistent selection ranks during chunk I/O caused a divide-by-zero in the hyperslab iterator. The chunk dimensionality is now also validated on open, and such a dataset is rejected with an error instead of crashing. + ### HTTP 403 errors in the ROS3 VFD for object keys with special characters The ROS3 VFD did not URI-encode the S3 object key when building the HTTP request path, so keys containing characters that AWS Signature Version 4 requires to be percent-encoded — such as the '=' in Hive-style `key=value` partition prefixes, '+', or spaces — produced a signed request whose signature did not match S3's server-side recomputation. S3 rejects such requests with `SignatureDoesNotMatch`, which surfaces as an HTTP 403 error (indistinguishable from a permissions error on a HEAD request), even though tools like the AWS CLI could access the same object. The object key is now percent-encoded exactly once when the request path is built, matching the behavior of other S3 clients. Note that URLs must now be passed to the ROS3 VFD with their object keys unencoded; a key that was pre-encoded as a workaround for this issue will now be double-encoded and fail to resolve. diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 4e08aca26a0..93e04471049 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -1120,6 +1120,13 @@ H5D__chunk_init(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool open_op) assert(dset); H5D_CHUNK_STORAGE_INDEX_CHK(sc); + /* When opening an existing dataset, validate that the stored chunk + * dimensionality is consistent with the dataspace rank. H5D__chunk_construct() + * performs this check at creation time, but it must also be performed + * here to catch malformed files. */ + if (open_op && dset->shared->layout.u.chunk.ndims != dset->shared->ndims + 1) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "dimensionality of chunks doesn't match the dataspace"); + if (NULL == (dapl = (H5P_genplist_t *)H5I_object(dapl_id))) HGOTO_ERROR(H5E_ID, H5E_BADID, FAIL, "can't find object for fapl ID"); diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index a41dbe746b6..81fce1c3dae 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -115,6 +115,7 @@ endforeach () # -------------------------------------------------------------------- set (HDF5_REFERENCE_TEST_FILES aggr.h5 + bad_chunk_ndims.h5 bad_compound.h5 bad_offset.h5 be_data.h5 @@ -1165,6 +1166,7 @@ endmacro () # generator executables set (H5_GENERATORS + gen_bad_chunk gen_bad_offset gen_bad_ohdr gen_bogus diff --git a/test/dsets.c b/test/dsets.c index ca40d2df8d6..627633d10ac 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -8031,6 +8031,64 @@ test_filters_endianess(void) return FAIL; } /* end test_filters_endianess() */ +/*------------------------------------------------------------------------- + * Function: test_chunk_dims_mismatch + * + * Purpose: Test that a malformed file whose stored chunk layout + * dimensionality does not match the dataset's dataspace rank + * is properly rejected at open-time. + * + * Return: Success: 0 + * Failure: -1 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_chunk_dims_mismatch(void) +{ + hid_t fid = H5I_INVALID_HID; + hid_t did = H5I_INVALID_HID; + const char *data_file = H5_get_srcdir_filename("bad_chunk_ndims.h5"); + + TESTING("rejection of chunk dimensionality that mismatches the dataspace"); + + if ((fid = H5Fopen(data_file, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) { + printf(" Could not open file %s. Try setting $srcdir to point at the " + "source directory of the test suite\n", + data_file); + goto error; + } + + /* Opening the dataset must fail cleanly */ + H5E_BEGIN_TRY + { + did = H5Dopen2(fid, "dset", H5P_DEFAULT); + } + H5E_END_TRY + + if (did >= 0) { + H5_FAILED(); + puts(" Opening a dataset with mismatched chunk/dataspace rank should have failed."); + goto error; + } + + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Dclose(did); + H5Fclose(fid); + } + H5E_END_TRY + return FAIL; +} /* end test_chunk_dims_mismatch() */ + /*------------------------------------------------------------------------- * Function: test_zero_dims * @@ -19064,6 +19122,7 @@ main(void) if (driver_is_default_compatible) { nerrors += (test_filters_endianess() < 0 ? 1 : 0); + nerrors += (test_chunk_dims_mismatch() < 0 ? 1 : 0); } nerrors += (test_zero_dims(file) < 0 ? 1 : 0); diff --git a/test/gen_bad_chunk.c b/test/gen_bad_chunk.c new file mode 100644 index 00000000000..a8ac2e2aca6 --- /dev/null +++ b/test/gen_bad_chunk.c @@ -0,0 +1,176 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the LICENSE file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Generate an HDF5 file with a chunked dataset whose stored chunk layout + * dimensionality does not match the dataset's dataspace rank. + * + * A valid 3-D chunked dataset of native int is written, then the version-3 + * chunk layout message's "dimensionality" byte is patched from 4 down to 3. + * The stored chunk rank includes an extra element-size dimension, so a valid + * 3-D dataset stores 4. Patching it to 3 makes the layout describe a 2-D + * chunk over a 3-D dataspace. + * + */ + +#include "h5test.h" + +#define BAD_CHUNK_FILE "bad_chunk_ndims.h5" +#define BAD_CHUNK_DSET "dset" + +/* Read an entire file into a newly allocated buffer. */ +static unsigned char * +slurp(const char *name, size_t *len_out) +{ + FILE *f = fopen(name, "rb"); + long len; + unsigned char *buf; + + if (!f) + return NULL; + if (fseek(f, 0, SEEK_END) != 0 || (len = ftell(f)) < 0 || fseek(f, 0, SEEK_SET) != 0) { + fclose(f); + return NULL; + } + if (NULL == (buf = malloc((size_t)len))) { + fclose(f); + return NULL; + } + if (fread(buf, 1, (size_t)len, f) != (size_t)len) { + free(buf); + fclose(f); + return NULL; + } + fclose(f); + *len_out = (size_t)len; + return buf; +} + +/* Write a buffer back out to a file. */ +static int +spew(const char *name, const unsigned char *buf, size_t len) +{ + FILE *f = fopen(name, "wb"); + if (!f) + return -1; + if (fwrite(buf, 1, len, f) != len) { + fclose(f); + return -1; + } + return fclose(f) == 0 ? 0 : -1; +} + +/* Find the unique occurrence of pattern in buf; return offset or (size_t)-1. */ +static size_t +find_once(const unsigned char *buf, size_t len, const unsigned char *pat, size_t patlen) +{ + size_t i, found = (size_t)-1; + + if (patlen == 0 || len < patlen) + return (size_t)-1; + for (i = 0; i <= len - patlen; i++) { + if (memcmp(buf + i, pat, patlen) == 0) { + if (found != (size_t)-1) + return (size_t)-1; /* not unique */ + found = i; + } + } + return found; +} + +int +main(void) +{ + hid_t fapl = H5I_INVALID_HID, file = H5I_INVALID_HID, sid = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID, dset = H5I_INVALID_HID; + hsize_t dims[3] = {3, 4, 5}; + hsize_t chunk[3] = {2, 2, 4}; /* edge lengths in elements; see the patch note below re: the 4 */ + int data[3 * 4 * 5]; + unsigned char *buf = NULL; + size_t len, off; + int i; + + /* The version-3 chunk layout stores, after the 3-byte header (version, + * class, dimensionality) and the 8-byte b-tree address, one little-endian + * uint32 per stored dimension: the three chunk edge lengths (in elements) + * followed by the element size in bytes. For this dataset those on-disk + * values are literally {2, 2, 4, 4} (the trailing 4 is sizeof(int)); this + * byte pattern uniquely locates the layout message. */ + static const unsigned char layout_dims[] = {2, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0}; + + for (i = 0; i < 3 * 4 * 5; i++) + data[i] = i; + + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + if (H5Pset_libver_bounds(fapl, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0) + TEST_ERROR; + if ((file = H5Fcreate(BAD_CHUNK_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR; + if ((sid = H5Screate_simple(3, dims, NULL)) < 0) + TEST_ERROR; + + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + if (H5Pset_chunk(dcpl, 3, chunk) < 0) + TEST_ERROR; + + if ((dset = H5Dcreate2(file, BAD_CHUNK_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + TEST_ERROR; + + if (H5Dclose(dset) < 0 || H5Pclose(dcpl) < 0 || H5Sclose(sid) < 0 || H5Fclose(file) < 0 || + H5Pclose(fapl) < 0) + TEST_ERROR; + + /* Patch the chunk layout's dimensionality byte from 4 to 3. */ + if (NULL == (buf = slurp(BAD_CHUNK_FILE, &len))) + TEST_ERROR; + if ((off = find_once(buf, len, layout_dims, sizeof(layout_dims))) == (size_t)-1) + TEST_ERROR; + /* Layout header is 3 bytes (version, class, ndims) + 8-byte address before + * the chunk sizes, so the ndims byte is 11 bytes before the sizes. + * + * Patching ndims from 4 to 3 makes the layout describe a 2-D chunk over the + * 3-D dataspace. The decoder then reads only the first three stored sizes, + * {2, 2, 4}, and treats the last of those (4) as the element-size + * dimension. Because the dataset's third chunk edge was chosen to be 4, + * that reinterpreted element size still equals sizeof(int), so the layout + * decodes consistently and the dataset opens instead of being rejected by + * the element-size check -- which is what let the original bug reach the + * I/O path and crash. */ + if (off < 11 || buf[off - 11] != 3 /* version */ || buf[off - 10] != 2 /* chunked */ || + buf[off - 9] != 4 /* ndims */) + TEST_ERROR; + buf[off - 9] = 3; + if (spew(BAD_CHUNK_FILE, buf, len) < 0) + TEST_ERROR; + + free(buf); + printf("Generated %s\n", BAD_CHUNK_FILE); + return EXIT_SUCCESS; + +error: + free(buf); + H5E_BEGIN_TRY + { + H5Dclose(dset); + H5Pclose(dcpl); + H5Sclose(sid); + H5Fclose(file); + H5Pclose(fapl); + } + H5E_END_TRY + fprintf(stderr, "failed to generate %s\n", BAD_CHUNK_FILE); + return EXIT_FAILURE; +} diff --git a/test/testfiles/bad_chunk_ndims.h5 b/test/testfiles/bad_chunk_ndims.h5 new file mode 100644 index 00000000000..83c5b0ded60 Binary files /dev/null and b/test/testfiles/bad_chunk_ndims.h5 differ