Skip to content

feat(storage): Introduce unified checksum options and deprecate old options #16260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
v-pratap wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(storage): Introduce unified checksum options and deprecate old options #16260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions google/cloud/storage/async/options.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,13 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
* checksum of an object during uploads and downloads.
*
* The option has no effect if the service does not return a CRC32C hash.
*
* @deprecated Use `UploadChecksumValidationOption` and
* `DownloadChecksumValidationOption` instead.
*/
struct EnableCrc32cValidationOption {
struct [[deprecated(
"Use UploadChecksumValidationOption and DownloadChecksumValidationOption "
"instead")]] EnableCrc32cValidationOption {
using Type = bool;
};

Expand All @@ -54,8 +59,13 @@ struct UseCrc32cValueOption {
*
* The option has no effect for partial downloads or any other circumstance
* where the service does not return a MD5 hash.
*
* @deprecated Use `UploadChecksumValidationOption` and
* `DownloadChecksumValidationOption` instead.
*/
struct EnableMD5ValidationOption {
struct [[deprecated(
"Use UploadChecksumValidationOption and DownloadChecksumValidationOption "
"instead")]] EnableMD5ValidationOption {
using Type = bool;
};

Expand Down
18 changes: 16 additions & 2 deletions google/cloud/storage/hashing_options.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_HASHING_OPTIONS_H
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_HASHING_OPTIONS_H

#include "google/cloud/internal/disable_deprecation_warnings.inc"
#include "google/cloud/storage/internal/complex_option.h"
#include "google/cloud/storage/version.h"
#include "google/cloud/version.h"
#include "absl/strings/string_view.h"
#include <string>

Expand Down Expand Up @@ -70,8 +72,14 @@ inline std::string ComputeMD5Hash(char const* payload) {
* for most applications. Disabling CRC32C checksums while MD5 hashes remain
* disabled exposes your application to data corruption. We recommend that all
* uploads to GCS and downloads from GCS use CRC32C checksums.
*
* @deprecated Use `UploadChecksumValidationOption` and
* `DownloadChecksumValidationOption` instead.
*/
struct DisableMD5Hash : public internal::ComplexOption<DisableMD5Hash, bool> {
struct [[deprecated(
"Use UploadChecksumValidationOption and DownloadChecksumValidationOption "
"instead")]] DisableMD5Hash
: public internal::ComplexOption<DisableMD5Hash, bool> {
using ComplexOption<DisableMD5Hash, bool>::ComplexOption;
// GCC <= 7.0 does not use the inherited default constructor, redeclare it
// explicitly
Expand Down Expand Up @@ -134,8 +142,13 @@ inline std::string ComputeCrc32cChecksum(char const* payload) {
* for most applications. Disabling CRC32C checksums while MD5 hashes remain
* disabled exposes your application to data corruption. We recommend that all
* uploads to GCS and downloads from GCS use CRC32C checksums.
*
* @deprecated Use `UploadChecksumValidationOption` and
* `DownloadChecksumValidationOption` instead.
*/
struct DisableCrc32cChecksum
struct [[deprecated(
"Use UploadChecksumValidationOption and DownloadChecksumValidationOption "
"instead")]] DisableCrc32cChecksum
: public internal::ComplexOption<DisableCrc32cChecksum, bool> {
using ComplexOption<DisableCrc32cChecksum, bool>::ComplexOption;
// GCC <= 7.0 does not use the inherited default constructor, redeclare it
Expand All @@ -149,4 +162,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
} // namespace cloud
} // namespace google

#include "google/cloud/internal/diagnostics_pop.inc"
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_HASHING_OPTIONS_H
19 changes: 19 additions & 0 deletions google/cloud/storage/hashing_options_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
// limitations under the License.

#include "google/cloud/storage/hashing_options.h"
#include "google/cloud/storage/options.h"
#include "google/cloud/options.h"
#include <gmock/gmock.h>
#include <string>

Expand Down Expand Up @@ -54,6 +56,23 @@ TEST(ComputeCrc32cChecksumTest, Simple) {
EXPECT_EQ("ImIEBA==", actual);
}

TEST(ChecksumOptionsTest, SetAndGet) {
Options options;
EXPECT_FALSE(options.has<UploadChecksumValidationOption>());
EXPECT_FALSE(options.has<DownloadChecksumValidationOption>());

options.set<UploadChecksumValidationOption>(ChecksumAlgorithm::kMD5);
options.set<DownloadChecksumValidationOption>(ChecksumAlgorithm::kCrc32c);

EXPECT_TRUE(options.has<UploadChecksumValidationOption>());
EXPECT_EQ(ChecksumAlgorithm::kMD5,
options.get<UploadChecksumValidationOption>());

EXPECT_TRUE(options.has<DownloadChecksumValidationOption>());
EXPECT_EQ(ChecksumAlgorithm::kCrc32c,
options.get<DownloadChecksumValidationOption>());
}

} // namespace
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
} // namespace storage
Expand Down
2 changes: 2 additions & 0 deletions google/cloud/storage/internal/hash_function.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_HASH_FUNCTION_H
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_HASH_FUNCTION_H

#include "google/cloud/internal/disable_deprecation_warnings.inc"
#include "google/cloud/storage/hashing_options.h"
#include "google/cloud/storage/internal/hash_values.h"
#include "google/cloud/storage/version.h"
Expand Down Expand Up @@ -108,4 +109,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
} // namespace cloud
} // namespace google

#include "google/cloud/internal/diagnostics_pop.inc"
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_HASH_FUNCTION_H
2 changes: 2 additions & 0 deletions google/cloud/storage/internal/object_requests.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_OBJECT_REQUESTS_H
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_OBJECT_REQUESTS_H

#include "google/cloud/internal/disable_deprecation_warnings.inc"
#include "google/cloud/storage/auto_finalize.h"
#include "google/cloud/storage/download_options.h"
#include "google/cloud/storage/hashing_options.h"
Expand Down Expand Up @@ -645,4 +646,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
} // namespace cloud
} // namespace google

#include "google/cloud/internal/diagnostics_pop.inc"
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_OBJECT_REQUESTS_H
38 changes: 38 additions & 0 deletions google/cloud/storage/options.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,43 @@ struct CAPathOption {

} // namespace internal

/**
* Supported checksum algorithms.
*
* @ingroup storage-options
*/
enum class ChecksumAlgorithm {

@kalragauri kalragauri Jul 16, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the old boolean system, a user could theoretically have passed DisableCrc32cChecksum(false) and DisableMD5Hash(false) to force the client to compute and validate both hashes. If that use case is valid, we may need a kBoth enum member.

@v-pratap v-pratap Jul 16, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I have discussed this with Rajiv, and for now we don't need the both verification, but later if at any stage we need this then we can add this feature. @rajeevpodar PTAL.

kNone, ///< Disable checksum validation
kCrc32c, ///< Use CRC32C for checksum validation
kMD5, ///< Use MD5 for checksum validation
};

/**
* Configure the checksum algorithm used for uploads.
*
* If set, the client computes (if necessary) and validates the checksum of
* an object during uploads. Set to `ChecksumAlgorithm::kNone` to disable
* checksum validation.
*
* @ingroup storage-options
*/
struct UploadChecksumValidationOption {
using Type = ChecksumAlgorithm;
};

/**
* Configure the checksum algorithm used for downloads.
*
* If set, the client computes (if necessary) and validates the checksum of
* an object during downloads. Set to `ChecksumAlgorithm::kNone` to disable
* checksum validation.
*
* @ingroup storage-options
*/
struct DownloadChecksumValidationOption {
using Type = ChecksumAlgorithm;
};

/**
* Configure the REST endpoint for the GCS client library.
*
Expand Down Expand Up @@ -325,6 +362,7 @@ using ClientOptionList = ::google::cloud::OptionList<
MaximumCurlSocketRecvSizeOption, MaximumCurlSocketSendSizeOption,
TransferStallTimeoutOption, RetryPolicyOption, BackoffPolicyOption,
IdempotencyPolicyOption, CARootsFilePathOption,
UploadChecksumValidationOption, DownloadChecksumValidationOption,
storage_experimental::HttpVersionOption>;

GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
Expand Down
2 changes: 2 additions & 0 deletions google/cloud/storage/parallel_upload.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_PARALLEL_UPLOAD_H
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_PARALLEL_UPLOAD_H

#include "google/cloud/internal/disable_deprecation_warnings.inc"
#include "google/cloud/storage/client.h"
#include "google/cloud/storage/internal/tuple_filter.h"
#include "google/cloud/storage/object_stream.h"
Expand Down Expand Up @@ -1217,4 +1218,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
} // namespace cloud
} // namespace google

#include "google/cloud/internal/diagnostics_pop.inc"
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_PARALLEL_UPLOAD_H
Loading