Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 14 additions & 23 deletions google/cloud/storage/client.cc
Original file line number Diff line number Diff line change
Expand Up @@ -612,11 +612,6 @@ Options ApplyPolicy(Options opts, IdempotencyPolicy const& p) {
return opts;
}

Options DefaultOptions(std::shared_ptr<oauth2_internal::Credentials> const&,
Options opts) {
return DefaultOptions(std::move(opts));
}

Options DefaultOptions(Options opts) {
auto ud = GetEnv("GOOGLE_CLOUD_UNIVERSE_DOMAIN");
if (ud && !ud->empty()) {
Expand All @@ -628,9 +623,6 @@ Options DefaultOptions(Options opts) {
"https://iamcredentials.googleapis.com", opts),
"/v1");
Options o;
if (!opts.has<UnifiedCredentialsOption>()) {
o.set<UnifiedCredentialsOption>(MakeGoogleDefaultCredentials());
}
// Storage has more stringent requirements w.r.t. self-signed JWTs
// than most services. Any scope makes the self-signed JWTs unusable with
// storage, but they remain usable with other services. We need to disable
Expand Down Expand Up @@ -771,21 +763,20 @@ Options DefaultOptions(Options opts) {
}

Options DefaultOptionsWithCredentials(Options opts) {
if (opts.has<UnifiedCredentialsOption>()) {
auto credentials =
rest_internal::MapCredentials(*opts.get<UnifiedCredentialsOption>());
return internal::DefaultOptions(std::move(credentials), std::move(opts));
}
if (GetEmulator().has_value()) {
return internal::DefaultOptions(
rest_internal::MapCredentials(
*google::cloud::MakeInsecureCredentials()),
std::move(opts));
}
auto credentials = rest_internal::MapCredentials(
*google::cloud::MakeGoogleDefaultCredentials(
google::cloud::internal::MakeAuthOptions(opts)));
return internal::DefaultOptions(std::move(credentials), std::move(opts));
// Capture the auth options before `DefaultOptions()` adds the components in
// `CLOUD_STORAGE_ENABLE_TRACING`. Those configure the storage client only:
// `http` would log the OAuth token requests, and any value would replace the
// `GOOGLE_CLOUD_CPP_ENABLE_TRACING` components used by the credentials.
auto auth_options = google::cloud::internal::MakeAuthOptions(opts);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Calling MakeAuthOptions(opts) before DefaultOptions() prevents GOOGLE_CLOUD_CPP_OPENTELEMETRY_TRACING from reaching the default credentials. Unlike LoggingComponentsOption, PopulateAuthOptions() does not read GOOGLE_CLOUD_CPP_OPENTELEMETRY_TRACING, so OAuth token requests lose OpenTelemetry spans unless OpenTelemetryTracingOption is set in code.

Consider copying OpenTelemetryTracingOption from opts into auth_options after DefaultOptions() runs in both DefaultOptionsWithCredentials() and DefaultOptionsGrpc(), with a unit test covering that env var.

opts = internal::DefaultOptions(std::move(opts));
if (opts.has<UnifiedCredentialsOption>()) return opts;
// The emulator does not support authentication, use insecure (sometimes
// called "anonymous") credentials.
opts.set<UnifiedCredentialsOption>(
GetEmulator().has_value() ? google::cloud::MakeInsecureCredentials()
: google::cloud::MakeGoogleDefaultCredentials(
std::move(auth_options)));
return opts;
}

Client ClientImplDetails::CreateWithDecorations(
Expand Down
3 changes: 0 additions & 3 deletions google/cloud/storage/client.h
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,6 @@ Options ApplyPolicies(Options opts, P&& head, Policies&&... tail) {
return ApplyPolicies(std::move(opts), std::forward<Policies>(tail)...);
}

Options DefaultOptions(
std::shared_ptr<oauth2_internal::Credentials> const& credentials,
Options opts);
Options DefaultOptions(Options opts = {});
Options DefaultOptionsWithCredentials(Options opts);

Expand Down
88 changes: 88 additions & 0 deletions google/cloud/storage/client_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@
#include "google/cloud/storage/retry_policy.h"
#include "google/cloud/storage/testing/canonical_errors.h"
#include "google/cloud/storage/testing/mock_client.h"
#include "google/cloud/common_options.h"
#include "google/cloud/credentials.h"
#include "google/cloud/internal/curl_options.h"
#include "google/cloud/internal/filesystem.h"
#include "google/cloud/internal/getenv.h"
#include "google/cloud/internal/random.h"
#include "google/cloud/internal/rest_options.h"
#include "google/cloud/internal/rest_response.h"
#include "google/cloud/opentelemetry_options.h"
#include "google/cloud/testing_util/credentials.h"
#include "google/cloud/testing_util/mock_backoff_policy.h"
#include "google/cloud/testing_util/opentelemetry_matchers.h"
#include "google/cloud/testing_util/scoped_environment.h"
Expand All @@ -34,6 +37,7 @@
#include <cstdlib>
#include <fstream>
#include <memory>
#include <set>
#include <string>
#include <utility>
#include <vector>
Expand All @@ -44,9 +48,11 @@ namespace storage {
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
namespace {

using ::google::cloud::internal::CredentialsVisitor;
using ::google::cloud::storage::internal::ClientImplDetails;
using ::google::cloud::storage::testing::canonical_errors::TransientError;
using ::google::cloud::testing_util::ScopedEnvironment;
using ::google::cloud::testing_util::TestCredentialsVisitor;
using ::testing::ElementsAre;
using ::testing::NotNull;
using ::testing::Return;
Expand Down Expand Up @@ -88,6 +94,26 @@ class ObservableBackoffPolicy : public ExponentialBackoffPolicy {
};
int ObservableBackoffPolicy::on_completion_call_count_;

/// Returns the name of the `UnifiedCredentialsOption` in @p options, or an
/// empty string if the option is not set.
std::string CredentialsName(Options const& options) {
if (!options.has<UnifiedCredentialsOption>()) return {};
TestCredentialsVisitor visitor;
CredentialsVisitor::dispatch(*options.get<UnifiedCredentialsOption>(),
visitor);
return visitor.name;
}

/// Returns the logging components of the `UnifiedCredentialsOption` in
/// @p options, or an empty set if the option is not set.
std::set<std::string> CredentialsLoggingComponents(Options const& options) {
if (!options.has<UnifiedCredentialsOption>()) return {};
TestCredentialsVisitor visitor;
CredentialsVisitor::dispatch(*options.get<UnifiedCredentialsOption>(),
visitor);
return visitor.options.get<LoggingComponentsOption>();
}

class ClientTest : public ::testing::Test {
public:
ClientTest()
Expand Down Expand Up @@ -285,6 +311,68 @@ TEST_F(ClientTest, EndpointsEmulator) {
EXPECT_EQ("http://localhost:1234/iamapi", options.get<IamEndpointOption>());
}

TEST_F(ClientTest, EmulatorUsesInsecureCredentials) {
testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_EMULATOR_ENDPOINT",
"http://localhost:1234");
Options const options = internal::DefaultOptionsWithCredentials(Options{});
EXPECT_EQ(CredentialsName(options), "InsecureCredentialsConfig");
}

TEST_F(ClientTest, OldEmulatorUsesInsecureCredentials) {
google::cloud::testing_util::UnsetEnv("CLOUD_STORAGE_EMULATOR_ENDPOINT");
testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_TESTBENCH_ENDPOINT",
"http://localhost:1234");
Options const options = internal::DefaultOptionsWithCredentials(Options{});
EXPECT_EQ(CredentialsName(options), "InsecureCredentialsConfig");
}

TEST_F(ClientTest, EmulatorKeepsExplicitCredentials) {
testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_EMULATOR_ENDPOINT",
"http://localhost:1234");
Options const options = internal::DefaultOptionsWithCredentials(
Options{}.set<UnifiedCredentialsOption>(MakeAccessTokenCredentials(
"token1", std::chrono::system_clock::now() + std::chrono::hours(1))));
EXPECT_EQ(CredentialsName(options), "AccessTokenConfig");
}

TEST_F(ClientTest, DefaultCredentialsKeepAuthOptions) {
Options const options = internal::DefaultOptionsWithCredentials(
Options{}.set<LoggingComponentsOption>({"auth"}));
EXPECT_EQ(CredentialsName(options), "GoogleDefaultCredentialsConfig");
EXPECT_THAT(CredentialsLoggingComponents(options), ElementsAre("auth"));
}

/// @test Verify `CLOUD_STORAGE_ENABLE_TRACING` configures the client logging
/// only. The default credentials use `GOOGLE_CLOUD_CPP_ENABLE_TRACING`, and
/// must not log the OAuth token requests when `http` tracing is enabled.
TEST_F(ClientTest, DefaultCredentialsIgnoreStorageTracingEnvironment) {
testing_util::ScopedEnvironment storage_tracing(
"CLOUD_STORAGE_ENABLE_TRACING", "raw-client,http");
testing_util::ScopedEnvironment common_tracing(
"GOOGLE_CLOUD_CPP_ENABLE_TRACING", "auth");
Options const options = internal::DefaultOptionsWithCredentials(Options{});
EXPECT_THAT(options.get<LoggingComponentsOption>(),
UnorderedElementsAre("raw-client", "http"));
EXPECT_EQ(CredentialsName(options), "GoogleDefaultCredentialsConfig");
EXPECT_THAT(CredentialsLoggingComponents(options), ElementsAre("auth"));
}

TEST_F(ClientTest, DefaultOptionsHasNoCredentials) {
Options const options = internal::DefaultOptions();
EXPECT_FALSE(options.has<UnifiedCredentialsOption>());
}

TEST_F(ClientTest, ClientConstructorUsesEmulatorCredentials) {
testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_EMULATOR_ENDPOINT",
"http://localhost:1234");
Client client(DisableTracing(Options{}));
std::shared_ptr<internal::StorageConnection> const connection =
ClientImplDetails::GetConnection(client);
ASSERT_THAT(connection, NotNull());
EXPECT_EQ(CredentialsName(connection->options()),
"InsecureCredentialsConfig");
}

TEST_F(ClientTest, OldEndpointsEmulator) {
google::cloud::testing_util::UnsetEnv("CLOUD_STORAGE_EMULATOR_ENDPOINT");
testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_TESTBENCH_ENDPOINT",
Expand Down
7 changes: 6 additions & 1 deletion google/cloud/storage/doc/environment-variables.dox
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@ be even simpler:
@section storage-env-endpoint Endpoint Overrides

`CLOUD_STORAGE_EMULATOR_ENDPOINT=...`: override the default endpoint used by
the library. Intended for testing only.
the library. Intended for testing only. Unless the application configures
`google::cloud::UnifiedCredentialsOption`, a `google::cloud::storage::Client`
created with this variable set uses insecure (anonymous) credentials, as the
emulator does not support authentication. This variable does not change the
gRPC endpoint or credentials used by `google::cloud::storage::MakeGrpcClient()`
or `google::cloud::storage::AsyncClient`.

`CLOUD_STORAGE_TESTBENCH_ENDPOINT=...`: **DEPRECATED** please use
`CLOUD_STORAGE_EMULATOR_ENDPOINT` instead.
Expand Down
15 changes: 9 additions & 6 deletions google/cloud/storage/internal/grpc/default_options.cc
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,16 @@ Options DefaultOptionsGrpc(
options = google::cloud::internal::MergeOptions(
std::move(options), Options{}.set<storage::UploadBufferSizeOption>(
kDefaultGrpcUploadBufferSize));
options =
storage::internal::DefaultOptionsWithCredentials(std::move(options));
if (!options.has<UnifiedCredentialsOption>() &&
!options.has<GrpcCredentialOption>()) {
// `DefaultOptions()` sets no credentials. Use Application Default
// Credentials unless `UnifiedCredentialsOption` is set, so the REST emulator
// endpoint does not change the gRPC credentials. As in
// `DefaultOptionsWithCredentials()`, capture the auth options before
// `DefaultOptions()` adds the `CLOUD_STORAGE_ENABLE_TRACING` components.
auto auth_options = google::cloud::internal::MakeAuthOptions(options);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per the repository style guide, auto auth_options hides the function return type Options and lacks an explicit qualifier.

options = storage::internal::DefaultOptions(std::move(options));
if (!options.has<UnifiedCredentialsOption>()) {
options.set<UnifiedCredentialsOption>(
google::cloud::MakeGoogleDefaultCredentials(
google::cloud::internal::MakeAuthOptions(options)));
google::cloud::MakeGoogleDefaultCredentials(std::move(auth_options)));
}

auto const preserve_creds =
Expand Down
93 changes: 93 additions & 0 deletions google/cloud/storage/internal/grpc/default_options_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,21 @@
#include "google/cloud/storage/internal/grpc/default_options.h"
#include "google/cloud/storage/grpc_plugin.h"
#include "google/cloud/storage/options.h"
#include "google/cloud/common_options.h"
#include "google/cloud/credentials.h"
#include "google/cloud/grpc_options.h"
#include "google/cloud/internal/detect_gcp.h"
#include "google/cloud/internal/unified_grpc_credentials.h"
#include "google/cloud/testing_util/chrono_output.h"
#include "google/cloud/testing_util/credentials.h"
#include "google/cloud/testing_util/scoped_environment.h"
#include "google/cloud/universe_domain_options.h"
#include "absl/strings/str_cat.h"
#include <gmock/gmock.h>
#include <grpcpp/grpcpp.h>
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <tuple>
#include <utility>
Expand All @@ -45,12 +48,43 @@ class MockGcpDetector : public google::cloud::internal::GcpDetector {

using ::google::cloud::internal::GrpcGoogleDefaultTlsOnlyOption;
using ::google::cloud::testing_util::ScopedEnvironment;
using ::testing::ElementsAre;
using ::testing::Return;
using ::testing::UnorderedElementsAre;

google::cloud::Options TestOptions() {
return Options{}.set<UnifiedCredentialsOption>(MakeInsecureCredentials());
}

std::shared_ptr<google::cloud::internal::GcpDetector> MakeNotOnGcpDetector() {
auto mock_detector = std::make_shared<MockGcpDetector>();
EXPECT_CALL(*mock_detector, IsGoogleCloudBios())
.WillRepeatedly(Return(false));
EXPECT_CALL(*mock_detector, IsGoogleCloudServerless())
.WillRepeatedly(Return(false));
return mock_detector;
}

/// Returns the name of the `UnifiedCredentialsOption` in @p options, or an
/// empty string if the option is not set.
std::string CredentialsName(Options const& options) {
if (!options.has<UnifiedCredentialsOption>()) return {};
::google::cloud::testing_util::TestCredentialsVisitor visitor;
::google::cloud::internal::CredentialsVisitor::dispatch(
*options.get<UnifiedCredentialsOption>(), visitor);
return visitor.name;
}

/// Returns the logging components of the `UnifiedCredentialsOption` in
/// @p options, or an empty set if the option is not set.
std::set<std::string> CredentialsLoggingComponents(Options const& options) {
if (!options.has<UnifiedCredentialsOption>()) return {};
::google::cloud::testing_util::TestCredentialsVisitor visitor;
::google::cloud::internal::CredentialsVisitor::dispatch(
*options.get<UnifiedCredentialsOption>(), visitor);
return visitor.options.get<LoggingComponentsOption>();
}

TEST(DefaultOptionsGrpc, DefaultOptionsGrpcChannelCount) {
using ::google::cloud::GrpcNumChannelsOption;
struct TestCase {
Expand Down Expand Up @@ -79,6 +113,65 @@ TEST(DefaultOptionsGrpc, DefaultOptionsGrpcChannelCount) {
}
}

/// @test Verify the REST emulator endpoints do not change gRPC credentials.
TEST(DefaultOptionsGrpc, RestEmulatorDoesNotChangeGrpcCredentials) {
for (char const* var : {"CLOUD_STORAGE_EMULATOR_ENDPOINT",
"CLOUD_STORAGE_TESTBENCH_ENDPOINT"}) {
SCOPED_TRACE(var);
ScopedEnvironment emulator("CLOUD_STORAGE_EMULATOR_ENDPOINT", {});
ScopedEnvironment old_emulator("CLOUD_STORAGE_TESTBENCH_ENDPOINT", {});
ScopedEnvironment testbench(
"CLOUD_STORAGE_EXPERIMENTAL_GRPC_TESTBENCH_ENDPOINT", {});
ScopedEnvironment rest(var, "http://localhost:1234");
Options const options =
DefaultOptionsGrpc(Options{}, MakeNotOnGcpDetector());
EXPECT_EQ(CredentialsName(options), "GoogleDefaultCredentialsConfig");
}
}

/// @test Verify the gRPC testbench still uses insecure credentials when the
/// REST emulator endpoint is also set.
TEST(DefaultOptionsGrpc, GrpcTestbenchStillInsecureWithRestEmulator) {
ScopedEnvironment emulator("CLOUD_STORAGE_EMULATOR_ENDPOINT",
"http://localhost:1234");
ScopedEnvironment testbench(
"CLOUD_STORAGE_EXPERIMENTAL_GRPC_TESTBENCH_ENDPOINT", "localhost:9999");
ScopedEnvironment preserve(
"GOOGLE_CLOUD_CPP_STORAGE_TESTING_PRESERVE_CREDENTIALS", {});
Options const options = DefaultOptionsGrpc(Options{}, MakeNotOnGcpDetector());
EXPECT_EQ(CredentialsName(options), "InsecureCredentialsConfig");
}

/// @test Verify explicit credentials are kept when the REST emulator endpoint
/// is set.
TEST(DefaultOptionsGrpc, GrpcKeepsExplicitCredentialsWithRestEmulator) {
ScopedEnvironment emulator("CLOUD_STORAGE_EMULATOR_ENDPOINT",
"http://localhost:1234");
ScopedEnvironment testbench(
"CLOUD_STORAGE_EXPERIMENTAL_GRPC_TESTBENCH_ENDPOINT", {});
Options const options = DefaultOptionsGrpc(
Options{}.set<UnifiedCredentialsOption>(MakeAccessTokenCredentials(
"token1", std::chrono::system_clock::now() + std::chrono::hours(1))),
MakeNotOnGcpDetector());
EXPECT_EQ(CredentialsName(options), "AccessTokenConfig");
}

/// @test Verify `CLOUD_STORAGE_ENABLE_TRACING` configures the client logging
/// only. The default credentials use `GOOGLE_CLOUD_CPP_ENABLE_TRACING`.
TEST(DefaultOptionsGrpc, DefaultCredentialsIgnoreStorageTracingEnvironment) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DefaultCredentialsIgnoreStorageTracingEnvironment passes empty Options{} with GOOGLE_CLOUD_CPP_ENABLE_TRACING=auth, which PopulateAuthOptions() already applies even when MakeGoogleDefaultCredentials() receives empty options. As a result, no test in this file fails if DefaultOptionsGrpc() drops auth_options on line 95.

ScopedEnvironment emulator("CLOUD_STORAGE_EMULATOR_ENDPOINT", {});
ScopedEnvironment testbench(
"CLOUD_STORAGE_EXPERIMENTAL_GRPC_TESTBENCH_ENDPOINT", {});
ScopedEnvironment storage_tracing("CLOUD_STORAGE_ENABLE_TRACING",
"raw-client,http");
ScopedEnvironment common_tracing("GOOGLE_CLOUD_CPP_ENABLE_TRACING", "auth");
Options const options = DefaultOptionsGrpc(Options{}, MakeNotOnGcpDetector());
EXPECT_THAT(options.get<LoggingComponentsOption>(),
UnorderedElementsAre("raw-client", "http"));
EXPECT_EQ(CredentialsName(options), "GoogleDefaultCredentialsConfig");
EXPECT_THAT(CredentialsLoggingComponents(options), ElementsAre("auth"));
}

TEST(DefaultOptionsGrpc, DefaultEndpointsCloudPath) {
auto mock_detector = std::make_shared<MockGcpDetector>();
EXPECT_CALL(*mock_detector, IsGoogleCloudBios())
Expand Down
Loading