diff --git a/google/cloud/storage/client.cc b/google/cloud/storage/client.cc index adf5afe483769..919b8ccc9fd7c 100644 --- a/google/cloud/storage/client.cc +++ b/google/cloud/storage/client.cc @@ -612,11 +612,6 @@ Options ApplyPolicy(Options opts, IdempotencyPolicy const& p) { return opts; } -Options DefaultOptions(std::shared_ptr const&, - Options opts) { - return DefaultOptions(std::move(opts)); -} - Options DefaultOptions(Options opts) { auto ud = GetEnv("GOOGLE_CLOUD_UNIVERSE_DOMAIN"); if (ud && !ud->empty()) { @@ -628,9 +623,6 @@ Options DefaultOptions(Options opts) { "https://iamcredentials.googleapis.com", opts), "/v1"); Options o; - if (!opts.has()) { - o.set(MakeGoogleDefaultCredentials()); - } // Storage has more stringent requirements w.r.t. self-signed JWTs // than most services. Any scope makes the self-signed JWTs unusable with // storage, but they remain usable with other services. We need to disable @@ -771,21 +763,20 @@ Options DefaultOptions(Options opts) { } Options DefaultOptionsWithCredentials(Options opts) { - if (opts.has()) { - auto credentials = - rest_internal::MapCredentials(*opts.get()); - return internal::DefaultOptions(std::move(credentials), std::move(opts)); - } - if (GetEmulator().has_value()) { - return internal::DefaultOptions( - rest_internal::MapCredentials( - *google::cloud::MakeInsecureCredentials()), - std::move(opts)); - } - auto credentials = rest_internal::MapCredentials( - *google::cloud::MakeGoogleDefaultCredentials( - google::cloud::internal::MakeAuthOptions(opts))); - return internal::DefaultOptions(std::move(credentials), std::move(opts)); + // Capture the auth options before `DefaultOptions()` adds the components in + // `CLOUD_STORAGE_ENABLE_TRACING`. Those configure the storage client only: + // `http` would log the OAuth token requests, and any value would replace the + // `GOOGLE_CLOUD_CPP_ENABLE_TRACING` components used by the credentials. + auto auth_options = google::cloud::internal::MakeAuthOptions(opts); + opts = internal::DefaultOptions(std::move(opts)); + if (opts.has()) return opts; + // The emulator does not support authentication, use insecure (sometimes + // called "anonymous") credentials. + opts.set( + GetEmulator().has_value() ? google::cloud::MakeInsecureCredentials() + : google::cloud::MakeGoogleDefaultCredentials( + std::move(auth_options))); + return opts; } Client ClientImplDetails::CreateWithDecorations( diff --git a/google/cloud/storage/client.h b/google/cloud/storage/client.h index f02e5c25ae79c..c9cfa60e1e9a6 100644 --- a/google/cloud/storage/client.h +++ b/google/cloud/storage/client.h @@ -66,9 +66,6 @@ Options ApplyPolicies(Options opts, P&& head, Policies&&... tail) { return ApplyPolicies(std::move(opts), std::forward(tail)...); } -Options DefaultOptions( - std::shared_ptr const& credentials, - Options opts); Options DefaultOptions(Options opts = {}); Options DefaultOptionsWithCredentials(Options opts); diff --git a/google/cloud/storage/client_test.cc b/google/cloud/storage/client_test.cc index 9321840b6d797..b477467efdcdd 100644 --- a/google/cloud/storage/client_test.cc +++ b/google/cloud/storage/client_test.cc @@ -16,6 +16,8 @@ #include "google/cloud/storage/retry_policy.h" #include "google/cloud/storage/testing/canonical_errors.h" #include "google/cloud/storage/testing/mock_client.h" +#include "google/cloud/common_options.h" +#include "google/cloud/credentials.h" #include "google/cloud/internal/curl_options.h" #include "google/cloud/internal/filesystem.h" #include "google/cloud/internal/getenv.h" @@ -23,6 +25,7 @@ #include "google/cloud/internal/rest_options.h" #include "google/cloud/internal/rest_response.h" #include "google/cloud/opentelemetry_options.h" +#include "google/cloud/testing_util/credentials.h" #include "google/cloud/testing_util/mock_backoff_policy.h" #include "google/cloud/testing_util/opentelemetry_matchers.h" #include "google/cloud/testing_util/scoped_environment.h" @@ -34,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -44,9 +48,11 @@ namespace storage { GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN namespace { +using ::google::cloud::internal::CredentialsVisitor; using ::google::cloud::storage::internal::ClientImplDetails; using ::google::cloud::storage::testing::canonical_errors::TransientError; using ::google::cloud::testing_util::ScopedEnvironment; +using ::google::cloud::testing_util::TestCredentialsVisitor; using ::testing::ElementsAre; using ::testing::NotNull; using ::testing::Return; @@ -88,6 +94,26 @@ class ObservableBackoffPolicy : public ExponentialBackoffPolicy { }; int ObservableBackoffPolicy::on_completion_call_count_; +/// Returns the name of the `UnifiedCredentialsOption` in @p options, or an +/// empty string if the option is not set. +std::string CredentialsName(Options const& options) { + if (!options.has()) return {}; + TestCredentialsVisitor visitor; + CredentialsVisitor::dispatch(*options.get(), + visitor); + return visitor.name; +} + +/// Returns the logging components of the `UnifiedCredentialsOption` in +/// @p options, or an empty set if the option is not set. +std::set CredentialsLoggingComponents(Options const& options) { + if (!options.has()) return {}; + TestCredentialsVisitor visitor; + CredentialsVisitor::dispatch(*options.get(), + visitor); + return visitor.options.get(); +} + class ClientTest : public ::testing::Test { public: ClientTest() @@ -285,6 +311,68 @@ TEST_F(ClientTest, EndpointsEmulator) { EXPECT_EQ("http://localhost:1234/iamapi", options.get()); } +TEST_F(ClientTest, EmulatorUsesInsecureCredentials) { + testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_EMULATOR_ENDPOINT", + "http://localhost:1234"); + Options const options = internal::DefaultOptionsWithCredentials(Options{}); + EXPECT_EQ(CredentialsName(options), "InsecureCredentialsConfig"); +} + +TEST_F(ClientTest, OldEmulatorUsesInsecureCredentials) { + google::cloud::testing_util::UnsetEnv("CLOUD_STORAGE_EMULATOR_ENDPOINT"); + testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_TESTBENCH_ENDPOINT", + "http://localhost:1234"); + Options const options = internal::DefaultOptionsWithCredentials(Options{}); + EXPECT_EQ(CredentialsName(options), "InsecureCredentialsConfig"); +} + +TEST_F(ClientTest, EmulatorKeepsExplicitCredentials) { + testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_EMULATOR_ENDPOINT", + "http://localhost:1234"); + Options const options = internal::DefaultOptionsWithCredentials( + Options{}.set(MakeAccessTokenCredentials( + "token1", std::chrono::system_clock::now() + std::chrono::hours(1)))); + EXPECT_EQ(CredentialsName(options), "AccessTokenConfig"); +} + +TEST_F(ClientTest, DefaultCredentialsKeepAuthOptions) { + Options const options = internal::DefaultOptionsWithCredentials( + Options{}.set({"auth"})); + EXPECT_EQ(CredentialsName(options), "GoogleDefaultCredentialsConfig"); + EXPECT_THAT(CredentialsLoggingComponents(options), ElementsAre("auth")); +} + +/// @test Verify `CLOUD_STORAGE_ENABLE_TRACING` configures the client logging +/// only. The default credentials use `GOOGLE_CLOUD_CPP_ENABLE_TRACING`, and +/// must not log the OAuth token requests when `http` tracing is enabled. +TEST_F(ClientTest, DefaultCredentialsIgnoreStorageTracingEnvironment) { + testing_util::ScopedEnvironment storage_tracing( + "CLOUD_STORAGE_ENABLE_TRACING", "raw-client,http"); + testing_util::ScopedEnvironment common_tracing( + "GOOGLE_CLOUD_CPP_ENABLE_TRACING", "auth"); + Options const options = internal::DefaultOptionsWithCredentials(Options{}); + EXPECT_THAT(options.get(), + UnorderedElementsAre("raw-client", "http")); + EXPECT_EQ(CredentialsName(options), "GoogleDefaultCredentialsConfig"); + EXPECT_THAT(CredentialsLoggingComponents(options), ElementsAre("auth")); +} + +TEST_F(ClientTest, DefaultOptionsHasNoCredentials) { + Options const options = internal::DefaultOptions(); + EXPECT_FALSE(options.has()); +} + +TEST_F(ClientTest, ClientConstructorUsesEmulatorCredentials) { + testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_EMULATOR_ENDPOINT", + "http://localhost:1234"); + Client client(DisableTracing(Options{})); + std::shared_ptr const connection = + ClientImplDetails::GetConnection(client); + ASSERT_THAT(connection, NotNull()); + EXPECT_EQ(CredentialsName(connection->options()), + "InsecureCredentialsConfig"); +} + TEST_F(ClientTest, OldEndpointsEmulator) { google::cloud::testing_util::UnsetEnv("CLOUD_STORAGE_EMULATOR_ENDPOINT"); testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_TESTBENCH_ENDPOINT", diff --git a/google/cloud/storage/doc/environment-variables.dox b/google/cloud/storage/doc/environment-variables.dox index c60f2d5fe9c7c..785a2fdc13337 100644 --- a/google/cloud/storage/doc/environment-variables.dox +++ b/google/cloud/storage/doc/environment-variables.dox @@ -22,7 +22,12 @@ be even simpler: @section storage-env-endpoint Endpoint Overrides `CLOUD_STORAGE_EMULATOR_ENDPOINT=...`: override the default endpoint used by -the library. Intended for testing only. +the library. Intended for testing only. Unless the application configures +`google::cloud::UnifiedCredentialsOption`, a `google::cloud::storage::Client` +created with this variable set uses insecure (anonymous) credentials, as the +emulator does not support authentication. This variable does not change the +gRPC endpoint or credentials used by `google::cloud::storage::MakeGrpcClient()` +or `google::cloud::storage::AsyncClient`. `CLOUD_STORAGE_TESTBENCH_ENDPOINT=...`: **DEPRECATED** please use `CLOUD_STORAGE_EMULATOR_ENDPOINT` instead. diff --git a/google/cloud/storage/internal/grpc/default_options.cc b/google/cloud/storage/internal/grpc/default_options.cc index 23a5e22287ac7..ffb4c353793b2 100644 --- a/google/cloud/storage/internal/grpc/default_options.cc +++ b/google/cloud/storage/internal/grpc/default_options.cc @@ -83,13 +83,16 @@ Options DefaultOptionsGrpc( options = google::cloud::internal::MergeOptions( std::move(options), Options{}.set( kDefaultGrpcUploadBufferSize)); - options = - storage::internal::DefaultOptionsWithCredentials(std::move(options)); - if (!options.has() && - !options.has()) { + // `DefaultOptions()` sets no credentials. Use Application Default + // Credentials unless `UnifiedCredentialsOption` is set, so the REST emulator + // endpoint does not change the gRPC credentials. As in + // `DefaultOptionsWithCredentials()`, capture the auth options before + // `DefaultOptions()` adds the `CLOUD_STORAGE_ENABLE_TRACING` components. + auto auth_options = google::cloud::internal::MakeAuthOptions(options); + options = storage::internal::DefaultOptions(std::move(options)); + if (!options.has()) { options.set( - google::cloud::MakeGoogleDefaultCredentials( - google::cloud::internal::MakeAuthOptions(options))); + google::cloud::MakeGoogleDefaultCredentials(std::move(auth_options))); } auto const preserve_creds = diff --git a/google/cloud/storage/internal/grpc/default_options_test.cc b/google/cloud/storage/internal/grpc/default_options_test.cc index 750abafee28b7..f0f813ad3bd41 100644 --- a/google/cloud/storage/internal/grpc/default_options_test.cc +++ b/google/cloud/storage/internal/grpc/default_options_test.cc @@ -15,11 +15,13 @@ #include "google/cloud/storage/internal/grpc/default_options.h" #include "google/cloud/storage/grpc_plugin.h" #include "google/cloud/storage/options.h" +#include "google/cloud/common_options.h" #include "google/cloud/credentials.h" #include "google/cloud/grpc_options.h" #include "google/cloud/internal/detect_gcp.h" #include "google/cloud/internal/unified_grpc_credentials.h" #include "google/cloud/testing_util/chrono_output.h" +#include "google/cloud/testing_util/credentials.h" #include "google/cloud/testing_util/scoped_environment.h" #include "google/cloud/universe_domain_options.h" #include "absl/strings/str_cat.h" @@ -27,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -45,12 +48,43 @@ class MockGcpDetector : public google::cloud::internal::GcpDetector { using ::google::cloud::internal::GrpcGoogleDefaultTlsOnlyOption; using ::google::cloud::testing_util::ScopedEnvironment; +using ::testing::ElementsAre; using ::testing::Return; +using ::testing::UnorderedElementsAre; google::cloud::Options TestOptions() { return Options{}.set(MakeInsecureCredentials()); } +std::shared_ptr MakeNotOnGcpDetector() { + auto mock_detector = std::make_shared(); + EXPECT_CALL(*mock_detector, IsGoogleCloudBios()) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*mock_detector, IsGoogleCloudServerless()) + .WillRepeatedly(Return(false)); + return mock_detector; +} + +/// Returns the name of the `UnifiedCredentialsOption` in @p options, or an +/// empty string if the option is not set. +std::string CredentialsName(Options const& options) { + if (!options.has()) return {}; + ::google::cloud::testing_util::TestCredentialsVisitor visitor; + ::google::cloud::internal::CredentialsVisitor::dispatch( + *options.get(), visitor); + return visitor.name; +} + +/// Returns the logging components of the `UnifiedCredentialsOption` in +/// @p options, or an empty set if the option is not set. +std::set CredentialsLoggingComponents(Options const& options) { + if (!options.has()) return {}; + ::google::cloud::testing_util::TestCredentialsVisitor visitor; + ::google::cloud::internal::CredentialsVisitor::dispatch( + *options.get(), visitor); + return visitor.options.get(); +} + TEST(DefaultOptionsGrpc, DefaultOptionsGrpcChannelCount) { using ::google::cloud::GrpcNumChannelsOption; struct TestCase { @@ -79,6 +113,65 @@ TEST(DefaultOptionsGrpc, DefaultOptionsGrpcChannelCount) { } } +/// @test Verify the REST emulator endpoints do not change gRPC credentials. +TEST(DefaultOptionsGrpc, RestEmulatorDoesNotChangeGrpcCredentials) { + for (char const* var : {"CLOUD_STORAGE_EMULATOR_ENDPOINT", + "CLOUD_STORAGE_TESTBENCH_ENDPOINT"}) { + SCOPED_TRACE(var); + ScopedEnvironment emulator("CLOUD_STORAGE_EMULATOR_ENDPOINT", {}); + ScopedEnvironment old_emulator("CLOUD_STORAGE_TESTBENCH_ENDPOINT", {}); + ScopedEnvironment testbench( + "CLOUD_STORAGE_EXPERIMENTAL_GRPC_TESTBENCH_ENDPOINT", {}); + ScopedEnvironment rest(var, "http://localhost:1234"); + Options const options = + DefaultOptionsGrpc(Options{}, MakeNotOnGcpDetector()); + EXPECT_EQ(CredentialsName(options), "GoogleDefaultCredentialsConfig"); + } +} + +/// @test Verify the gRPC testbench still uses insecure credentials when the +/// REST emulator endpoint is also set. +TEST(DefaultOptionsGrpc, GrpcTestbenchStillInsecureWithRestEmulator) { + ScopedEnvironment emulator("CLOUD_STORAGE_EMULATOR_ENDPOINT", + "http://localhost:1234"); + ScopedEnvironment testbench( + "CLOUD_STORAGE_EXPERIMENTAL_GRPC_TESTBENCH_ENDPOINT", "localhost:9999"); + ScopedEnvironment preserve( + "GOOGLE_CLOUD_CPP_STORAGE_TESTING_PRESERVE_CREDENTIALS", {}); + Options const options = DefaultOptionsGrpc(Options{}, MakeNotOnGcpDetector()); + EXPECT_EQ(CredentialsName(options), "InsecureCredentialsConfig"); +} + +/// @test Verify explicit credentials are kept when the REST emulator endpoint +/// is set. +TEST(DefaultOptionsGrpc, GrpcKeepsExplicitCredentialsWithRestEmulator) { + ScopedEnvironment emulator("CLOUD_STORAGE_EMULATOR_ENDPOINT", + "http://localhost:1234"); + ScopedEnvironment testbench( + "CLOUD_STORAGE_EXPERIMENTAL_GRPC_TESTBENCH_ENDPOINT", {}); + Options const options = DefaultOptionsGrpc( + Options{}.set(MakeAccessTokenCredentials( + "token1", std::chrono::system_clock::now() + std::chrono::hours(1))), + MakeNotOnGcpDetector()); + EXPECT_EQ(CredentialsName(options), "AccessTokenConfig"); +} + +/// @test Verify `CLOUD_STORAGE_ENABLE_TRACING` configures the client logging +/// only. The default credentials use `GOOGLE_CLOUD_CPP_ENABLE_TRACING`. +TEST(DefaultOptionsGrpc, DefaultCredentialsIgnoreStorageTracingEnvironment) { + ScopedEnvironment emulator("CLOUD_STORAGE_EMULATOR_ENDPOINT", {}); + ScopedEnvironment testbench( + "CLOUD_STORAGE_EXPERIMENTAL_GRPC_TESTBENCH_ENDPOINT", {}); + ScopedEnvironment storage_tracing("CLOUD_STORAGE_ENABLE_TRACING", + "raw-client,http"); + ScopedEnvironment common_tracing("GOOGLE_CLOUD_CPP_ENABLE_TRACING", "auth"); + Options const options = DefaultOptionsGrpc(Options{}, MakeNotOnGcpDetector()); + EXPECT_THAT(options.get(), + UnorderedElementsAre("raw-client", "http")); + EXPECT_EQ(CredentialsName(options), "GoogleDefaultCredentialsConfig"); + EXPECT_THAT(CredentialsLoggingComponents(options), ElementsAre("auth")); +} + TEST(DefaultOptionsGrpc, DefaultEndpointsCloudPath) { auto mock_detector = std::make_shared(); EXPECT_CALL(*mock_detector, IsGoogleCloudBios())