From f4cbede6de559c24e0c40de113076aef91734662 Mon Sep 17 00:00:00 2001 From: Nidhi Nandwani Date: Wed, 19 Aug 2026 05:57:30 +0000 Subject: [PATCH 1/2] feat(storage): Parameterized test setup for RCU Integration Testing --- .../cloud/storage/spi/v1/HttpStorageRpc.java | 23 +- .../it/runner/CrossRunIntersection.java | 160 +++++++++-- .../storage/it/runner/StorageITRunner.java | 62 +++-- .../it/runner/annotations/Backend.java | 2 + .../it/runner/annotations/Colocation.java | 22 ++ .../it/runner/annotations/CrossRun.java | 12 + .../it/runner/annotations/LocationType.java | 23 ++ .../it/runner/annotations/SingleBackend.java | 4 + .../it/runner/registry/BackendResources.java | 248 ++++++++++++++++-- .../it/runner/registry/BucketInfoShim.java | 42 +++ .../storage/it/runner/registry/Registry.java | 5 +- 11 files changed, 539 insertions(+), 64 deletions(-) create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/LocationType.java diff --git a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index ec7d8e02e411..0ee0dcfe4e10 100644 --- a/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -158,11 +158,26 @@ public HttpStorageRpc(StorageOptions options, JsonFactory jsonFactory) { initializer = censusHttpModule.getHttpRequestInitializer(initializer); initializer = new InvocationIdInitializer(initializer, applicationName, tm); batchRequestInitializer = censusHttpModule.getHttpRequestInitializer(null); - storage = + String host = options.getHost(); + Storage.Builder storageBuilder = new Storage.Builder(transport, jsonFactory, initializer) - .setRootUrl(options.getHost()) - .setApplicationName(applicationName) - .build(); + .setApplicationName(applicationName); + if (host != null) { + java.net.URI uri = java.net.URI.create(host); + String path = uri.getPath(); + if (path != null && !path.isEmpty() && !"/".equals(path)) { + String rootUrl = host.substring(0, host.indexOf(path)); + String servicePath = path.startsWith("/") ? path.substring(1) : path; + if (!servicePath.endsWith("/")) { + servicePath += "/"; + } + storageBuilder.setRootUrl(rootUrl); + storageBuilder.setServicePath(servicePath); + } else { + storageBuilder.setRootUrl(host); + } + } + storage = storageBuilder.build(); } public Storage getStorage() { diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java index a48ba9d3ce5f..8aa30c50290a 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java @@ -20,10 +20,14 @@ import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.Colocation; import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; +import java.util.Collections; import java.util.Locale; +import java.util.Set; import java.util.Objects; import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.ThreadSafe; @@ -39,10 +43,18 @@ public final class CrossRunIntersection { private final @Nullable Backend backend; private final @Nullable Transport transport; + private final @Nullable LocationType locationType; + private final @Nullable Colocation colocation; - private CrossRunIntersection(@Nullable Backend backend, @Nullable Transport transport) { + private CrossRunIntersection( + @Nullable Backend backend, + @Nullable Transport transport, + @Nullable LocationType locationType, + @Nullable Colocation colocation) { this.backend = backend; this.transport = transport; + this.locationType = locationType; + this.colocation = colocation; } @Nullable @@ -55,11 +67,21 @@ public Transport getTransport() { return transport; } + @Nullable + public LocationType getLocationType() { + return locationType; + } + + @Nullable + public Colocation getColocation() { + return colocation; + } + public CrossRunIntersection clearBackend() { if (backend == null) { return this; } else { - return new CrossRunIntersection(null, transport); + return new CrossRunIntersection(null, transport, locationType, colocation); } } @@ -67,7 +89,23 @@ public CrossRunIntersection clearTransport() { if (transport == null) { return this; } else { - return new CrossRunIntersection(backend, null); + return new CrossRunIntersection(backend, null, locationType, colocation); + } + } + + public CrossRunIntersection clearLocationType() { + if (locationType == null) { + return this; + } else { + return new CrossRunIntersection(backend, transport, null, colocation); + } + } + + public CrossRunIntersection clearColocation() { + if (colocation == null) { + return this; + } else { + return new CrossRunIntersection(backend, transport, locationType, null); } } @@ -76,7 +114,7 @@ public CrossRunIntersection withBackend(Backend backend) { if (this.backend == backend) { return this; } else { - return new CrossRunIntersection(backend, transport); + return new CrossRunIntersection(backend, transport, locationType, colocation); } } @@ -85,7 +123,25 @@ public CrossRunIntersection withTransport(Transport transport) { if (this.transport == transport) { return this; } else { - return new CrossRunIntersection(backend, transport); + return new CrossRunIntersection(backend, transport, locationType, colocation); + } + } + + public CrossRunIntersection withLocationType(LocationType locationType) { + requireNonNull(locationType, "locationType must be non null"); + if (this.locationType == locationType) { + return this; + } else { + return new CrossRunIntersection(backend, transport, locationType, colocation); + } + } + + public CrossRunIntersection withColocation(Colocation colocation) { + requireNonNull(colocation, "colocation must be non null"); + if (this.colocation == colocation) { + return this; + } else { + return new CrossRunIntersection(backend, transport, locationType, colocation); } } @@ -107,6 +163,20 @@ public boolean anyMatch(CrossRunIntersection other) { l = l.clearTransport(); } + if (l.locationType == null) { + r = r.clearLocationType(); + } + if (r.locationType == null) { + l = l.clearLocationType(); + } + + if (l.colocation == null) { + r = r.clearColocation(); + } + if (r.colocation == null) { + l = l.clearColocation(); + } + return l.equals(r); } @@ -119,7 +189,9 @@ public boolean anyMatch(CrossRunIntersection other) { public String fmtSuiteName() { String t = transport != null ? transport.toString() : "NULL_TRANSPORT"; String b = backend != null ? backend.toString() : "NULL_BACKEND"; - return String.format(Locale.US, "[%s][%s]", t, b); + String lt = locationType != null ? locationType.toString() : "NULL_LOCATION"; + String c = colocation != null ? colocation.toString() : "NULL_COLOCATION"; + return String.format(Locale.US, "[%s][%s][%s][%s]", t, b, lt, c); } @Override @@ -131,12 +203,15 @@ public boolean equals(Object o) { return false; } CrossRunIntersection crossRunIntersection = (CrossRunIntersection) o; - return backend == crossRunIntersection.backend && transport == crossRunIntersection.transport; + return backend == crossRunIntersection.backend + && transport == crossRunIntersection.transport + && locationType == crossRunIntersection.locationType + && colocation == crossRunIntersection.colocation; } @Override public int hashCode() { - return Objects.hash(backend, transport); + return Objects.hash(backend, transport, locationType, colocation); } @Override @@ -144,41 +219,74 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("backend", backend) .add("transport", transport) + .add("locationType", locationType) + .add("colocation", colocation) .toString(); } - public static CrossRunIntersection of(@Nullable Backend t, @Nullable Transport s) { - return new CrossRunIntersection(t, s); + public static CrossRunIntersection of(@Nullable Backend b, @Nullable Transport t) { + return new CrossRunIntersection(b, t, null, null); + } + + public static CrossRunIntersection of( + @Nullable Backend b, + @Nullable Transport t, + @Nullable LocationType lt, + @Nullable Colocation c) { + return new CrossRunIntersection(b, t, lt, c); } public static ImmutableSet expand(CrossRun.Ignore i) { ImmutableSet backends = ImmutableSet.copyOf(i.backends()); ImmutableSet transports = ImmutableSet.copyOf(i.transports()); - return expand(backends, transports); + ImmutableSet locations = ImmutableSet.copyOf(i.locations()); + ImmutableSet colocations = ImmutableSet.copyOf(i.colocations()); + return expand(backends, transports, locations, colocations); } public static ImmutableSet expand(CrossRun.Exclude i) { ImmutableSet backends = ImmutableSet.copyOf(i.backends()); ImmutableSet transports = ImmutableSet.copyOf(i.transports()); - return expand(backends, transports); + ImmutableSet locations = ImmutableSet.copyOf(i.locations()); + ImmutableSet colocations = ImmutableSet.copyOf(i.colocations()); + return expand(backends, transports, locations, colocations); } public static ImmutableSet expand( - ImmutableSet backends, ImmutableSet<@Nullable Transport> transports) { - if (backends.isEmpty() && transports.isEmpty()) { + ImmutableSet backends, + ImmutableSet<@Nullable Transport> transports, + ImmutableSet<@Nullable LocationType> locations, + ImmutableSet<@Nullable Colocation> colocations) { + if (backends.isEmpty() && transports.isEmpty() && locations.isEmpty() && colocations.isEmpty()) { return ImmutableSet.of(); - } else if (!backends.isEmpty() && !transports.isEmpty()) { - return backends.stream() - .flatMap(t -> transports.stream().map(s -> new CrossRunIntersection(t, s))) - .collect(ImmutableSet.toImmutableSet()); - } else if (!backends.isEmpty()) { - return backends.stream() - .map(t -> new CrossRunIntersection(t, null)) - .collect(ImmutableSet.toImmutableSet()); - } else { - return transports.stream() - .map(s -> new CrossRunIntersection(null, s)) - .collect(ImmutableSet.toImmutableSet()); } + + Set<@Nullable Backend> bSet = + backends.isEmpty() ? Collections.singleton((Backend) null) : backends; + Set<@Nullable Transport> tSet = + transports.isEmpty() ? Collections.singleton((Transport) null) : transports; + Set<@Nullable LocationType> lSet = + locations.isEmpty() ? Collections.singleton((LocationType) null) : locations; + Set<@Nullable Colocation> cSet = + colocations.isEmpty() ? Collections.singleton((Colocation) null) : colocations; + + return bSet.stream() + .flatMap( + b -> + tSet.stream() + .flatMap( + t -> + lSet.stream() + .flatMap( + l -> + cSet.stream() + .map(c -> new CrossRunIntersection(b, t, l, c))))) + .filter( + i -> + !(i.backend == null + && i.transport == null + && i.locationType == null + && i.colocation == null)) + .collect(ImmutableSet.toImmutableSet()); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java index 208892bb4dea..0cb06fa66414 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java @@ -23,6 +23,8 @@ import com.google.cloud.storage.it.runner.annotations.Parameterized; import com.google.cloud.storage.it.runner.annotations.Parameterized.Parameter; import com.google.cloud.storage.it.runner.annotations.Parameterized.ParametersProvider; +import com.google.cloud.storage.it.runner.annotations.Colocation; +import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.annotations.SingleBackend; import com.google.cloud.storage.it.runner.registry.Registry; import com.google.common.collect.ImmutableList; @@ -166,7 +168,16 @@ private static List computeRunners(Class klass, Registry registry) .flatMap( b -> ImmutableSet.copyOf(crossRun.transports()).stream() - .map(t -> CrossRunIntersection.of(b, t))) + .flatMap( + t -> + ImmutableSet.copyOf(crossRun.locations()).stream() + .flatMap( + l -> + ImmutableSet.copyOf(crossRun.colocations()).stream() + .map( + c -> + CrossRunIntersection.of( + b, t, l, c))))) .flatMap( c -> { TestInitializer ti = registry.newTestInitializerForCell(c); @@ -187,23 +198,38 @@ private static List computeRunners(Class klass, Registry registry) .collect(ImmutableList.toImmutableList())); } else { Backend backend = singleBackend.value(); - CrossRunIntersection crossRunIntersection = CrossRunIntersection.of(backend, null); - TestInitializer ti = registry.newTestInitializerForCell(crossRunIntersection); - if (parameters != null) { - return SneakyException.unwrap( - () -> - parameters.stream() - .map( - param -> - StorageITLeafRunner.unsafeOf( - testClass, - crossRunIntersection, - fmtParam(param), - ti.andThen(setFieldTo(testClass, param)))) - .collect(ImmutableList.toImmutableList())); - } else { - return ImmutableList.of(StorageITLeafRunner.of(testClass, crossRunIntersection, null, ti)); - } + boolean isDefault = + singleBackend.locations().length == 1 + && singleBackend.locations()[0] == LocationType.REGIONAL_STANDARD + && singleBackend.colocations().length == 1 + && singleBackend.colocations()[0] == Colocation.CO_LOCATED; + + return SneakyException.unwrap( + () -> + ImmutableSet.copyOf(singleBackend.locations()).stream() + .flatMap( + l -> + ImmutableSet.copyOf(singleBackend.colocations()).stream() + .map(c -> CrossRunIntersection.of(backend, null, l, c))) + .flatMap( + c -> { + TestInitializer ti = registry.newTestInitializerForCell(c); + if (parameters != null) { + return parameters.stream() + .map( + param -> + StorageITLeafRunner.unsafeOf( + testClass, + c, + isDefault ? fmtParam(param) : fmtParam(c, param), + ti.andThen(setFieldTo(testClass, param)))); + } else { + return Stream.of( + StorageITLeafRunner.unsafeOf( + testClass, c, isDefault ? null : c.fmtSuiteName(), ti)); + } + }) + .collect(ImmutableList.toImmutableList())); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Backend.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Backend.java index 6e6c1b37b4bc..56e77a5931d7 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Backend.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Backend.java @@ -20,6 +20,8 @@ public enum Backend { /** Use the "Production" GCS endpoints */ PROD, + /** Use the GCS Pre-prod (Staging) endpoints */ + PREPROD, /** Use the test bench as a backend */ TEST_BENCH } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java new file mode 100644 index 000000000000..b3982eb2bc76 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java @@ -0,0 +1,22 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it.runner.annotations; + +public enum Colocation { + CO_LOCATED, + NON_CO_LOCATED +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java index 960e56e94a72..fe6c31b0b757 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java @@ -39,6 +39,10 @@ Transport[] transports(); + LocationType[] locations() default {LocationType.REGIONAL_STANDARD}; + + Colocation[] colocations() default {Colocation.CO_LOCATED}; + /** * Exclude a method from being included in the generated test suite if it's backend and transport * match with those defined. When matching, if the empty set is defined as a value it will be @@ -53,6 +57,10 @@ Backend[] backends() default {}; TransportCompatibility.Transport[] transports() default {}; + + LocationType[] locations() default {}; + + Colocation[] colocations() default {}; } /** @@ -69,6 +77,10 @@ Backend[] backends() default {}; TransportCompatibility.Transport[] transports() default {}; + + LocationType[] locations() default {}; + + Colocation[] colocations() default {}; } /** diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/LocationType.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/LocationType.java new file mode 100644 index 000000000000..f9c47b924a8b --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/LocationType.java @@ -0,0 +1,23 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it.runner.annotations; + +public enum LocationType { + REGIONAL_STANDARD, + REGIONAL_RAPID, + ZONAL_RAPID +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java index d9d98b9ac1ea..09fc2d6a17be 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java @@ -34,4 +34,8 @@ @Retention(RetentionPolicy.RUNTIME) public @interface SingleBackend { Backend value(); + + LocationType[] locations() default {LocationType.REGIONAL_STANDARD}; + + Colocation[] colocations() default {Colocation.CO_LOCATED}; } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java index bfe9a5768754..f82357b533aa 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java @@ -35,8 +35,14 @@ import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.GrpcPlainRequestLoggingInterceptor; +import com.google.cloud.storage.it.runner.CrossRunIntersection; import com.google.cloud.storage.it.runner.annotations.Backend; import com.google.cloud.storage.it.runner.annotations.BucketType; +import com.google.cloud.storage.it.runner.annotations.Colocation; +import com.google.cloud.storage.it.runner.annotations.LocationType; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.junit.runners.model.FrameworkField; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; import com.google.storage.control.v2.StorageControlClient; @@ -48,20 +54,23 @@ import java.util.Locale; import java.util.UUID; + /** The set of resources which are defined for a single backend. */ final class BackendResources implements ManagedLifecycle { private final Backend backend; private final ProtectedBucketNames protectedBucketNames; - + private final ConcurrentMap dynamicBuckets; private final ImmutableList> registryEntries; private BackendResources( Backend backend, ProtectedBucketNames protectedBucketNames, + ConcurrentMap dynamicBuckets, ImmutableList> registryEntries) { this.backend = backend; this.protectedBucketNames = protectedBucketNames; + this.dynamicBuckets = dynamicBuckets; this.registryEntries = registryEntries; } @@ -80,6 +89,8 @@ public void start() {} @Override public void stop() { protectedBucketNames.stop(); + dynamicBuckets.values().forEach(BucketInfoShim::stop); + dynamicBuckets.clear(); } @Override @@ -93,6 +104,8 @@ static BackendResources of( TestRunScopedInstance otelSdk, TestRunScopedInstance zone) { ProtectedBucketNames protectedBucketNames = new ProtectedBucketNames(); + ConcurrentMap dynamicBuckets = new ConcurrentHashMap<>(); + TestRunScopedInstance storageJson = TestRunScopedInstance.of( "fixture/STORAGE/[JSON][" + backend.name() + "]", @@ -106,6 +119,13 @@ static BackendResources of( .setHost(Registry.getInstance().testBench().getBaseUri()) .setProjectId("test-project-id"); break; + case PREPROD: + optionsBuilder = + StorageOptions.http() + .setHost("https://storage-preprod-test-unified.googleusercontent.com/storage/v1_preprod/") + .setProjectId(getPreprodProjectId()) + .setOpenTelemetry(otelSdk.get().get()); + break; default: // PROD, java8 doesn't have exhaustive checking for enum switch // Register the exporters with OpenTelemetry optionsBuilder = StorageOptions.http().setOpenTelemetry(otelSdk.get().get()); @@ -130,6 +150,13 @@ static BackendResources of( .setAttemptDirectPath(false) .setProjectId("test-project-id"); break; + case PREPROD: + optionsBuilder = + StorageOptions.grpc() + .setHost("storage-preprod-test-grpc.googleusercontent.com:443") + .setProjectId(getPreprodProjectId()) + .setOpenTelemetry(otelSdk.get().get()); + break; default: // PROD, java8 doesn't have exhaustive checking for enum switch // Register the exporters with OpenTelemetry optionsBuilder = StorageOptions.grpc().setOpenTelemetry(otelSdk.get().get()); @@ -168,6 +195,18 @@ static BackendResources of( .setEndpoint(endpoint) .setTransportChannelProvider(instantiatingGrpcChannelProvider); break; + case PREPROD: + String preProdEndpoint = "storage-preprod-test-grpc.googleusercontent.com:443"; + builder = + StorageControlSettings.newBuilder() + .setEndpoint(preProdEndpoint) + .setTransportChannelProvider( + StorageControlStubSettings.defaultGrpcTransportProviderBuilder() + .setInterceptorProvider( + GrpcPlainRequestLoggingInterceptor.getInterceptorProvider()) + .setEndpoint(preProdEndpoint) + .build()); + break; default: // PROD, java8 doesn't have exhaustive checking for enum switch builder = StorageControlSettings.newBuilder() @@ -186,20 +225,12 @@ static BackendResources of( throw new RuntimeException(e); } }); - TestRunScopedInstance bucket = + TestRunScopedInstance bucket = TestRunScopedInstance.of( - "fixture/BUCKET/[" + backend.name() + "]", - () -> { - String bucketName = - String.format(Locale.US, "java-storage-grpc-%s", UUID.randomUUID()); - protectedBucketNames.add(bucketName); - return new BucketInfoShim( - BucketInfo.newBuilder(bucketName) - .setLocation(zone.get().get().getRegion()) - .build(), - storageJson.get().getStorage(), - ctrl.get().getCtrl()); - }); + "fixture/BUCKET/[" + backend.name() + "]/DYNAMIC", + () -> + new DynamicBucketLifecycle( + backend, storageJson, ctrl, zone, protectedBucketNames, dynamicBuckets)); TestRunScopedInstance bucketRp = TestRunScopedInstance.of( "fixture/BUCKET/[" + backend.name() + "]/REQUESTER_PAYS", @@ -278,7 +309,18 @@ static BackendResources of( TestRunScopedInstance objectsFixture = TestRunScopedInstance.of( "fixture/OBJECTS/[" + backend.name() + "]", - () -> new ObjectsFixture(storageJson.get().getStorage(), bucket.get().getBucketInfo())); + () -> + new ObjectsFixture( + storageJson.get().getStorage(), + bucket + .get() + .resolve( + null, + CrossRunIntersection.of( + backend, + null, + LocationType.REGIONAL_STANDARD, + Colocation.CO_LOCATED)))); TestRunScopedInstance objectsFixtureRp = TestRunScopedInstance.of( "fixture/OBJECTS/[" + backend.name() + "]/REQUESTER_PAYS", @@ -298,6 +340,7 @@ static BackendResources of( return new BackendResources( backend, protectedBucketNames, + dynamicBuckets, ImmutableList.of( RegistryEntry.of( 40, Storage.class, storageJson, transportAndBackendAre(Transport.HTTP, backend)), @@ -343,4 +386,179 @@ static BackendResources of( backendIs(backend).and(bucketTypeIs(BucketType.HNS))), RegistryEntry.of(100, KmsFixture.class, kmsFixture, backendIs(backend)))); } + + private static final class BucketKey { + private final LocationType locationType; + private final Colocation colocation; + + private BucketKey(LocationType locationType, Colocation colocation) { + this.locationType = locationType; + this.colocation = colocation; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof BucketKey)) { + return false; + } + BucketKey bucketKey = (BucketKey) o; + return locationType == bucketKey.locationType && colocation == bucketKey.colocation; + } + + @Override + public int hashCode() { + return java.util.Objects.hash(locationType, colocation); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("locationType", locationType) + .add("colocation", colocation) + .toString(); + } + } + + private static final class DynamicBucketLifecycle + implements Registry.StatelessManagedLifecycle { + private final Backend backend; + private final TestRunScopedInstance storageJson; + private final TestRunScopedInstance ctrl; + private final TestRunScopedInstance zone; + private final ProtectedBucketNames protectedBucketNames; + private final ConcurrentMap dynamicBuckets; + + private DynamicBucketLifecycle( + Backend backend, + TestRunScopedInstance storageJson, + TestRunScopedInstance ctrl, + TestRunScopedInstance zone, + ProtectedBucketNames protectedBucketNames, + ConcurrentMap dynamicBuckets) { + this.backend = backend; + this.storageJson = storageJson; + this.ctrl = ctrl; + this.zone = zone; + this.protectedBucketNames = protectedBucketNames; + this.dynamicBuckets = dynamicBuckets; + } + + @Override + public BucketInfo resolve(FrameworkField ff, CrossRunIntersection crossRunIntersection) { + LocationType lt = crossRunIntersection.getLocationType(); + Colocation col = crossRunIntersection.getColocation(); + + if (lt == null) { + lt = LocationType.REGIONAL_STANDARD; + } + if (col == null) { + col = Colocation.CO_LOCATED; + } + + BucketKey key = new BucketKey(lt, col); + BucketInfoShim shim = dynamicBuckets.computeIfAbsent(key, this::createBucketShim); + return (BucketInfo) shim.get(); + } + + private BucketInfoShim createBucketShim(BucketKey key) { + Zone z = zone.get().get(); + String region = z.getRegion(); + String zoneName = z.getZone(); + + String rAlt; + String vAlt; + if ("us-east1".equals(region)) { + rAlt = "us-central1"; + vAlt = "us-central1-a"; + } else { + rAlt = "us-east1"; + vAlt = "us-east1-b"; + } + + String targetRegion = region; + String targetZone = zoneName; + + if (key.colocation == Colocation.NON_CO_LOCATED) { + targetRegion = rAlt; + targetZone = vAlt; + } + + BucketInfo.Builder builder; + String prefix; + + switch (key.locationType) { + case REGIONAL_STANDARD: + prefix = "java-storage-reg-std"; + builder = BucketInfo.newBuilder("").setLocation(targetRegion); + break; + case REGIONAL_RAPID: + prefix = "java-storage-reg-rapid"; + builder = + BucketInfo.newBuilder("") + .setLocation(targetRegion) + .setHierarchicalNamespace( + HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()); + break; + case ZONAL_RAPID: + prefix = "java-storage-zon-rapid"; + builder = + BucketInfo.newBuilder("") + .setLocation(targetRegion) + .setCustomPlacementConfig( + CustomPlacementConfig.newBuilder() + .setDataLocations(ImmutableList.of(targetZone)) + .build()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .setHierarchicalNamespace( + HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()); + break; + default: + throw new IllegalArgumentException("Unknown location type: " + key.locationType); + } + + String bucketName = + String.format( + Locale.US, + "%s-%s-%s", + prefix, + backend.name().toLowerCase(Locale.US), + UUID.randomUUID().toString().substring(0, 8)); + + builder.setName(bucketName); + protectedBucketNames.add(bucketName); + + BucketInfoShim shim = + new BucketInfoShim( + builder.build(), + key.locationType, + targetZone, + storageJson.get().getStorage(), + ctrl.get().getCtrl()); + + shim.start(); + return shim; + } + } + + private static String getPreprodProjectId() { + String projectId = System.getenv("GOOGLE_CLOUD_PROJECT"); + if (projectId == null || projectId.isEmpty()) { + projectId = System.getProperty("google.cloud.project"); + } + if (projectId == null || projectId.isEmpty()) { + projectId = "gcs-hyd-connector-benchmarks"; + } + return projectId; + } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java index 0c981895038b..8a0e3b71f7d2 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java @@ -22,6 +22,10 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.it.BucketCleaner; +import com.google.cloud.storage.it.runner.annotations.LocationType; +import com.google.protobuf.Duration; +import com.google.storage.control.v2.BucketName; +import com.google.storage.control.v2.RapidCache; import com.google.storage.control.v2.StorageControlClient; import java.util.Locale; @@ -29,13 +33,26 @@ final class BucketInfoShim implements ManagedLifecycle { private final BucketInfo bucketInfo; + private final LocationType locationType; + private final String targetZone; private final Storage s; private final StorageControlClient ctrl; private BucketInfo createdBucket; BucketInfoShim(BucketInfo bucketInfo, Storage s, StorageControlClient ctrl) { + this(bucketInfo, LocationType.REGIONAL_STANDARD, null, s, ctrl); + } + + BucketInfoShim( + BucketInfo bucketInfo, + LocationType locationType, + String targetZone, + Storage s, + StorageControlClient ctrl) { this.bucketInfo = bucketInfo; + this.locationType = locationType; + this.targetZone = targetZone; this.s = s; this.ctrl = ctrl; } @@ -53,6 +70,31 @@ public Object get() { public void start() { try { createdBucket = s.create(bucketInfo).asBucketInfo(); + if (locationType == LocationType.REGIONAL_RAPID) { + if (ctrl == null) { + throw new IllegalStateException( + "StorageControlClient is required for REGIONAL_RAPID but was not provided"); + } + String cacheName = + String.format( + Locale.US, + "projects/_/buckets/%s/rapidCaches/%s", + createdBucket.getName(), + targetZone); + RapidCache rapidCache = + RapidCache.newBuilder() + .setName(cacheName) + .setZone(targetZone) + .setCacheType("rapid-cache-ultra") + .setTtl(Duration.newBuilder().setSeconds(86400).build()) // 24 hours + .build(); + try { + ctrl.createRapidCacheAsync(BucketName.format("_", createdBucket.getName()), rapidCache) + .get(); + } catch (Exception e) { + throw new RuntimeException("Failed to create Rapid Cache: " + e.getMessage(), e); + } + } } catch (StorageException se) { String msg = se.getMessage().toLowerCase(Locale.US); if (se.getCode() == 400 && (msg.contains("not a valid zone in location")) diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java index c232cd32de6c..0db457ae507f 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java @@ -91,6 +91,8 @@ public final class Registry extends RunListener { private final BackendResources prodBackendResources = BackendResources.of(Backend.PROD, otelSdk, zone); + private final BackendResources preProdBackendResources = + BackendResources.of(Backend.PREPROD, otelSdk, zone); private final BackendResources testBenchBackendResource = BackendResources.of(Backend.TEST_BENCH, otelSdk, zone); @@ -104,6 +106,7 @@ public final class Registry extends RunListener { registryEntry(3, Backend.class, CrossRunIntersection::getBackend), registryEntry(4, Transport.class, CrossRunIntersection::getTransport)) .addAll(prodBackendResources.getRegistryEntries()) + .addAll(preProdBackendResources.getRegistryEntries()) .addAll(testBenchBackendResource.getRegistryEntries()) .build(); @@ -281,7 +284,7 @@ private void shutdown() { } @FunctionalInterface - private interface StatelessManagedLifecycle extends ManagedLifecycle { + interface StatelessManagedLifecycle extends ManagedLifecycle { T resolve(FrameworkField ff, CrossRunIntersection crossRunIntersection); @Override From 5dc69d5671d30b9d73caeb5a99e691a40a7ad4f1 Mon Sep 17 00:00:00 2001 From: Nidhi Nandwani Date: Thu, 20 Aug 2026 09:35:28 +0000 Subject: [PATCH 2/2] add integration test files --- .../storage/ITObjectReadSessionFakeTest.java | 204 ++++++- .../cloud/storage/ITRcuBidiWriteTest.java | 552 ++++++++++++++++++ .../cloud/storage/it/ITRapidCacheTest.java | 220 +++++++ .../storage/it/ITRcuBidiReadTempTest.java | 155 +++++ .../cloud/storage/it/ITRcuBidiReadTest.java | 359 ++++++++++++ .../it/runner/CrossRunIntersection.java | 84 +-- .../storage/it/runner/StorageITRunner.java | 19 +- .../it/runner/annotations/Colocation.java | 22 - .../it/runner/annotations/CrossRun.java | 6 - .../it/runner/annotations/SingleBackend.java | 1 - .../it/runner/registry/BackendResources.java | 82 ++- .../it/runner/registry/BucketInfoShim.java | 37 +- .../storage/it/runner/registry/Registry.java | 13 +- 13 files changed, 1609 insertions(+), 145 deletions(-) create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java create mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java delete mode 100644 java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java index 1cb32704982b..eaeb4019bcee 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITObjectReadSessionFakeTest.java @@ -210,8 +210,16 @@ public void bidiReadObjectRedirectedError() throws Exception { req3, respond -> respond.onNext(res2))); + GrpcRequestAuditing requestAuditing = new GrpcRequestAuditing(); try (FakeServer fakeServer = FakeServer.of(fake); - Storage storage = fakeServer.getGrpcStorageOptions().toBuilder().build().getService()) { + Storage storage = + fakeServer.getGrpcStorageOptions().toBuilder() + .setGrpcInterceptorProvider( + () -> + ImmutableList.of( + requestAuditing, GrpcPlainRequestLoggingInterceptor.getInstance())) + .build() + .getService()) { BlobId id = BlobId.of("b", "o"); ApiFuture futureBlobDescriptor = storage.blobReadSession(id); @@ -222,6 +230,12 @@ public void bidiReadObjectRedirectedError() throws Exception { .get(1, TimeUnit.SECONDS); assertThat(xxd(actual)).isEqualTo(xxd(content.getBytes())); + + requestAuditing + .assertRequestHeader("x-goog-request-params") + .containsExactly( + "bucket=projects/_/buckets/b", + "bucket=projects/_/buckets/b&routing_token=" + routingToken); } } } @@ -1805,6 +1819,193 @@ static ObjectReadSessionImpl getObjectReadSessionImpl(BlobReadSession bd) { return orsi; } + private static Consumer> onRedirect( + BidiReadHandle handle, String token) { + return respond -> { + BidiReadObjectRedirectedError redirect = + BidiReadObjectRedirectedError.newBuilder() + .setReadHandle(handle) + .setRoutingToken(token) + .build(); + + com.google.rpc.Status grpcStatusDetails = + com.google.rpc.Status.newBuilder() + .setCode(com.google.rpc.Code.UNAVAILABLE_VALUE) + .setMessage("redirect") + .addDetails(Any.pack(redirect)) + .build(); + + Metadata trailers = new Metadata(); + trailers.put(GRPC_STATUS_DETAILS_KEY, grpcStatusDetails); + StatusRuntimeException statusRuntimeException = + Status.UNAVAILABLE.withDescription("redirect").asRuntimeException(trailers); + respond.onError(statusRuntimeException); + }; + } + + @Test + public void bidiReadObjectRedirectedError_redirectCounterResetOnResponse() throws Exception { + BidiReadHandle handle1 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-1")).build(); + BidiReadHandle handle2 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-2")).build(); + BidiReadHandle handle3 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-3")).build(); + BidiReadHandle handle4 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-4")).build(); + BidiReadHandle handle5 = BidiReadHandle.newBuilder().setHandle(ByteString.copyFromUtf8("handle-5")).build(); + + BidiReadObjectRequest req_read_1 = read(1, 10, 5); + + BidiReadObjectRequest req_read_1_redirected_1 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle1) + .setRoutingToken("token-1") + .build()) + .addReadRanges(getReadRange(1, 10, 5)) + .build(); + + BidiReadObjectRequest req_read_2 = + BidiReadObjectRequest.newBuilder() + .addReadRanges(getReadRange(2, 15, 5)) + .build(); + + BidiReadObjectRequest req_read_2_redirected_2 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle2) + .setRoutingToken("token-2") + .build()) + .addReadRanges(getReadRange(2, 15, 5)) + .build(); + + BidiReadObjectRequest req_read_3 = + BidiReadObjectRequest.newBuilder() + .addReadRanges(getReadRange(3, 20, 5)) + .build(); + + BidiReadObjectRequest req_read_3_redirected_3 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle3) + .setRoutingToken("token-3") + .build()) + .addReadRanges(getReadRange(3, 20, 5)) + .build(); + + BidiReadObjectRequest req_read_3_redirected_4 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle4) + .setRoutingToken("token-4") + .build()) + .addReadRanges(getReadRange(3, 20, 5)) + .build(); + + BidiReadObjectRequest req_read_3_redirected_5 = + BidiReadObjectRequest.newBuilder() + .setReadObjectSpec( + BidiReadObjectSpec.newBuilder() + .setBucket(METADATA.getBucket()) + .setObject(METADATA.getName()) + .setGeneration(1) + .setReadHandle(handle5) + .setRoutingToken("token-5") + .build()) + .addReadRanges(getReadRange(3, 20, 5)) + .build(); + + ChecksummedTestContent content1 = ChecksummedTestContent.of(ALL_OBJECT_BYTES, 10, 5); + BidiReadObjectResponse res_read_1 = + BidiReadObjectResponse.newBuilder() + .setMetadata(METADATA) + .addObjectDataRanges( + ObjectRangeData.newBuilder() + .setChecksummedData(content1.asChecksummedData()) + .setReadRange(getReadRange(1, 10, 5)) + .setRangeEnd(true) + .build()) + .build(); + + ChecksummedTestContent content2 = ChecksummedTestContent.of(ALL_OBJECT_BYTES, 15, 5); + BidiReadObjectResponse res_read_2 = + BidiReadObjectResponse.newBuilder() + .setMetadata(METADATA) + .addObjectDataRanges( + ObjectRangeData.newBuilder() + .setChecksummedData(content2.asChecksummedData()) + .setReadRange(getReadRange(2, 15, 5)) + .setRangeEnd(true) + .build()) + .build(); + + ChecksummedTestContent content3 = ChecksummedTestContent.of(ALL_OBJECT_BYTES, 20, 5); + BidiReadObjectResponse res_read_3 = + BidiReadObjectResponse.newBuilder() + .setMetadata(METADATA) + .addObjectDataRanges( + ObjectRangeData.newBuilder() + .setChecksummedData(content3.asChecksummedData()) + .setReadRange(getReadRange(3, 20, 5)) + .setRangeEnd(true) + .build()) + .build(); + + FakeStorage fake = + FakeStorage.of( + ImmutableMap.>>builder() + .put(REQ_OPEN, respond -> respond.onNext(RES_OPEN)) + .put(req_read_1, onRedirect(handle1, "token-1")) + .put(req_read_1_redirected_1, respond -> respond.onNext(res_read_1)) + .put(req_read_2, onRedirect(handle2, "token-2")) + .put(req_read_2_redirected_2, respond -> respond.onNext(res_read_2)) + .put(req_read_3, onRedirect(handle3, "token-3")) + .put(req_read_3_redirected_3, onRedirect(handle4, "token-4")) + .put(req_read_3_redirected_4, onRedirect(handle5, "token-5")) + .put(req_read_3_redirected_5, respond -> respond.onNext(res_read_3)) + .build()); + + try (FakeServer fakeServer = FakeServer.of(fake); + Storage storage = fakeServer.getGrpcStorageOptions().toBuilder().build().getService()) { + + BlobId id = BlobId.of("b", "o"); + ApiFuture futureBlobDescriptor = storage.blobReadSession(id); + + try (BlobReadSession bd = futureBlobDescriptor.get(5, TimeUnit.SECONDS)) { + // Read 1 (should trigger Redirect 1, then succeed) + byte[] actual1 = + bd.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(10L, 5L))) + .get(1, TimeUnit.SECONDS); + assertThat(xxd(actual1)).isEqualTo(xxd(content1.getBytes())); + + // Read 2 (should trigger Redirect 2, then succeed) + byte[] actual2 = + bd.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(15L, 5L))) + .get(1, TimeUnit.SECONDS); + assertThat(xxd(actual2)).isEqualTo(xxd(content2.getBytes())); + + // Read 3 (should trigger Redirect 3, Redirect 4, Redirect 5, then succeed) + byte[] actual3 = + bd.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(20L, 5L))) + .get(1, TimeUnit.SECONDS); + assertThat(xxd(actual3)).isEqualTo(xxd(content3.getBytes())); + } + } + } + static final class FakeStorage extends StorageImplBase { private final Map>> db; @@ -1823,6 +2024,7 @@ public void onNext(BidiReadObjectRequest req) { if (db.containsKey(req)) { db.get(req).accept(respond); } else { + System.err.println("FakeStorage: UNEXPECTED REQUEST:\n" + req); respond.onError(TestUtils.apiException(Code.UNIMPLEMENTED, "Unexpected request")); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java new file mode 100644 index 000000000000..dff3a407d577 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ITRcuBidiWriteTest.java @@ -0,0 +1,552 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.storage; + +import static com.google.cloud.storage.TestUtils.assertAll; +import static com.google.cloud.storage.TestUtils.xxd; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; + +import com.google.api.core.ApiFuture; +import com.google.cloud.storage.BlobAppendableUpload.AppendableUploadWriteableByteChannel; +import com.google.cloud.storage.BlobAppendableUploadConfig.CloseAction; +import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown; +import com.google.cloud.storage.FlushPolicy.MaxFlushSizeFlushPolicy; +import com.google.cloud.storage.FlushPolicy.MinFlushSizeFlushPolicy; +import com.google.cloud.storage.ITRcuBidiWriteTest.UploadConfigParameters; +import com.google.cloud.storage.MetadataField.PartRange; +import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.it.ChecksummedTestContent; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.annotations.LocationType; +import com.google.cloud.storage.it.runner.annotations.Parameterized; +import com.google.cloud.storage.it.runner.annotations.Parameterized.Parameter; +import com.google.cloud.storage.it.runner.annotations.Parameterized.ParametersProvider; +import com.google.cloud.storage.it.runner.registry.Generator; +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Paths; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@CrossRun( + backends = {Backend.PROD}, + transports = Transport.GRPC, + locations = { + LocationType.REGIONAL_RAPID + }) +@Parameterized(UploadConfigParameters.class) +public final class ITRcuBidiWriteTest { + + @Inject public Generator generator; + + @Inject public Storage storage; + + @Inject public BucketInfo bucket; + + @Inject public Backend backend; + + @Parameter public Param p; + + @Test + public void appendableUpload_emptyObject() + throws IOException, ExecutionException, InterruptedException, TimeoutException { + assumeTrue( + "only run once", + p.content.length() == UploadConfigParameters.objectSizes.get(0) + && p.uploadConfig.getCloseAction() == UploadConfigParameters.closeActions.get(0) + && p.uploadConfig.getFlushPolicy().equals(UploadConfigParameters.flushPolicies.get(0))); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bucket, UUID.randomUUID().toString()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + upload.open().close(); + + BlobInfo actual = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(actual.getSize()).isEqualTo(0); + assertThat(actual.getCrc32c()) + .isEqualTo(Utils.crc32cCodec.encode(Crc32cValue.zero().getValue())); + + assumeFalse( + "Testbench doesn't handle {read_id: 1, read_offset: 0} for a 0 byte object", + backend == Backend.TEST_BENCH); + byte[] actualBytes = readAllBytes(actual); + assertThat(xxd(actualBytes)).isEqualTo(xxd(new byte[0])); + } + + @Test + public void appendableUpload_bytes() + throws IOException, ExecutionException, InterruptedException, TimeoutException { + checkTestbenchIssue733(); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bucket, UUID.randomUUID().toString()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + // cut out the middle + 1 byte + int length = p.content.length(); + int mid = length / 2; + ChecksummedTestContent a1 = p.content.slice(0, mid); + ChecksummedTestContent a2 = p.content.slice(mid + 1, length - mid - 1); + ChecksummedTestContent a1_a2 = a1.concat(a2); + Crc32cLengthKnown c1_c2 = Crc32cValue.of(a1_a2.getCrc32c(), a1_a2.length()); + + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written1 = Buffers.emptyTo(ByteBuffer.wrap(a1.getBytes()), channel); + assertThat(written1).isEqualTo(a1.length()); + int written2 = Buffers.emptyTo(ByteBuffer.wrap(a2.getBytes()), channel); + assertThat(written2).isEqualTo(a2.length()); + } + + BlobInfo actual = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(actual.getSize()).isEqualTo(c1_c2.getLength()); + assertThat(actual.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(c1_c2.getValue())); + + byte[] actualBytes = readAllBytes(actual); + assertThat(xxd(actualBytes)).isEqualTo(xxd(a1_a2.getBytes())); + } + + @Test + public void explicitFlush() + throws IOException, ExecutionException, InterruptedException, TimeoutException { + checkTestbenchIssue733(); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bucket, UUID.randomUUID().toString()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + ByteBuffer src = p.content.asByteBuffer(); + ByteBuffer zed = src.slice(); + zed.limit(zed.position() + 1); + src.position(src.position() + 1); + + int written = channel.write(zed); + assertThat(written).isEqualTo(1); + channel.flush(); + + written = StorageChannelUtils.blockingEmptyTo(src, channel); + assertThat(written).isEqualTo(p.content.length() - 1); + } + + BlobInfo gen1 = upload.getResult().get(3, TimeUnit.SECONDS); + assertThat(gen1.getSize()).isEqualTo(p.content.length()); + assertThat(gen1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(p.content.getCrc32c())); + } + + @Test + // Pending work in testbench: https://github.com/googleapis/storage-testbench/issues/723 + // manually verified internally on 2025-03-25 + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void appendableBlobUploadTakeover() throws Exception { + + List chunks = p.content.chunkup((p.content.length() / 2) + 1); + assertThat(chunks).hasSize(2); + + ChecksummedTestContent c1 = chunks.get(0); + ChecksummedTestContent c2 = chunks.get(1); + + BlobId id = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + BlobAppendableUploadConfig doNotFinalizeConfig = + p.uploadConfig.withCloseAction(CloseAction.CLOSE_WITHOUT_FINALIZING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(id).setStorageClass(StorageClass.valueOf("RAPID")).build(), + doNotFinalizeConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(c1.getBytes()), channel); + assertThat(written).isEqualTo(c1.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(c1.length()); + assertThat(done1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(c1.getCrc32c())); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(c2.getBytes()), channel); + assertThat(written).isEqualTo(c2.length()); + } + BlobInfo done2 = takeOver.getResult().get(5, TimeUnit.SECONDS); + + assertThat(done2.getSize()).isEqualTo(p.content.length()); + assertThat(done2.getCrc32c()).isAnyOf(Utils.crc32cCodec.encode(p.content.getCrc32c()), null); + } + + @Test + public void testUploadFileUsingAppendable() throws Exception { + checkTestbenchIssue733(); + + String objectName = UUID.randomUUID().toString(); + String fileName = + ParallelCompositeUploadBlobWriteSessionConfig.PartNamingStrategy.noPrefix() + .fmtName(objectName, PartRange.of(1)); + BlobId bid = BlobId.of(bucket.getName(), objectName); + int fileSize = p.content.length(); + try (TmpFile tmpFile = + TmpFile.of(Paths.get(System.getProperty("java.io.tmpdir")), fileName + ".", ".bin")) { + try (SeekableByteChannel w = tmpFile.writer()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), w); + assertThat(written).isEqualTo(p.content.length()); + } + + BlobAppendableUpload appendable = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (SeekableByteChannel r = tmpFile.reader(); + AppendableUploadWriteableByteChannel w = appendable.open()) { + long copied = ByteStreams.copy(r, w); + assertThat(copied).isEqualTo(fileSize); + } + BlobInfo bi = appendable.getResult().get(5, TimeUnit.SECONDS); + assertThat(bi.getSize()).isEqualTo(fileSize); + } + } + + @Test + // Pending work in testbench: https://github.com/googleapis/storage-testbench/issues/723 + // manually verified internally on 2025-03-25 + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void takeoverJustToFinalizeWorks() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(p.content.length()); + assertThat(done1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(p.content.getCrc32c())); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + takeOver.open().finalizeAndClose(); + + BlobInfo done2 = takeOver.getResult().get(5, TimeUnit.SECONDS); + assertAll( + () -> assertThat(done2).isNotNull(), + () -> assertThat(done2.getSize()).isEqualTo(p.content.length()), + () -> assertThat(done2.getCrc32c()).isNotNull()); + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void explicitFinalizeWithCorrectChecksum() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + + String expectedCrc = Utils.crc32cCodec.encode(p.content.getCrc32c()); + channel.finalizeAndClose(expectedCrc); + } + + BlobInfo gen1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(gen1.getSize()).isEqualTo(p.content.length()); + assertThat(gen1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(p.content.getCrc32c())); + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void explicitFinalizeWithIncorrectChecksumFails() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + + String badCrc = Utils.crc32cCodec.encode(Crc32cValue.zero().getValue()); + try { + channel.finalizeAndClose(badCrc); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (IOException e) { + assertThat(e.getMessage().toLowerCase()).contains("mismatch"); + } + } + + try { + upload.getResult().get(5, TimeUnit.SECONDS); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (ExecutionException e) { + // The server rejects it + assertThat(e.getCause().getMessage().toLowerCase()).contains("mismatch"); + } + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void takeoverJustToFinalizeWithIncorrectChecksumFails() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(p.content.length()); + assertThat(done1.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(p.content.getCrc32c())); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + String badCrc = Utils.crc32cCodec.encode(Crc32cValue.zero().getValue()); + try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { + try { + channel.finalizeAndClose(badCrc); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (IOException e) { + assertThat(e.getMessage().toLowerCase()).contains("mismatch"); + } + } + + try { + takeOver.getResult().get(5, TimeUnit.SECONDS); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (ExecutionException e) { + // The server rejects it + assertThat(e.getCause().getMessage().toLowerCase()).contains("mismatch"); + } + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void takeoverAndAppendWithCorrectChecksumWorks() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(p.content.length()); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + + ChecksummedTestContent fullContent = p.content.concat(p.content); + String expectedCrc = Utils.crc32cCodec.encode(fullContent.getCrc32c()); + channel.finalizeAndClose(expectedCrc); + } + + BlobInfo done2 = takeOver.getResult().get(5, TimeUnit.SECONDS); + assertThat(done2.getSize()).isEqualTo(p.content.length() * 2); + ChecksummedTestContent fullContent = p.content.concat(p.content); + assertThat(done2.getCrc32c()).isEqualTo(Utils.crc32cCodec.encode(fullContent.getCrc32c())); + } + + @Test + @CrossRun.Ignore(backends = {Backend.TEST_BENCH}) + public void takeoverAndAppendWithIncorrectChecksumFails() throws Exception { + BlobId bid = BlobId.of(bucket.getName(), UUID.randomUUID().toString()); + assumeTrue( + "manually finalizing", + p.uploadConfig.getCloseAction() != CloseAction.FINALIZE_WHEN_CLOSING); + + BlobAppendableUpload upload = + storage.blobAppendableUpload( + BlobInfo.newBuilder(bid).setStorageClass(StorageClass.valueOf("RAPID")).build(), + p.uploadConfig); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + } + BlobInfo done1 = upload.getResult().get(5, TimeUnit.SECONDS); + assertThat(done1.getSize()).isEqualTo(p.content.length()); + + BlobAppendableUpload takeOver = + storage.blobAppendableUpload( + BlobInfo.newBuilder(done1.getBlobId()) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(), p.uploadConfig); + + try (AppendableUploadWriteableByteChannel channel = takeOver.open()) { + int written = Buffers.emptyTo(ByteBuffer.wrap(p.content.getBytes()), channel); + assertThat(written).isEqualTo(p.content.length()); + + String badCrc = Utils.crc32cCodec.encode(Crc32cValue.zero().getValue()); + try { + channel.finalizeAndClose(badCrc); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (IOException e) { + assertThat(e.getMessage().toLowerCase()).contains("mismatch"); + } + } + + try { + takeOver.getResult().get(5, TimeUnit.SECONDS); + org.junit.Assert.fail("Expected exception due to checksum mismatch"); + } catch (ExecutionException e) { + assertThat(e.getCause().getMessage().toLowerCase()).contains("mismatch"); + } + } + + private void checkTestbenchIssue733() { + if (backend == Backend.TEST_BENCH + && p.uploadConfig.getCloseAction() == CloseAction.FINALIZE_WHEN_CLOSING) { + int estimatedMessageCount = 0; + FlushPolicy flushPolicy = p.uploadConfig.getFlushPolicy(); + if (flushPolicy instanceof MinFlushSizeFlushPolicy) { + MinFlushSizeFlushPolicy min = (MinFlushSizeFlushPolicy) flushPolicy; + estimatedMessageCount = p.content.length() / min.getMinFlushSize(); + } else if (flushPolicy instanceof MaxFlushSizeFlushPolicy) { + MaxFlushSizeFlushPolicy max = (MaxFlushSizeFlushPolicy) flushPolicy; + estimatedMessageCount = p.content.length() / max.getMaxFlushSize(); + } + // if our int division results in a partial message, ensure we are counting at least one + // message. We have a separate test specifically for empty objects. + estimatedMessageCount = Math.max(estimatedMessageCount, 1); + assumeTrue( + "testbench broken https://github.com/googleapis/storage-testbench/issues/733", + estimatedMessageCount > 1); + } + } + + private byte[] readAllBytes(BlobInfo actual) + throws IOException, InterruptedException, ExecutionException, TimeoutException { + ApiFuture blobReadSessionFuture = storage.blobReadSession(actual.getBlobId()); + try (BlobReadSession read = blobReadSessionFuture.get(2_372, TimeUnit.MILLISECONDS)) { + ApiFuture futureBytes = read.readAs(ReadProjectionConfigs.asFutureBytes()); + return futureBytes.get(2_273, TimeUnit.MILLISECONDS); + } + } + + public static final class UploadConfigParameters implements ParametersProvider { + + private static final ImmutableList flushPolicies = + ImmutableList.of( + FlushPolicy.minFlushSize(1_000), + FlushPolicy.minFlushSize(1_000).withMaxPendingBytes(5_000), + FlushPolicy.maxFlushSize(500_000), + FlushPolicy.minFlushSize(), + FlushPolicy.maxFlushSize()); + private static final ImmutableList closeActions = + ImmutableList.copyOf(CloseAction.values()); + public static final ImmutableList objectSizes = + ImmutableList.of(5, 500, 5_000, 500_000, 5_000_000); + + @Override + public ImmutableList parameters() { + ImmutableList.Builder builder = ImmutableList.builder(); + for (FlushPolicy fp : flushPolicies) { + for (CloseAction ca : closeActions) { + for (int size : objectSizes) { + Param param = + new Param( + ChecksummedTestContent.gen(size), + BlobAppendableUploadConfig.of().withFlushPolicy(fp).withCloseAction(ca)); + builder.add(param); + } + } + } + return builder.build(); + } + } + + public static final class Param { + private final ChecksummedTestContent content; + private final BlobAppendableUploadConfig uploadConfig; + + private Param(ChecksummedTestContent content, BlobAppendableUploadConfig uploadConfig) { + this.content = content; + this.uploadConfig = uploadConfig; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("byteCount", content) + .add("uploadConfig", uploadConfig) + .toString(); + } + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java new file mode 100644 index 000000000000..d871d9054ba0 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRapidCacheTest.java @@ -0,0 +1,220 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.StatusCode; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.BucketInfo.CustomPlacementConfig; +import com.google.cloud.storage.BucketInfo.HierarchicalNamespace; +import com.google.cloud.storage.BucketInfo.IamConfiguration; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageClass; +import com.google.cloud.storage.StorageOptions; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Duration; +import com.google.protobuf.FieldMask; +import com.google.storage.control.v2.BucketName; +import com.google.storage.control.v2.RapidCache; +import com.google.storage.control.v2.StorageControlClient; +import com.google.storage.control.v2.StorageControlSettings; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class ITRapidCacheTest { + + private static final String PROJECT_ID = "gcs-hyd-connector-benchmarks"; + private static StorageControlClient controlClient; + private static Storage storageClient; + private static String bucketName; + + // Shared Cache Details + private static String cacheId; + private static String cacheName; + + @BeforeClass + public static void setUpClass() throws Exception { + // Initialize standard Storage client for preprod (gRPC) + storageClient = StorageOptions.grpc() + .setProjectId(PROJECT_ID) + .setHost("storage-preprod-test-grpc.googleusercontent.com:443") + .build() + .getService(); + + // Initialize StorageControl client for preprod (gRPC) + StorageControlSettings controlSettings = StorageControlSettings.newBuilder() + .setEndpoint("storage-preprod-test-grpc.googleusercontent.com:443") + .build(); + controlClient = StorageControlClient.create(controlSettings); + + // Create HNS enabled regional bucket in preprod us-central1 + bucketName = "java-storage-preprod-rapid-" + UUID.randomUUID().toString().substring(0, 8); + BucketInfo bucketInfo = BucketInfo.newBuilder(bucketName) + .setLocation("us-central1") + .setHierarchicalNamespace(HierarchicalNamespace.newBuilder().setEnabled(true).build()) + .setIamConfiguration( + IamConfiguration.newBuilder() + .setIsUniformBucketLevelAccessEnabled(true) + .build()) + .build(); + storageClient.create(bucketInfo); + + // Define shared cache ID (forced to be the zone name by the backend) + cacheId = "us-central1-a"; + cacheName = String.format("projects/_/buckets/%s/rapidCaches/%s", bucketName, cacheId); + } + + @AfterClass + public static void tearDownClass() throws Exception { + if (storageClient != null && bucketName != null) { + try { + storageClient.delete(bucketName); + } catch (Exception e) { + System.err.println("Failed to clean up preprod bucket: " + e.getMessage()); + } + } + if (controlClient != null) { + controlClient.close(); + } + } + + // --- Test Cases (Alphabetical Sort Order matches Logical Lifecycle) --- + + @Test + public void createRapidCache() throws Exception { + RapidCache rapidCache = RapidCache.newBuilder() + .setName(cacheName) + .setZone("us-central1-a") + .setCacheType("rapid-cache-ultra") + .setTtl(Duration.newBuilder().setSeconds(86400).build()) // 24 hours + .build(); + + RapidCache created = controlClient.createRapidCacheAsync( + BucketName.format("_", bucketName), rapidCache).get(); + + assertThat(created).isNotNull(); + assertThat(created.getName()).isEqualTo(cacheName); + assertThat(created.getState()).isEqualTo("running"); + } + + @Test + public void createRapidCache_duplicate() throws Exception { + RapidCache rapidCache = RapidCache.newBuilder() + .setName(cacheName) // Use the same name as the shared cache + .setZone("us-central1-a") + .setCacheType("rapid-cache-ultra") + .build(); + + try { + controlClient.createRapidCacheAsync( + BucketName.format("_", bucketName), rapidCache).get(); + fail("Expected AlreadyExists exception"); + } catch (ExecutionException e) { + assertThat(e.getCause()).isInstanceOf(ApiException.class); + ApiException apiException = (ApiException) e.getCause(); + assertThat(apiException.getStatusCode().getCode()).isEqualTo(StatusCode.Code.ALREADY_EXISTS); + } + } + + @Test + public void createRapidCache_invalidConfig() { + String invalidCacheId = "invalid-cache-" + UUID.randomUUID().toString().substring(0, 8); + String invalidCacheName = String.format("projects/_/buckets/%s/rapidCaches/%s", bucketName, invalidCacheId); + + RapidCache rapidCache = RapidCache.newBuilder() + .setName(invalidCacheName) + .setZone("invalid-zone") + .setCacheType("rapid-cache-ultra") + .build(); + + try { + controlClient.createRapidCacheAsync( + BucketName.format("_", bucketName), rapidCache).get(); + fail("Expected InvalidArgument exception"); + } catch (ExecutionException e) { + assertThat(e.getCause()).isInstanceOf(ApiException.class); + ApiException apiException = (ApiException) e.getCause(); + assertThat(apiException.getStatusCode().getCode()).isEqualTo(StatusCode.Code.INVALID_ARGUMENT); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("Interrupted"); + } + } + + @Test + public void getRapidCache() throws Exception { + RapidCache retrieved = controlClient.getRapidCache(cacheName); + assertThat(retrieved).isNotNull(); + assertThat(retrieved.getName()).isEqualTo(cacheName); + assertThat(retrieved.getState()).isEqualTo("running"); + } + + @Test + public void getRapidCache_nonExistent() { + String nonExistentCacheName = String.format("projects/_/buckets/%s/rapidCaches/non-existent-cache", bucketName); + + try { + controlClient.getRapidCache(nonExistentCacheName); + fail("Expected NotFound exception"); + } catch (ApiException e) { + assertThat(e.getStatusCode().getCode()).isEqualTo(StatusCode.Code.NOT_FOUND); + } + } + + @Test + public void listRapidCaches() throws Exception { + StorageControlClient.ListRapidCachesPagedResponse response = + controlClient.listRapidCaches(BucketName.format("_", bucketName)); + + List names = new ArrayList<>(); + for (RapidCache rc : response.iterateAll()) { + names.add(rc.getName()); + } + + assertThat(names).contains(cacheName); + } + + @Test + @Ignore("b/483013082: UpdateRapidCache returns 500 Internal error in PreProd") + public void updateRapidCache() throws Exception { + RapidCache toUpdate = RapidCache.newBuilder() + .setName(cacheName) + .setZone("us-central1-a") + .setCacheType("rapid-cache-ultra") + .setTtl(Duration.newBuilder().setSeconds(172800).build()) // 48h + .build(); + + FieldMask updateMask = FieldMask.newBuilder().addPaths("ttl").build(); + + // No try-catch wrapping of ExecutionException. Let the test fail naturally if update fails. + RapidCache updated = controlClient.updateRapidCacheAsync(toUpdate, updateMask).get(); + assertThat(updated.getTtl().getSeconds()).isEqualTo(172800); + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java new file mode 100644 index 000000000000..0da263be7a24 --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTempTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.api.core.ApiFuture; +import com.google.cloud.storage.AsyncSessionClosedException; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.BlobReadSession; +import com.google.cloud.storage.RangeSpec; +import com.google.cloud.storage.ReadProjectionConfigs; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.ZeroCopySupport.DisposableByteString; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public final class ITRcuBidiReadTempTest { + + private static final String PROJECT_ID = "gcs-hyd-connector-benchmarks"; + private static final String BUCKET_NAME = "java-storage-reg-rapid-preprod-3fe2bb58"; // Reusing active bucket with running cache + private static Storage storage; + + @BeforeClass + public static void setUpClass() throws Exception { + System.out.println("Initializing storage client pointing to pre-prod endpoint..."); + storage = StorageOptions.grpc() + .setHost("storage-preprod-test-grpc.googleusercontent.com:443") + .setProjectId(PROJECT_ID) + .build() + .getService(); + } + + @AfterClass + public static void tearDownClass() throws Exception { + // No cleanup of BUCKET_NAME since it is a shared pre-created bucket + } + + @Test + public void readPostStreamClose() throws Exception { + System.out.println("Running readPostStreamClose against bucket " + BUCKET_NAME); + + // Generate 5MB of random data + int dataSize = 5 * 1024 * 1024; + byte[] data = new byte[dataSize]; + new Random().nextBytes(data); + + BlobId blobId = BlobId.of(BUCKET_NAME, "test-bidi-read-close-temp-" + UUID.randomUUID()); + storage.create(BlobInfo.newBuilder(blobId).build(), data); + + try { + ApiFuture futureSession = storage.blobReadSession(blobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start a future read for the entire object + ApiFuture readFuture = session.readAs(ReadProjectionConfigs.asFutureBytes()); + + // Close the session immediately while the transfer is in flight + session.close(); + + // Resolving the future should now fail since the session is closed + try { + readFuture.get(5, TimeUnit.SECONDS); + fail("Expected ExecutionException when reading after session close"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertThat(cause).isInstanceOf(StorageException.class); + assertThat(cause.getCause()).isInstanceOf(AsyncSessionClosedException.class); + System.out.println("Successfully caught expected exception: " + cause.getMessage()); + } + } + } finally { + storage.delete(blobId); + } + } + + @Test + public void zeroCopyRangeReads() throws Exception { + System.out.println("Running zeroCopyRangeReads against bucket " + BUCKET_NAME); + + // Generate 1MB of random data + int dataSize = 1024 * 1024; + byte[] data = new byte[dataSize]; + new Random().nextBytes(data); + + BlobId blobId = BlobId.of(BUCKET_NAME, "test-bidi-zero-copy-temp-" + UUID.randomUUID()); + storage.create(BlobInfo.newBuilder(blobId).build(), data); + + try { + // Define 3 non-overlapping ranges + RangeSpec r1 = RangeSpec.of(0, 1000); + RangeSpec r2 = RangeSpec.of(50000, 250000); + RangeSpec r3 = RangeSpec.of(800000, 10000); + + ApiFuture futureSession = storage.blobReadSession(blobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start concurrent zero-copy range reads + ApiFuture f1 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r1)); + ApiFuture f2 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r2)); + ApiFuture f3 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r3)); + + // Resolve and verify Range 1 + try (DisposableByteString d1 = f1.get(10, TimeUnit.SECONDS)) { + assertThat(d1).isNotNull(); + byte[] expected = Arrays.copyOfRange(data, 0, 1000); + assertThat(d1.byteString().toByteArray()).isEqualTo(expected); + } + + // Resolve and verify Range 2 + try (DisposableByteString d2 = f2.get(10, TimeUnit.SECONDS)) { + assertThat(d2).isNotNull(); + byte[] expected = Arrays.copyOfRange(data, 50000, 50000 + 250000); + assertThat(d2.byteString().toByteArray()).isEqualTo(expected); + } + + // Resolve and verify Range 3 + try (DisposableByteString d3 = f3.get(10, TimeUnit.SECONDS)) { + assertThat(d3).isNotNull(); + byte[] expected = Arrays.copyOfRange(data, 800000, 800000 + 10000); + assertThat(d3.byteString().toByteArray()).isEqualTo(expected); + } + System.out.println("Successfully validated concurrent zero-copy range reads!"); + } + } finally { + storage.delete(blobId); + } + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java new file mode 100644 index 000000000000..4f9b995acd5b --- /dev/null +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITRcuBidiReadTest.java @@ -0,0 +1,359 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.OutOfRangeException; +import com.google.cloud.storage.AsyncSessionClosedException; +import com.google.cloud.storage.BlobAppendableUpload; +import com.google.cloud.storage.BlobAppendableUpload.AppendableUploadWriteableByteChannel; +import com.google.cloud.storage.BlobAppendableUploadConfig; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.BlobReadSession; +import com.google.cloud.storage.Storage.BlobWriteOption; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.StorageClass; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.RangeSpec; +import com.google.cloud.storage.ReadProjectionConfigs; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.ZeroCopySupport.DisposableByteString; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.annotations.LocationType; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@CrossRun( + backends = {Backend.PROD}, + transports = {Transport.GRPC}, + locations = { + LocationType.REGIONAL_RAPID + }) +public final class ITRcuBidiReadTest { + + @Inject public Storage storage; + @Inject public BucketInfo bucket; + @Inject public Backend backend; + @Inject public Transport transport; + + private static boolean initialized = false; + private static Storage staticStorage; + private static BlobId closeTestBlobId; + private static BlobId zeroCopyTestBlobId; + private static BlobId multipleRangeTestBlobId; + private static BlobId outOfRangeTestBlobId; + + private static byte[] closeTestData; + private static byte[] zeroCopyTestData; + private static byte[] multipleRangeTestData; + private static byte[] outOfRangeTestData; + + @Before + public void setUp() throws Exception { + if (initialized) { + return; + } + initialized = true; + staticStorage = storage; + + // Generate test data + closeTestData = new byte[5 * 1024 * 1024]; + new Random().nextBytes(closeTestData); + + zeroCopyTestData = new byte[1024 * 1024]; + new Random().nextBytes(zeroCopyTestData); + + multipleRangeTestData = new byte[2 * 1024 * 1024]; + new Random().nextBytes(multipleRangeTestData); + + outOfRangeTestData = new byte[100 * 1024]; + new Random().nextBytes(outOfRangeTestData); + + closeTestBlobId = BlobId.of(bucket.getName(), "test-bidi-read-close-" + UUID.randomUUID()); + zeroCopyTestBlobId = BlobId.of(bucket.getName(), "test-bidi-zero-copy-" + UUID.randomUUID()); + multipleRangeTestBlobId = BlobId.of(bucket.getName(), "test-bidi-multiple-range-" + UUID.randomUUID()); + outOfRangeTestBlobId = BlobId.of(bucket.getName(), "test-bidi-out-of-range-" + UUID.randomUUID()); + + System.out.println("Pre-creating objects for read integration tests..."); + createObjectForWarming(closeTestBlobId, closeTestData); + createObjectForWarming(zeroCopyTestBlobId, zeroCopyTestData); + createObjectForWarming(multipleRangeTestBlobId, multipleRangeTestData); + createObjectForWarming(outOfRangeTestBlobId, outOfRangeTestData); + + if (bucket.getName().contains("reg-rapid")) { + System.out.println("Regional Rapid bucket detected. Triggering Ingest-On-Read on all objects..."); + triggerIngestOnRead(closeTestBlobId); + triggerIngestOnRead(zeroCopyTestBlobId); + triggerIngestOnRead(multipleRangeTestBlobId); + triggerIngestOnRead(outOfRangeTestBlobId); + + System.out.println("Sleeping for 30 minutes to allow background uptiering to RZ..."); + Thread.sleep(30 * 60 * 1000); // 30 minutes + System.out.println("Wake up! Continuing with integration test execution."); + } + } + + private void createObjectForWarming(BlobId blobId, byte[] data) throws Exception { + StorageClass storageClass = bucket.getStorageClass(); + if (StorageClass.valueOf("RAPID").equals(storageClass)) { + System.out.println("Bucket is ZONAL_RAPID, writing via Appendable upload with RAPID storage class..."); + BlobInfo info = BlobInfo.newBuilder(blobId) + .setStorageClass(StorageClass.valueOf("RAPID")) + .build(); + BlobAppendableUploadConfig config = BlobAppendableUploadConfig.of(); + BlobAppendableUpload upload = + storage.blobAppendableUpload(info, config, BlobWriteOption.doesNotExist()); + try (AppendableUploadWriteableByteChannel channel = upload.open()) { + ByteBuffer buffer = ByteBuffer.wrap(data); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.finalizeAndClose(); + } + upload.getResult().get(10, TimeUnit.SECONDS); + } else { + System.out.println("Writing object " + blobId + " via standard create..."); + storage.create(BlobInfo.newBuilder(blobId).build(), data); + } + } + + private void triggerIngestOnRead(BlobId blobId) { + try { + ApiFuture futureSession = storage.blobReadSession(blobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + ApiFuture readFuture = session.readAs( + ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(0, 100))); + readFuture.get(10, TimeUnit.SECONDS); + } + } catch (Exception e) { + System.out.println("Warning: Ingest-on-read trigger failed for " + blobId + ": " + e.getMessage()); + } + } + + @AfterClass + public static void tearDownClass() { + if (staticStorage != null) { + System.out.println("Cleaning up pre-created integration test objects..."); + try { + if (closeTestBlobId != null) staticStorage.delete(closeTestBlobId); + if (zeroCopyTestBlobId != null) staticStorage.delete(zeroCopyTestBlobId); + if (multipleRangeTestBlobId != null) staticStorage.delete(multipleRangeTestBlobId); + if (outOfRangeTestBlobId != null) staticStorage.delete(outOfRangeTestBlobId); + } catch (Exception e) { + System.out.println("Error cleaning up integration test objects: " + e.getMessage()); + } + } + } + + @Test + public void readPostStreamClose() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: readPostStreamClose against bucket " + bucket.getName()); + + ApiFuture futureSession = storage.blobReadSession(closeTestBlobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start a future read for the entire object + ApiFuture readFuture = session.readAs(ReadProjectionConfigs.asFutureBytes()); + + // Close the session immediately while the transfer is in flight + session.close(); + + // Resolving the future should now fail since the session is closed + try { + readFuture.get(5, TimeUnit.SECONDS); + Assert.fail("Expected ExecutionException when reading after session close"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertThat(cause).isInstanceOf(StorageException.class); + assertThat(cause.getCause()).isInstanceOf(AsyncSessionClosedException.class); + System.out.println(">>> SUCCESS: readPostStreamClose verified AsyncSessionClosedException."); + } + } + } + + @Test + public void zeroCopyRangeReads() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: zeroCopyRangeReads against bucket " + bucket.getName()); + + try { + // Define 3 non-overlapping ranges + RangeSpec r1 = RangeSpec.of(0, 1000); + RangeSpec r2 = RangeSpec.of(50000, 250000); + RangeSpec r3 = RangeSpec.of(800000, 10000); + + ApiFuture futureSession = storage.blobReadSession(zeroCopyTestBlobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start concurrent zero-copy range reads + ApiFuture f1 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r1)); + ApiFuture f2 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r2)); + ApiFuture f3 = + session.readAs(ReadProjectionConfigs.asFutureByteString().withRangeSpec(r3)); + + // Resolve and verify Range 1 + try (DisposableByteString d1 = f1.get(10, TimeUnit.SECONDS)) { + assertThat(d1).isNotNull(); + byte[] expected = Arrays.copyOfRange(zeroCopyTestData, 0, 1000); + assertThat(d1.byteString().toByteArray()).isEqualTo(expected); + } + + // Resolve and verify Range 2 + try (DisposableByteString d2 = f2.get(10, TimeUnit.SECONDS)) { + assertThat(d2).isNotNull(); + byte[] expected = Arrays.copyOfRange(zeroCopyTestData, 50000, 50000 + 250000); + assertThat(d2.byteString().toByteArray()).isEqualTo(expected); + } + + // Resolve and verify Range 3 + try (DisposableByteString d3 = f3.get(10, TimeUnit.SECONDS)) { + assertThat(d3).isNotNull(); + byte[] expected = Arrays.copyOfRange(zeroCopyTestData, 800000, 800000 + 10000); + assertThat(d3.byteString().toByteArray()).isEqualTo(expected); + } + System.out.println(">>> SUCCESS: zeroCopyRangeReads concurrent offsets verified."); + } + } finally { + // Do not delete + } + } + + @Test + public void multipleRangedRead() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: multipleRangedRead against bucket " + bucket.getName()); + + try { + // Define 4 non-overlapping ranges (each 512KB) + int rangeSize = 512 * 1024; + RangeSpec r1 = RangeSpec.of(0, rangeSize); + RangeSpec r2 = RangeSpec.of(rangeSize, rangeSize); + RangeSpec r3 = RangeSpec.of(2 * rangeSize, rangeSize); + RangeSpec r4 = RangeSpec.of(3 * rangeSize, rangeSize); + + ApiFuture futureSession = storage.blobReadSession(multipleRangeTestBlobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start concurrent byte range reads + ApiFuture f1 = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(r1)); + ApiFuture f2 = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(r2)); + ApiFuture f3 = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(r3)); + ApiFuture f4 = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(r4)); + + // Resolve and verify Range 1 + byte[] b1 = f1.get(10, TimeUnit.SECONDS); + assertThat(b1).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 0, rangeSize)); + + // Resolve and verify Range 2 + byte[] b2 = f2.get(10, TimeUnit.SECONDS); + assertThat(b2).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, rangeSize, 2 * rangeSize)); + + // Resolve and verify Range 3 + byte[] b3 = f3.get(10, TimeUnit.SECONDS); + assertThat(b3).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 2 * rangeSize, 3 * rangeSize)); + + // Resolve and verify Range 4 + byte[] b4 = f4.get(10, TimeUnit.SECONDS); + assertThat(b4).isEqualTo(Arrays.copyOfRange(multipleRangeTestData, 3 * rangeSize, 4 * rangeSize)); + + System.out.println(">>> SUCCESS: multipleRangedRead concurrent offsets verified."); + } + } finally { + // Do not delete + } + } + + @Test + public void readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404"); + + String nonExistentBucketName = "java-storage-non-existent-bucket-" + UUID.randomUUID(); + BlobId blobId = BlobId.of(nonExistentBucketName, "someobject"); + + ApiFuture futureObjectReadSession = storage.blobReadSession(blobId); + + try { + futureObjectReadSession.get(10, TimeUnit.SECONDS); + Assert.fail("Expected ExecutionException when reading from non-existent bucket"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertThat(cause).isInstanceOf(StorageException.class); + StorageException se = (StorageException) cause; + assertThat(se.getCode()).isIn(Arrays.asList(404, 403)); + System.out.println(">>> SUCCESS: readFromBucketThatDoesNotExistShouldRaiseStorageExceptionWith404 verified StorageException 404 or 403."); + } + } + + @Test + public void outOfRange() throws Exception { + Assume.assumeTrue(transport == Transport.GRPC); + System.out.println(">>> START: outOfRange against bucket " + bucket.getName()); + + try { + ApiFuture futureSession = storage.blobReadSession(outOfRangeTestBlobId); + try (BlobReadSession session = futureSession.get(10, TimeUnit.SECONDS)) { + // Start a valid range read on the session first to verify it succeeds + ApiFuture fValid = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.of(0, 1000))); + byte[] bytes = fValid.get(10, TimeUnit.SECONDS); + assertThat(bytes).isEqualTo(Arrays.copyOfRange(outOfRangeTestData, 0, 1000)); + + // Start an out-of-bounds range read (offset > size) + ApiFuture fOutOfRange = + session.readAs(ReadProjectionConfigs.asFutureBytes().withRangeSpec(RangeSpec.beginAt(100 * 1024 + 1000))); + + // Verify that resolving it throws OutOfRangeException + try { + fOutOfRange.get(10, TimeUnit.SECONDS); + Assert.fail("Expected ExecutionException for out-of-bounds range read"); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + assertThat(cause).isInstanceOf(StorageException.class); + assertThat(cause.getCause()).isInstanceOf(OutOfRangeException.class); + } + System.out.println(">>> SUCCESS: outOfRange verified valid read success and subsequent out of range exception."); + } + } finally { + // Do not delete + } + } +} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java index 8aa30c50290a..814f89ba6196 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/CrossRunIntersection.java @@ -20,7 +20,6 @@ import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.it.runner.annotations.Backend; -import com.google.cloud.storage.it.runner.annotations.Colocation; import com.google.cloud.storage.it.runner.annotations.CrossRun; import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.common.base.MoreObjects; @@ -44,17 +43,14 @@ public final class CrossRunIntersection { private final @Nullable Backend backend; private final @Nullable Transport transport; private final @Nullable LocationType locationType; - private final @Nullable Colocation colocation; private CrossRunIntersection( @Nullable Backend backend, @Nullable Transport transport, - @Nullable LocationType locationType, - @Nullable Colocation colocation) { + @Nullable LocationType locationType) { this.backend = backend; this.transport = transport; this.locationType = locationType; - this.colocation = colocation; } @Nullable @@ -72,16 +68,11 @@ public LocationType getLocationType() { return locationType; } - @Nullable - public Colocation getColocation() { - return colocation; - } - public CrossRunIntersection clearBackend() { if (backend == null) { return this; } else { - return new CrossRunIntersection(null, transport, locationType, colocation); + return new CrossRunIntersection(null, transport, locationType); } } @@ -89,7 +80,7 @@ public CrossRunIntersection clearTransport() { if (transport == null) { return this; } else { - return new CrossRunIntersection(backend, null, locationType, colocation); + return new CrossRunIntersection(backend, null, locationType); } } @@ -97,15 +88,7 @@ public CrossRunIntersection clearLocationType() { if (locationType == null) { return this; } else { - return new CrossRunIntersection(backend, transport, null, colocation); - } - } - - public CrossRunIntersection clearColocation() { - if (colocation == null) { - return this; - } else { - return new CrossRunIntersection(backend, transport, locationType, null); + return new CrossRunIntersection(backend, transport, null); } } @@ -114,7 +97,7 @@ public CrossRunIntersection withBackend(Backend backend) { if (this.backend == backend) { return this; } else { - return new CrossRunIntersection(backend, transport, locationType, colocation); + return new CrossRunIntersection(backend, transport, locationType); } } @@ -123,7 +106,7 @@ public CrossRunIntersection withTransport(Transport transport) { if (this.transport == transport) { return this; } else { - return new CrossRunIntersection(backend, transport, locationType, colocation); + return new CrossRunIntersection(backend, transport, locationType); } } @@ -132,16 +115,7 @@ public CrossRunIntersection withLocationType(LocationType locationType) { if (this.locationType == locationType) { return this; } else { - return new CrossRunIntersection(backend, transport, locationType, colocation); - } - } - - public CrossRunIntersection withColocation(Colocation colocation) { - requireNonNull(colocation, "colocation must be non null"); - if (this.colocation == colocation) { - return this; - } else { - return new CrossRunIntersection(backend, transport, locationType, colocation); + return new CrossRunIntersection(backend, transport, locationType); } } @@ -170,13 +144,6 @@ public boolean anyMatch(CrossRunIntersection other) { l = l.clearLocationType(); } - if (l.colocation == null) { - r = r.clearColocation(); - } - if (r.colocation == null) { - l = l.clearColocation(); - } - return l.equals(r); } @@ -190,8 +157,7 @@ public String fmtSuiteName() { String t = transport != null ? transport.toString() : "NULL_TRANSPORT"; String b = backend != null ? backend.toString() : "NULL_BACKEND"; String lt = locationType != null ? locationType.toString() : "NULL_LOCATION"; - String c = colocation != null ? colocation.toString() : "NULL_COLOCATION"; - return String.format(Locale.US, "[%s][%s][%s][%s]", t, b, lt, c); + return String.format(Locale.US, "[%s][%s][%s]", t, b, lt); } @Override @@ -205,13 +171,12 @@ public boolean equals(Object o) { CrossRunIntersection crossRunIntersection = (CrossRunIntersection) o; return backend == crossRunIntersection.backend && transport == crossRunIntersection.transport - && locationType == crossRunIntersection.locationType - && colocation == crossRunIntersection.colocation; + && locationType == crossRunIntersection.locationType; } @Override public int hashCode() { - return Objects.hash(backend, transport, locationType, colocation); + return Objects.hash(backend, transport, locationType); } @Override @@ -220,44 +185,39 @@ public String toString() { .add("backend", backend) .add("transport", transport) .add("locationType", locationType) - .add("colocation", colocation) .toString(); } public static CrossRunIntersection of(@Nullable Backend b, @Nullable Transport t) { - return new CrossRunIntersection(b, t, null, null); + return new CrossRunIntersection(b, t, null); } public static CrossRunIntersection of( @Nullable Backend b, @Nullable Transport t, - @Nullable LocationType lt, - @Nullable Colocation c) { - return new CrossRunIntersection(b, t, lt, c); + @Nullable LocationType lt) { + return new CrossRunIntersection(b, t, lt); } public static ImmutableSet expand(CrossRun.Ignore i) { ImmutableSet backends = ImmutableSet.copyOf(i.backends()); ImmutableSet transports = ImmutableSet.copyOf(i.transports()); ImmutableSet locations = ImmutableSet.copyOf(i.locations()); - ImmutableSet colocations = ImmutableSet.copyOf(i.colocations()); - return expand(backends, transports, locations, colocations); + return expand(backends, transports, locations); } public static ImmutableSet expand(CrossRun.Exclude i) { ImmutableSet backends = ImmutableSet.copyOf(i.backends()); ImmutableSet transports = ImmutableSet.copyOf(i.transports()); ImmutableSet locations = ImmutableSet.copyOf(i.locations()); - ImmutableSet colocations = ImmutableSet.copyOf(i.colocations()); - return expand(backends, transports, locations, colocations); + return expand(backends, transports, locations); } public static ImmutableSet expand( ImmutableSet backends, ImmutableSet<@Nullable Transport> transports, - ImmutableSet<@Nullable LocationType> locations, - ImmutableSet<@Nullable Colocation> colocations) { - if (backends.isEmpty() && transports.isEmpty() && locations.isEmpty() && colocations.isEmpty()) { + ImmutableSet<@Nullable LocationType> locations) { + if (backends.isEmpty() && transports.isEmpty() && locations.isEmpty()) { return ImmutableSet.of(); } @@ -267,8 +227,6 @@ public static ImmutableSet expand( transports.isEmpty() ? Collections.singleton((Transport) null) : transports; Set<@Nullable LocationType> lSet = locations.isEmpty() ? Collections.singleton((LocationType) null) : locations; - Set<@Nullable Colocation> cSet = - colocations.isEmpty() ? Collections.singleton((Colocation) null) : colocations; return bSet.stream() .flatMap( @@ -277,16 +235,12 @@ public static ImmutableSet expand( .flatMap( t -> lSet.stream() - .flatMap( - l -> - cSet.stream() - .map(c -> new CrossRunIntersection(b, t, l, c))))) + .map(l -> new CrossRunIntersection(b, t, l)))) .filter( i -> !(i.backend == null && i.transport == null - && i.locationType == null - && i.colocation == null)) + && i.locationType == null)) .collect(ImmutableSet.toImmutableSet()); } } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java index 0cb06fa66414..4e04768d58ab 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/StorageITRunner.java @@ -23,7 +23,6 @@ import com.google.cloud.storage.it.runner.annotations.Parameterized; import com.google.cloud.storage.it.runner.annotations.Parameterized.Parameter; import com.google.cloud.storage.it.runner.annotations.Parameterized.ParametersProvider; -import com.google.cloud.storage.it.runner.annotations.Colocation; import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.annotations.SingleBackend; import com.google.cloud.storage.it.runner.registry.Registry; @@ -171,13 +170,10 @@ private static List computeRunners(Class klass, Registry registry) .flatMap( t -> ImmutableSet.copyOf(crossRun.locations()).stream() - .flatMap( + .map( l -> - ImmutableSet.copyOf(crossRun.colocations()).stream() - .map( - c -> - CrossRunIntersection.of( - b, t, l, c))))) + CrossRunIntersection.of( + b, t, l)))) .flatMap( c -> { TestInitializer ti = registry.newTestInitializerForCell(c); @@ -200,17 +196,12 @@ private static List computeRunners(Class klass, Registry registry) Backend backend = singleBackend.value(); boolean isDefault = singleBackend.locations().length == 1 - && singleBackend.locations()[0] == LocationType.REGIONAL_STANDARD - && singleBackend.colocations().length == 1 - && singleBackend.colocations()[0] == Colocation.CO_LOCATED; + && singleBackend.locations()[0] == LocationType.REGIONAL_STANDARD; return SneakyException.unwrap( () -> ImmutableSet.copyOf(singleBackend.locations()).stream() - .flatMap( - l -> - ImmutableSet.copyOf(singleBackend.colocations()).stream() - .map(c -> CrossRunIntersection.of(backend, null, l, c))) + .map(l -> CrossRunIntersection.of(backend, null, l)) .flatMap( c -> { TestInitializer ti = registry.newTestInitializerForCell(c); diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java deleted file mode 100644 index b3982eb2bc76..000000000000 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/Colocation.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.storage.it.runner.annotations; - -public enum Colocation { - CO_LOCATED, - NON_CO_LOCATED -} diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java index fe6c31b0b757..9926ce4ac0c6 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/CrossRun.java @@ -41,8 +41,6 @@ LocationType[] locations() default {LocationType.REGIONAL_STANDARD}; - Colocation[] colocations() default {Colocation.CO_LOCATED}; - /** * Exclude a method from being included in the generated test suite if it's backend and transport * match with those defined. When matching, if the empty set is defined as a value it will be @@ -59,8 +57,6 @@ TransportCompatibility.Transport[] transports() default {}; LocationType[] locations() default {}; - - Colocation[] colocations() default {}; } /** @@ -79,8 +75,6 @@ TransportCompatibility.Transport[] transports() default {}; LocationType[] locations() default {}; - - Colocation[] colocations() default {}; } /** diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java index 09fc2d6a17be..348620aa6a67 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/annotations/SingleBackend.java @@ -37,5 +37,4 @@ LocationType[] locations() default {LocationType.REGIONAL_STANDARD}; - Colocation[] colocations() default {Colocation.CO_LOCATED}; } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java index f82357b533aa..ca1eaa25287e 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BackendResources.java @@ -38,7 +38,6 @@ import com.google.cloud.storage.it.runner.CrossRunIntersection; import com.google.cloud.storage.it.runner.annotations.Backend; import com.google.cloud.storage.it.runner.annotations.BucketType; -import com.google.cloud.storage.it.runner.annotations.Colocation; import com.google.cloud.storage.it.runner.annotations.LocationType; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -61,16 +60,25 @@ final class BackendResources implements ManagedLifecycle { private final Backend backend; private final ProtectedBucketNames protectedBucketNames; private final ConcurrentMap dynamicBuckets; + private final TestRunScopedInstance storageJson; + private final TestRunScopedInstance storageGrpc; + private final TestRunScopedInstance ctrl; private final ImmutableList> registryEntries; private BackendResources( Backend backend, ProtectedBucketNames protectedBucketNames, ConcurrentMap dynamicBuckets, + TestRunScopedInstance storageJson, + TestRunScopedInstance storageGrpc, + TestRunScopedInstance ctrl, ImmutableList> registryEntries) { this.backend = backend; this.protectedBucketNames = protectedBucketNames; this.dynamicBuckets = dynamicBuckets; + this.storageJson = storageJson; + this.storageGrpc = storageGrpc; + this.ctrl = ctrl; this.registryEntries = registryEntries; } @@ -98,6 +106,14 @@ public String toString() { return MoreObjects.toStringHelper(this).add("backend", backend).toString(); } + public Storage getStorage(Transport transport) { + return transport == Transport.GRPC ? storageGrpc.get().getStorage() : storageJson.get().getStorage(); + } + + public StorageControlClient getStorageControlClient() { + return ctrl.get().getCtrl(); + } + @SuppressWarnings("SwitchStatementWithTooFewBranches") static BackendResources of( Backend backend, @@ -128,7 +144,10 @@ static BackendResources of( break; default: // PROD, java8 doesn't have exhaustive checking for enum switch // Register the exporters with OpenTelemetry - optionsBuilder = StorageOptions.http().setOpenTelemetry(otelSdk.get().get()); + optionsBuilder = + StorageOptions.http() + .setProjectId(getPreprodProjectId()) + .setOpenTelemetry(otelSdk.get().get()); break; } HttpStorageOptions built = optionsBuilder.build(); @@ -159,7 +178,10 @@ static BackendResources of( break; default: // PROD, java8 doesn't have exhaustive checking for enum switch // Register the exporters with OpenTelemetry - optionsBuilder = StorageOptions.grpc().setOpenTelemetry(otelSdk.get().get()); + optionsBuilder = + StorageOptions.grpc() + .setProjectId(getPreprodProjectId()) + .setOpenTelemetry(otelSdk.get().get()); break; } GrpcStorageOptions built = @@ -239,6 +261,7 @@ static BackendResources of( String.format(Locale.US, "java-storage-grpc-rp-%s", UUID.randomUUID()); protectedBucketNames.add(bucketName); return new BucketInfoShim( + backend, BucketInfo.newBuilder(bucketName) .setLocation(zone.get().get().getRegion()) .setRequesterPays(true) @@ -254,6 +277,7 @@ static BackendResources of( String.format(Locale.US, "java-storage-grpc-v-%s", UUID.randomUUID()); protectedBucketNames.add(bucketName); return new BucketInfoShim( + backend, BucketInfo.newBuilder(bucketName) .setLocation(zone.get().get().getRegion()) .setVersioningEnabled(true) @@ -269,6 +293,7 @@ static BackendResources of( String.format(Locale.US, "java-storage-grpc-hns-%s", UUID.randomUUID()); protectedBucketNames.add(bucketName); return new BucketInfoShim( + backend, BucketInfo.newBuilder(bucketName) .setLocation(zone.get().get().getRegion()) .setHierarchicalNamespace( @@ -289,6 +314,7 @@ static BackendResources of( String.format(Locale.US, "java-storage-grpc-rapid-%s", UUID.randomUUID()); protectedBucketNames.add(bucketName); return new BucketInfoShim( + backend, BucketInfo.newBuilder(bucketName) .setLocation("us-central1") .setCustomPlacementConfig( @@ -319,8 +345,7 @@ static BackendResources of( CrossRunIntersection.of( backend, null, - LocationType.REGIONAL_STANDARD, - Colocation.CO_LOCATED)))); + LocationType.REGIONAL_STANDARD)))); TestRunScopedInstance objectsFixtureRp = TestRunScopedInstance.of( "fixture/OBJECTS/[" + backend.name() + "]/REQUESTER_PAYS", @@ -341,6 +366,9 @@ static BackendResources of( backend, protectedBucketNames, dynamicBuckets, + storageJson, + storageGrpc, + ctrl, ImmutableList.of( RegistryEntry.of( 40, Storage.class, storageJson, transportAndBackendAre(Transport.HTTP, backend)), @@ -389,11 +417,9 @@ static BackendResources of( private static final class BucketKey { private final LocationType locationType; - private final Colocation colocation; - private BucketKey(LocationType locationType, Colocation colocation) { + private BucketKey(LocationType locationType) { this.locationType = locationType; - this.colocation = colocation; } @Override @@ -405,19 +431,18 @@ public boolean equals(Object o) { return false; } BucketKey bucketKey = (BucketKey) o; - return locationType == bucketKey.locationType && colocation == bucketKey.colocation; + return locationType == bucketKey.locationType; } @Override public int hashCode() { - return java.util.Objects.hash(locationType, colocation); + return java.util.Objects.hash(locationType); } @Override public String toString() { return MoreObjects.toStringHelper(this) .add("locationType", locationType) - .add("colocation", colocation) .toString(); } } @@ -449,16 +474,12 @@ private DynamicBucketLifecycle( @Override public BucketInfo resolve(FrameworkField ff, CrossRunIntersection crossRunIntersection) { LocationType lt = crossRunIntersection.getLocationType(); - Colocation col = crossRunIntersection.getColocation(); if (lt == null) { lt = LocationType.REGIONAL_STANDARD; } - if (col == null) { - col = Colocation.CO_LOCATED; - } - BucketKey key = new BucketKey(lt, col); + BucketKey key = new BucketKey(lt); BucketInfoShim shim = dynamicBuckets.computeIfAbsent(key, this::createBucketShim); return (BucketInfo) shim.get(); } @@ -468,22 +489,20 @@ private BucketInfoShim createBucketShim(BucketKey key) { String region = z.getRegion(); String zoneName = z.getZone(); - String rAlt; - String vAlt; - if ("us-east1".equals(region)) { - rAlt = "us-central1"; - vAlt = "us-central1-a"; - } else { - rAlt = "us-east1"; - vAlt = "us-east1-b"; - } - String targetRegion = region; String targetZone = zoneName; - if (key.colocation == Colocation.NON_CO_LOCATED) { - targetRegion = rAlt; - targetZone = vAlt; + Storage storageClientToUse = storageJson.get().getStorage(); + StorageControlClient controlClientToUse = ctrl.get().getCtrl(); + + if (key.locationType == LocationType.REGIONAL_RAPID) { + targetRegion = "us-central1"; + targetZone = "us-central1-a"; + if (backend == Backend.PROD) { + BackendResources preprod = Registry.getInstance().getPreProdBackendResources(); + storageClientToUse = preprod.getStorage(Transport.GRPC); + controlClientToUse = preprod.getStorageControlClient(); + } } BucketInfo.Builder builder; @@ -540,11 +559,12 @@ private BucketInfoShim createBucketShim(BucketKey key) { BucketInfoShim shim = new BucketInfoShim( + backend, builder.build(), key.locationType, targetZone, - storageJson.get().getStorage(), - ctrl.get().getCtrl()); + storageClientToUse, + controlClientToUse); shim.start(); return shim; diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java index 8a0e3b71f7d2..69be2b41df30 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/BucketInfoShim.java @@ -22,6 +22,7 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.it.BucketCleaner; +import com.google.cloud.storage.it.runner.annotations.Backend; import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.protobuf.Duration; import com.google.storage.control.v2.BucketName; @@ -32,6 +33,7 @@ /** Shim to lift a BucketInfo to be a managed bucket instance */ final class BucketInfoShim implements ManagedLifecycle { + private final Backend backend; private final BucketInfo bucketInfo; private final LocationType locationType; private final String targetZone; @@ -41,15 +43,21 @@ final class BucketInfoShim implements ManagedLifecycle { private BucketInfo createdBucket; BucketInfoShim(BucketInfo bucketInfo, Storage s, StorageControlClient ctrl) { - this(bucketInfo, LocationType.REGIONAL_STANDARD, null, s, ctrl); + this(Backend.PROD, bucketInfo, LocationType.REGIONAL_STANDARD, null, s, ctrl); + } + + BucketInfoShim(Backend backend, BucketInfo bucketInfo, Storage s, StorageControlClient ctrl) { + this(backend, bucketInfo, LocationType.REGIONAL_STANDARD, null, s, ctrl); } BucketInfoShim( + Backend backend, BucketInfo bucketInfo, LocationType locationType, String targetZone, Storage s, StorageControlClient ctrl) { + this.backend = backend; this.bucketInfo = bucketInfo; this.locationType = locationType; this.targetZone = targetZone; @@ -63,13 +71,22 @@ public BucketInfo getBucketInfo() { @Override public Object get() { - return bucketInfo; + return createdBucket != null ? createdBucket : bucketInfo; } @Override public void start() { try { + if (locationType == LocationType.REGIONAL_RAPID && backend != Backend.TEST_BENCH) { + System.out.println(">>> REUSING static pre-created RCU bucket java-storage-reg-rapid-preprod-3fe2bb58 for REGIONAL_RAPID test!"); + createdBucket = BucketInfo.newBuilder("java-storage-reg-rapid-preprod-3fe2bb58") + .setLocation("US-CENTRAL1") + .build(); + return; + } + System.out.println("Starting resource creation for LocationType: " + locationType + " in zone: " + targetZone); createdBucket = s.create(bucketInfo).asBucketInfo(); + System.out.println("Successfully created bucket: " + createdBucket.getName() + " (Location: " + createdBucket.getLocation() + ")"); if (locationType == LocationType.REGIONAL_RAPID) { if (ctrl == null) { throw new IllegalStateException( @@ -89,17 +106,29 @@ public void start() { .setTtl(Duration.newBuilder().setSeconds(86400).build()) // 24 hours .build(); try { + System.out.println("Submitting CreateRapidCache LRO for bucket: " + createdBucket.getName() + " in zone: " + targetZone); ctrl.createRapidCacheAsync(BucketName.format("_", createdBucket.getName()), rapidCache) - .get(); + .get(30, java.util.concurrent.TimeUnit.SECONDS); + System.out.println("Successfully created Rapid Cache in zone: " + targetZone); + } catch (java.util.concurrent.TimeoutException te) { + System.out.println("WARNING: CreateRapidCache LRO timed out after 30s. Skipping test."); + assumeTrue( + "Skipping test because Rapid Cache creation LRO timed out (30s) in zone: " + targetZone, + false); } catch (Exception e) { - throw new RuntimeException("Failed to create Rapid Cache: " + e.getMessage(), e); + System.out.println("WARNING: CreateRapidCache LRO failed: " + e.getMessage() + ". Skipping test."); + assumeTrue( + "Skipping test due to failure during Rapid Cache creation: " + e.getMessage(), + false); } } } catch (StorageException se) { String msg = se.getMessage().toLowerCase(Locale.US); + System.out.println("StorageException caught during resource creation: " + msg); if (se.getCode() == 400 && (msg.contains("not a valid zone in location")) || msg.contains("custom placement config") || msg.contains("zonal")) { + System.out.println("Skipping test: setup unavailable in current zone."); assumeTrue( "Skipping test due to bucket setup unavailable in current zone. (" + msg + ")", false); } diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java index 0db457ae507f..8bdb507f71c3 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/it/runner/registry/Registry.java @@ -16,8 +16,10 @@ package com.google.cloud.storage.it.runner.registry; +import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.it.runner.annotations.LocationType; import com.google.cloud.storage.it.runner.CrossRunIntersection; import com.google.cloud.storage.it.runner.TestInitializer; import com.google.cloud.storage.it.runner.annotations.Backend; @@ -106,7 +108,6 @@ public final class Registry extends RunListener { registryEntry(3, Backend.class, CrossRunIntersection::getBackend), registryEntry(4, Transport.class, CrossRunIntersection::getTransport)) .addAll(prodBackendResources.getRegistryEntries()) - .addAll(preProdBackendResources.getRegistryEntries()) .addAll(testBenchBackendResource.getRegistryEntries()) .build(); @@ -157,6 +158,10 @@ TestBench testBench() { return testBench.get(); } + BackendResources getPreProdBackendResources() { + return preProdBackendResources; + } + @Nullable public Description getCurrentTest() { return currentTest.get().desc; @@ -224,6 +229,12 @@ public Object resolve(FrameworkField ff, CrossRunIntersection crossRunIntersecti } else { finalCrossRunIntersection = crossRunIntersection; } + if (ff.getType() == Storage.class + && finalCrossRunIntersection.getLocationType() == LocationType.REGIONAL_RAPID + && finalCrossRunIntersection.getBackend() == Backend.PROD) { + return preProdBackendResources.getStorage(finalCrossRunIntersection.getTransport()); + } + Optional> first = entries.stream() .filter(re -> re.getPredicate().test(ff, finalCrossRunIntersection))