diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 459487d38827..c8316751ebd3 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:ae72564aa9c368b9ccd96f4af21f87889fd83b9e60635b80844deb5a2ccd08aa + digest: sha256:142286d973c7b6d58186070f203b50058a20a7d7b42147996db24921a18da1b0 diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in index 6aa7cf2b55ee..9cc4a1baae11 100644 --- a/.kokoro/requirements.in +++ b/.kokoro/requirements.in @@ -1,7 +1,8 @@ -gcp-docuploader==0.6.3 +gcp-docuploader==0.6.4 google-crc32c==1.3.0 googleapis-common-protos==1.56.3 -gcp-releasetool==1.8.7 +gcp-releasetool==1.9.1 +cryptography==38.0.3 cachetools==4.2.4 cffi==1.15.1 jeepney==0.7.1 @@ -29,5 +30,5 @@ zipp==3.6.0 rsa==4.9 six==1.16.0 attrs==22.1.0 -google-auth==2.11.0 +google-auth==2.14.0 idna==3.4 \ No newline at end of file diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 02ae42bb4039..2dce197bf45a 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile --allow-unsafe --generate-hashes requirements.in +# pip-compile --generate-hashes requirements.in # attrs==22.1.0 \ --hash=sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6 \ @@ -104,43 +104,44 @@ colorlog==6.7.0 \ --hash=sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662 \ --hash=sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5 # via gcp-docuploader -cryptography==38.0.1 \ - --hash=sha256:0297ffc478bdd237f5ca3a7dc96fc0d315670bfa099c04dc3a4a2172008a405a \ - --hash=sha256:10d1f29d6292fc95acb597bacefd5b9e812099d75a6469004fd38ba5471a977f \ - --hash=sha256:16fa61e7481f4b77ef53991075de29fc5bacb582a1244046d2e8b4bb72ef66d0 \ - --hash=sha256:194044c6b89a2f9f169df475cc167f6157eb9151cc69af8a2a163481d45cc407 \ - --hash=sha256:1db3d807a14931fa317f96435695d9ec386be7b84b618cc61cfa5d08b0ae33d7 \ - --hash=sha256:3261725c0ef84e7592597606f6583385fed2a5ec3909f43bc475ade9729a41d6 \ - --hash=sha256:3b72c360427889b40f36dc214630e688c2fe03e16c162ef0aa41da7ab1455153 \ - --hash=sha256:3e3a2599e640927089f932295a9a247fc40a5bdf69b0484532f530471a382750 \ - --hash=sha256:3fc26e22840b77326a764ceb5f02ca2d342305fba08f002a8c1f139540cdfaad \ - --hash=sha256:5067ee7f2bce36b11d0e334abcd1ccf8c541fc0bbdaf57cdd511fdee53e879b6 \ - --hash=sha256:52e7bee800ec869b4031093875279f1ff2ed12c1e2f74923e8f49c916afd1d3b \ - --hash=sha256:64760ba5331e3f1794d0bcaabc0d0c39e8c60bf67d09c93dc0e54189dfd7cfe5 \ - --hash=sha256:765fa194a0f3372d83005ab83ab35d7c5526c4e22951e46059b8ac678b44fa5a \ - --hash=sha256:79473cf8a5cbc471979bd9378c9f425384980fcf2ab6534b18ed7d0d9843987d \ - --hash=sha256:896dd3a66959d3a5ddcfc140a53391f69ff1e8f25d93f0e2e7830c6de90ceb9d \ - --hash=sha256:89ed49784ba88c221756ff4d4755dbc03b3c8d2c5103f6d6b4f83a0fb1e85294 \ - --hash=sha256:ac7e48f7e7261207d750fa7e55eac2d45f720027d5703cd9007e9b37bbb59ac0 \ - --hash=sha256:ad7353f6ddf285aeadfaf79e5a6829110106ff8189391704c1d8801aa0bae45a \ - --hash=sha256:b0163a849b6f315bf52815e238bc2b2346604413fa7c1601eea84bcddb5fb9ac \ - --hash=sha256:b6c9b706316d7b5a137c35e14f4103e2115b088c412140fdbd5f87c73284df61 \ - --hash=sha256:c2e5856248a416767322c8668ef1845ad46ee62629266f84a8f007a317141013 \ - --hash=sha256:ca9f6784ea96b55ff41708b92c3f6aeaebde4c560308e5fbbd3173fbc466e94e \ - --hash=sha256:d1a5bd52d684e49a36582193e0b89ff267704cd4025abefb9e26803adeb3e5fb \ - --hash=sha256:d3971e2749a723e9084dd507584e2a2761f78ad2c638aa31e80bc7a15c9db4f9 \ - --hash=sha256:d4ef6cc305394ed669d4d9eebf10d3a101059bdcf2669c366ec1d14e4fb227bd \ - --hash=sha256:d9e69ae01f99abe6ad646947bba8941e896cb3aa805be2597a0400e0764b5818 +cryptography==38.0.3 \ + --hash=sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d \ + --hash=sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd \ + --hash=sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146 \ + --hash=sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7 \ + --hash=sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436 \ + --hash=sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0 \ + --hash=sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828 \ + --hash=sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b \ + --hash=sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55 \ + --hash=sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36 \ + --hash=sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50 \ + --hash=sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2 \ + --hash=sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a \ + --hash=sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8 \ + --hash=sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0 \ + --hash=sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548 \ + --hash=sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320 \ + --hash=sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748 \ + --hash=sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249 \ + --hash=sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959 \ + --hash=sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f \ + --hash=sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0 \ + --hash=sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd \ + --hash=sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220 \ + --hash=sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c \ + --hash=sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722 # via + # -r requirements.in # gcp-releasetool # secretstorage -gcp-docuploader==0.6.3 \ - --hash=sha256:ba8c9d76b3bbac54b0311c503a373b00edc2dc02d6d54ea9507045adb8e870f7 \ - --hash=sha256:c0f5aaa82ce1854a386197e4e359b120ad6d4e57ae2c812fce42219a3288026b +gcp-docuploader==0.6.4 \ + --hash=sha256:01486419e24633af78fd0167db74a2763974765ee8078ca6eb6964d0ebd388af \ + --hash=sha256:70861190c123d907b3b067da896265ead2eeb9263969d6955c9e0bb091b5ccbf # via -r requirements.in -gcp-releasetool==1.8.7 \ - --hash=sha256:3d2a67c9db39322194afb3b427e9cb0476ce8f2a04033695f0aeb63979fc2b37 \ - --hash=sha256:5e4d28f66e90780d77f3ecf1e9155852b0c3b13cbccb08ab07e66b2357c8da8d +gcp-releasetool==1.9.1 \ + --hash=sha256:952f4055d5d986b070ae2a71c4410b250000f9cc5a1e26398fcd55a5bbc5a15f \ + --hash=sha256:d0d3c814a97c1a237517e837d8cfa668ced8df4b882452578ecef4a4e79c583b # via -r requirements.in google-api-core==2.8.2 \ --hash=sha256:06f7244c640322b508b125903bb5701bebabce8832f85aba9335ec00b3d02edc \ @@ -149,9 +150,9 @@ google-api-core==2.8.2 \ # -r requirements.in # google-cloud-core # google-cloud-storage -google-auth==2.11.0 \ - --hash=sha256:be62acaae38d0049c21ca90f27a23847245c9f161ff54ede13af2cb6afecbac9 \ - --hash=sha256:ed65ecf9f681832298e29328e1ef0a3676e3732b2e56f41532d45f70a22de0fb +google-auth==2.14.0 \ + --hash=sha256:1ad5b0e6eba5f69645971abb3d2c197537d5914070a8c6d30299dfdb07c5c700 \ + --hash=sha256:cf24817855d874ede2efd071aa22125445f555de1685b739a9782fcf408c2a3d # via # -r requirements.in # gcp-releasetool diff --git a/.readme-partials.yml b/.readme-partials.yml index 0cf67de41ec8..db6f2b40d434 100644 --- a/.readme-partials.yml +++ b/.readme-partials.yml @@ -1,7 +1,7 @@ custom_content: | ## About Cloud Bigtable - [Cloud Bigtable][cloud-bigtable] is Google's NoSQL Big Data database service. It's + [Cloud Bigtable][product-docs] is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f28f8bf6ab0..31b4027d0540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [2.15.1](https://github.com/googleapis/java-bigtable/compare/v2.15.0...v2.15.1) (2022-11-08) + + +### Bug Fixes + +* Call record attempt compeletion on permanent failures ([#1502](https://github.com/googleapis/java-bigtable/issues/1502)) ([f409c47](https://github.com/googleapis/java-bigtable/commit/f409c4769b526b38590671178c007a9c57f9c182)) +* Fix the connectivity error count caculation ([#1401](https://github.com/googleapis/java-bigtable/issues/1401)) ([1f8cfd7](https://github.com/googleapis/java-bigtable/commit/1f8cfd722c749a8d3e30572fbdda6a2e3594d7a6)) +* Only record retry count when it's > 0 ([#1488](https://github.com/googleapis/java-bigtable/issues/1488)) ([445a667](https://github.com/googleapis/java-bigtable/commit/445a66737b45835444b26261b6f017f7fc278f45)) + + +### Dependencies + +* Update dependency com.google.cloud:google-cloud-shared-dependencies to v3.0.6 ([#1501](https://github.com/googleapis/java-bigtable/issues/1501)) ([8f61c64](https://github.com/googleapis/java-bigtable/commit/8f61c64aba815a942daae17ccb024d325c03df8a)) +* Update dependency org.graalvm.buildtools:junit-platform-native to v0.9.17 ([#1495](https://github.com/googleapis/java-bigtable/issues/1495)) ([1b7c21a](https://github.com/googleapis/java-bigtable/commit/1b7c21abc793db0e4706f119e934eecdcfce9994)) +* Update dependency org.graalvm.buildtools:native-maven-plugin to v0.9.17 ([#1496](https://github.com/googleapis/java-bigtable/issues/1496)) ([74779e3](https://github.com/googleapis/java-bigtable/commit/74779e3621c9893bf3a84dc68ddcb624785c8d71)) + ## [2.15.0](https://github.com/googleapis/java-bigtable/compare/v2.14.1...v2.15.0) (2022-10-26) diff --git a/README.md b/README.md index de9341d00adf..3bdaf2e336da 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - 26.1.3 + 26.1.4 pom import @@ -41,7 +41,7 @@ If you are using Maven without BOM, add this to your dependencies: com.google.cloud google-cloud-bigtable - 2.14.1 + 2.15.0 ``` @@ -49,20 +49,20 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.1.3') +implementation platform('com.google.cloud:libraries-bom:26.1.4') implementation 'com.google.cloud:google-cloud-bigtable' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigtable:2.14.1' +implementation 'com.google.cloud:google-cloud-bigtable:2.15.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.14.1" +libraryDependencies += "com.google.cloud" % "google-cloud-bigtable" % "2.15.0" ``` ## Authentication @@ -99,7 +99,7 @@ use this Cloud Bigtable Client Library. ## About Cloud Bigtable -[Cloud Bigtable][cloud-bigtable] is Google's NoSQL Big Data database service. It's +[Cloud Bigtable][product-docs] is Google's NoSQL Big Data database service. It's the same database that powers many core Google services, including Search, Analytics, Maps, and Gmail. diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml index 3d0667837e05..f7180d5bb2d4 100644 --- a/google-cloud-bigtable-bom/pom.xml +++ b/google-cloud-bigtable-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-bigtable-bom - 2.15.0 + 2.15.1 pom com.google.cloud @@ -63,42 +63,42 @@ com.google.cloud google-cloud-bigtable - 2.15.0 + 2.15.1 com.google.cloud google-cloud-bigtable-emulator - 0.152.0 + 0.152.1 com.google.cloud google-cloud-bigtable-emulator-core - 0.152.0 + 0.152.1 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.15.0 + 2.15.1 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.15.0 + 2.15.1 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.15.0 + 2.15.1 com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.15.0 + 2.15.1 com.google.cloud google-cloud-bigtable-stats - 2.15.0 + 2.15.1 diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml index ef43d9f78c99..341e7f5d41e0 100644 --- a/google-cloud-bigtable-deps-bom/pom.xml +++ b/google-cloud-bigtable-deps-bom/pom.xml @@ -13,7 +13,7 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.15.0 + 2.15.1 pom @@ -66,7 +66,7 @@ com.google.cloud google-cloud-shared-dependencies - 3.0.5 + 3.0.6 pom import diff --git a/google-cloud-bigtable-emulator-core/pom.xml b/google-cloud-bigtable-emulator-core/pom.xml index 0c6352386655..ba5c62d8afcb 100644 --- a/google-cloud-bigtable-emulator-core/pom.xml +++ b/google-cloud-bigtable-emulator-core/pom.xml @@ -7,11 +7,11 @@ google-cloud-bigtable-parent com.google.cloud - 2.15.0 + 2.15.1 google-cloud-bigtable-emulator-core - 0.152.0 + 0.152.1 A Java wrapper for the Cloud Bigtable emulator. diff --git a/google-cloud-bigtable-emulator/pom.xml b/google-cloud-bigtable-emulator/pom.xml index b608c31a99b6..5cde249c7bf8 100644 --- a/google-cloud-bigtable-emulator/pom.xml +++ b/google-cloud-bigtable-emulator/pom.xml @@ -5,7 +5,7 @@ 4.0.0 google-cloud-bigtable-emulator - 0.152.0 + 0.152.1 Google Cloud Java - Bigtable Emulator https://github.com/googleapis/java-bigtable @@ -14,7 +14,7 @@ com.google.cloud google-cloud-bigtable-parent - 2.15.0 + 2.15.1 scm:git:git@github.com:googleapis/java-bigtable.git @@ -81,14 +81,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.15.0 + 2.15.1 pom import com.google.cloud google-cloud-bigtable-bom - 2.15.0 + 2.15.1 pom import @@ -99,7 +99,7 @@ com.google.cloud google-cloud-bigtable-emulator-core - 0.152.0 + 0.152.1 diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml index a52b749deb52..117f121d0dd1 100644 --- a/google-cloud-bigtable-stats/pom.xml +++ b/google-cloud-bigtable-stats/pom.xml @@ -5,7 +5,7 @@ com.google.cloud google-cloud-bigtable-parent - 2.15.0 + 2.15.1 4.0.0 @@ -13,7 +13,7 @@ through Stackdriver. Built-in metrics will be implemented with shaded OpenCensus so it won't interfere with customer's application metrics. --> google-cloud-bigtable-stats - 2.15.0 + 2.15.1 Experimental project to shade OpenCensus dependencies. @@ -21,7 +21,7 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.15.0 + 2.15.1 pom import diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml index c5baa2ca44d7..1440a33e12ab 100644 --- a/google-cloud-bigtable/pom.xml +++ b/google-cloud-bigtable/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-bigtable - 2.15.0 + 2.15.1 jar Google Cloud Bigtable https://github.com/googleapis/java-bigtable @@ -12,11 +12,11 @@ com.google.cloud google-cloud-bigtable-parent - 2.15.0 + 2.15.1 - 2.15.0 + 2.15.1 google-cloud-bigtable @@ -43,14 +43,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.15.0 + 2.15.1 pom import com.google.cloud google-cloud-bigtable-bom - 2.15.0 + 2.15.1 pom import diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java index 8fdc3f3a974e..d321a88db241 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/Version.java @@ -20,6 +20,6 @@ @InternalApi("For internal use only") public final class Version { // {x-version-update-start:google-cloud-bigtable:current} - public static String VERSION = "2.15.0"; + public static String VERSION = "2.15.1"; // {x-version-update-end} } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/RowMergerUtil.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/RowMergerUtil.java new file mode 100644 index 000000000000..9fbc356d5398 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/RowMergerUtil.java @@ -0,0 +1,55 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.internal; + +import com.google.api.core.InternalApi; +import com.google.bigtable.v2.ReadRowsResponse; +import com.google.cloud.bigtable.data.v2.models.DefaultRowAdapter; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowAdapter.RowBuilder; +import com.google.cloud.bigtable.data.v2.stub.readrows.RowMerger; +import java.util.ArrayList; +import java.util.List; + +@InternalApi("For internal google use only") +public class RowMergerUtil implements AutoCloseable { + private final RowMerger merger; + + public RowMergerUtil() { + RowBuilder rowBuilder = new DefaultRowAdapter().createRowBuilder(); + merger = new RowMerger<>(rowBuilder); + } + + @Override + public void close() { + if (merger.hasPartialFrame()) { + throw new IllegalStateException("Tried to close merger with unmerged partial data"); + } + } + + public List parseReadRowsResponses(Iterable responses) { + List rows = new ArrayList<>(); + + for (ReadRowsResponse response : responses) { + merger.push(response); + while (merger.hasFullFrame()) { + rows.add(merger.pop()); + } + } + + return rows; + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/DefaultRowAdapter.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/DefaultRowAdapter.java index 279ebf640771..2ff139081c79 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/DefaultRowAdapter.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/models/DefaultRowAdapter.java @@ -75,7 +75,8 @@ public Row createRowFromProto(com.google.bigtable.v2.Row row) { return builder.finishRow(); } - /** {@inheritDoc} */ + /** Internal implementation detail for {@link DefaultRowAdapter}. */ + @InternalApi() public class DefaultRowBuilder implements RowBuilder { private ByteString currentKey; private TreeMap> cellsByFamily; diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index 5ec4c726e4ee..17c968c60f55 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -21,12 +21,9 @@ import com.google.api.gax.rpc.ResponseObserver; import com.google.api.gax.rpc.ServerStreamingCallable; import com.google.api.gax.rpc.StreamController; -import com.google.bigtable.v2.ResponseParams; import com.google.cloud.bigtable.data.v2.stub.SafeResponseObserver; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; -import com.google.protobuf.InvalidProtocolBufferException; -import io.grpc.Metadata; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; @@ -101,65 +98,13 @@ protected void onResponseImpl(ResponseT response) { @Override protected void onErrorImpl(Throwable t) { - // server-timing metric will be added through GrpcResponseMetadata#onHeaders(Metadata), - // so it's not checking trailing metadata here. - Metadata metadata = responseMetadata.getMetadata(); - Long latency = Util.getGfeLatency(metadata); - tracer.recordGfeMetadata(latency, t); - try { - // Check both headers and trailers because in different environments the metadata - // could be returned in headers or trailers - if (metadata != null) { - byte[] trailers = metadata.get(Util.METADATA_KEY); - if (trailers == null) { - Metadata trailingMetadata = responseMetadata.getTrailingMetadata(); - if (trailingMetadata != null) { - trailers = trailingMetadata.get(Util.METADATA_KEY); - } - } - // If the response is terminated abnormally and we didn't get location information in - // trailers or headers, skip setting the locations - if (trailers != null) { - ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); - tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); - } - } - } catch (InvalidProtocolBufferException e) { - t.addSuppressed(t); - } - + Util.recordMetricsFromMetadata(responseMetadata, tracer, t); outerObserver.onError(t); } @Override protected void onCompleteImpl() { - Metadata metadata = responseMetadata.getMetadata(); - Long latency = Util.getGfeLatency(metadata); - tracer.recordGfeMetadata(latency, null); - try { - // Check both headers and trailers because in different environments the metadata - // could be returned in headers or trailers - if (metadata != null) { - byte[] trailers = metadata.get(Util.METADATA_KEY); - if (trailers == null) { - Metadata trailingMetadata = responseMetadata.getTrailingMetadata(); - if (trailingMetadata != null) { - trailers = trailingMetadata.get(Util.METADATA_KEY); - } - } - // If the response is terminated abnormally and we didn't get location information in - // trailers or headers, skip setting the locations - if (trailers != null) { - ResponseParams decodedTrailers = ResponseParams.parseFrom(trailers); - tracer.setLocations(decodedTrailers.getZoneId(), decodedTrailers.getClusterId()); - } - } - } catch (InvalidProtocolBufferException e) { - // InvalidProtocolBufferException will only throw if something changed on - // the server side. Location info won't be populated as a result. Ignore - // this error and don't bubble it up to user. - } - + Util.recordMetricsFromMetadata(responseMetadata, tracer, null); outerObserver.onComplete(); } } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index 50d24ecbaf6b..4b73a3479702 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -22,11 +22,8 @@ import com.google.api.gax.grpc.GrpcResponseMetadata; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.UnaryCallable; -import com.google.bigtable.v2.ResponseParams; import com.google.common.base.Preconditions; import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.InvalidProtocolBufferException; -import io.grpc.Metadata; import javax.annotation.Nonnull; /** @@ -79,56 +76,12 @@ class BigtableTracerUnaryCallback implements ApiFutureCallback 1) { + recorder.putRetryCount(attemptCount - 1); + } // serverLatencyTimer should already be stopped in recordAttemptCompletion recorder.putOperationLatencies(operationLatency); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java index 0e356ebaf930..5b045f15efba 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/Util.java @@ -16,6 +16,7 @@ package com.google.cloud.bigtable.data.v2.stub.metrics; import com.google.api.core.InternalApi; +import com.google.api.gax.grpc.GrpcResponseMetadata; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.StatusCode; @@ -25,10 +26,12 @@ import com.google.bigtable.v2.MutateRowsRequest; import com.google.bigtable.v2.ReadModifyWriteRowRequest; import com.google.bigtable.v2.ReadRowsRequest; +import com.google.bigtable.v2.ResponseParams; import com.google.bigtable.v2.SampleRowKeysRequest; import com.google.bigtable.v2.TableName; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; +import com.google.protobuf.InvalidProtocolBufferException; import io.grpc.Metadata; import io.grpc.Status; import io.grpc.StatusException; @@ -57,7 +60,7 @@ public class Util { private static final Metadata.Key SERVER_TIMING_HEADER_KEY = Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER); private static final Pattern SERVER_TIMING_HEADER_PATTERN = Pattern.compile(".*dur=(?\\d+)"); - static final Metadata.Key METADATA_KEY = + static final Metadata.Key LOCATION_METADATA_KEY = Metadata.Key.of("x-goog-ext-425905942-bin", Metadata.BINARY_BYTE_MARSHALLER); /** Convert an exception into a value that can be used to create an OpenCensus tag value. */ @@ -136,16 +139,62 @@ static Map> createStatsHeaders(ApiCallContext apiCallContex return headers.build(); } - static Long getGfeLatency(Metadata metadata) { - if (metadata != null && metadata.get(SERVER_TIMING_HEADER_KEY) != null) { - String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY); - Matcher matcher = SERVER_TIMING_HEADER_PATTERN.matcher(serverTiming); - // this should always be true - if (matcher.find()) { - long latency = Long.valueOf(matcher.group("dur")); - return latency; + private static Long getGfeLatency(@Nullable Metadata metadata) { + if (metadata == null) { + return null; + } + String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY); + if (serverTiming == null) { + return null; + } + Matcher matcher = SERVER_TIMING_HEADER_PATTERN.matcher(serverTiming); + // this should always be true + if (matcher.find()) { + long latency = Long.valueOf(matcher.group("dur")); + return latency; + } + return null; + } + + private static ResponseParams getResponseParams(@Nullable Metadata metadata) { + if (metadata == null) { + return null; + } + byte[] responseParams = metadata.get(Util.LOCATION_METADATA_KEY); + if (responseParams != null) { + try { + return ResponseParams.parseFrom(responseParams); + } catch (InvalidProtocolBufferException e) { } } return null; } + + static void recordMetricsFromMetadata( + GrpcResponseMetadata responseMetadata, BigtableTracer tracer, Throwable throwable) { + Metadata metadata = responseMetadata.getMetadata(); + + // Get the response params from the metadata. Check both headers and trailers + // because in different environments the metadata could be returned in headers or trailers + @Nullable ResponseParams responseParams = getResponseParams(responseMetadata.getMetadata()); + if (responseParams == null) { + responseParams = getResponseParams(responseMetadata.getTrailingMetadata()); + } + // Set tracer locations if response params is not null + if (responseParams != null) { + tracer.setLocations(responseParams.getZoneId(), responseParams.getClusterId()); + } + + // server-timing metric will be added through GrpcResponseMetadata#onHeaders(Metadata), + // so it's not checking trailing metadata here. + @Nullable Long latency = getGfeLatency(metadata); + // For direct path, we won't see GFE server-timing header. However, if we received the + // location info, we know that there isn't a connectivity issue. Set the latency to + // 0 so gfe missing header won't get incremented. + if (responseParams != null && latency == null) { + latency = 0L; + } + // Record gfe metrics + tracer.recordGfeMetadata(latency, throwable); + } } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/RowMergerUtilTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/RowMergerUtilTest.java new file mode 100644 index 000000000000..a734ea068af4 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/RowMergerUtilTest.java @@ -0,0 +1,148 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2.internal; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.bigtable.v2.ReadRowsResponse; +import com.google.bigtable.v2.ReadRowsResponse.CellChunk; +import com.google.cloud.bigtable.data.v2.models.Row; +import com.google.cloud.bigtable.data.v2.models.RowCell; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.ByteString; +import com.google.protobuf.BytesValue; +import com.google.protobuf.StringValue; +import java.util.List; +import junit.framework.TestCase; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class RowMergerUtilTest extends TestCase { + + @Test + public void testEmpty() { + try (RowMergerUtil util = new RowMergerUtil()) {} + + try (RowMergerUtil util = new RowMergerUtil()) { + util.parseReadRowsResponses(ImmutableList.of()); + } + } + + @Test + public void testSingle() { + List responses = + ImmutableList.of( + ReadRowsResponse.newBuilder() + .addChunks( + CellChunk.newBuilder() + .setRowKey(ByteString.copyFromUtf8("key")) + .setFamilyName(StringValue.newBuilder().setValue("family")) + .setQualifier( + BytesValue.newBuilder().setValue(ByteString.copyFromUtf8("qualifier"))) + .setTimestampMicros(1000) + .setValue(ByteString.copyFromUtf8("value")) + .setCommitRow(true)) + .build()); + try (RowMergerUtil util = new RowMergerUtil()) { + List rows = util.parseReadRowsResponses(responses); + assertThat(rows) + .containsExactly( + Row.create( + ByteString.copyFromUtf8("key"), + ImmutableList.of( + RowCell.create( + "family", + ByteString.copyFromUtf8("qualifier"), + 1000, + ImmutableList.of(), + ByteString.copyFromUtf8("value"))))); + } + } + + @Test + public void testMultiple() { + List responses = + ImmutableList.of( + ReadRowsResponse.newBuilder() + .addChunks( + CellChunk.newBuilder() + .setRowKey(ByteString.copyFromUtf8("key")) + .setFamilyName(StringValue.newBuilder().setValue("family")) + .setQualifier( + BytesValue.newBuilder().setValue(ByteString.copyFromUtf8("qualifier"))) + .setTimestampMicros(1000) + .setValue(ByteString.copyFromUtf8("value")) + .setCommitRow(true)) + .build(), + ReadRowsResponse.newBuilder() + .addChunks( + CellChunk.newBuilder() + .setRowKey(ByteString.copyFromUtf8("key2")) + .setFamilyName(StringValue.newBuilder().setValue("family")) + .setQualifier( + BytesValue.newBuilder().setValue(ByteString.copyFromUtf8("qualifier"))) + .setTimestampMicros(1000) + .setValue(ByteString.copyFromUtf8("value")) + .setCommitRow(true)) + .build()); + try (RowMergerUtil util = new RowMergerUtil()) { + assertThat(util.parseReadRowsResponses(responses)) + .containsExactly( + Row.create( + ByteString.copyFromUtf8("key"), + ImmutableList.of( + RowCell.create( + "family", + ByteString.copyFromUtf8("qualifier"), + 1000, + ImmutableList.of(), + ByteString.copyFromUtf8("value")))), + Row.create( + ByteString.copyFromUtf8("key2"), + ImmutableList.of( + RowCell.create( + "family", + ByteString.copyFromUtf8("qualifier"), + 1000, + ImmutableList.of(), + ByteString.copyFromUtf8("value"))))); + } + } + + @Test + public void testPartial() { + List responses = + ImmutableList.of( + ReadRowsResponse.newBuilder() + .addChunks( + CellChunk.newBuilder() + .setRowKey(ByteString.copyFromUtf8("key")) + .setFamilyName(StringValue.newBuilder().setValue("family")) + .setQualifier( + BytesValue.newBuilder().setValue(ByteString.copyFromUtf8("qualifier"))) + .setTimestampMicros(1000) + .setValue(ByteString.copyFromUtf8("value")) + .setCommitRow(false)) + .build()); + + RowMergerUtil util = new RowMergerUtil(); + util.parseReadRowsResponses(responses); + Assert.assertThrows(IllegalStateException.class, util::close); + } +} diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java index 34201d961a6d..e55f3a160d7a 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java @@ -48,7 +48,6 @@ public class BuiltinMetricsIT { public static String[] VIEWS = { "operation_latencies", - "retry_count", "attempt_latencies", "connectivity_error_count", "application_blocking_latencies" diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index 76bb7384ed1e..a2c6d417b358 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -25,13 +25,19 @@ import com.google.api.client.util.Lists; import com.google.api.core.SettableApiFuture; +import com.google.api.gax.batching.Batcher; +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.batching.FlowControlSettings; import com.google.api.gax.rpc.ClientContext; +import com.google.api.gax.rpc.NotFoundException; import com.google.api.gax.rpc.ResponseObserver; import com.google.api.gax.rpc.StreamController; import com.google.api.gax.tracing.SpanName; import com.google.bigtable.v2.BigtableGrpc; import com.google.bigtable.v2.MutateRowRequest; import com.google.bigtable.v2.MutateRowResponse; +import com.google.bigtable.v2.MutateRowsRequest; +import com.google.bigtable.v2.MutateRowsResponse; import com.google.bigtable.v2.ReadRowsRequest; import com.google.bigtable.v2.ReadRowsResponse; import com.google.bigtable.v2.ResponseParams; @@ -40,6 +46,7 @@ import com.google.cloud.bigtable.data.v2.models.Query; import com.google.cloud.bigtable.data.v2.models.Row; import com.google.cloud.bigtable.data.v2.models.RowMutation; +import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStub; import com.google.cloud.bigtable.data.v2.stub.EnhancedBigtableStubSettings; import com.google.cloud.bigtable.stats.StatsRecorderWrapper; @@ -66,6 +73,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -85,6 +93,8 @@ public class BuiltinMetricsTracerTest { private static final String INSTANCE_ID = "fake-instance"; private static final String APP_PROFILE_ID = "default"; private static final String TABLE_ID = "fake-table"; + + private static final String BAD_TABLE_ID = "non-exist-table"; private static final String ZONE = "us-west-1"; private static final String CLUSTER = "cluster-0"; private static final long FAKE_SERVER_TIMING = 50; @@ -106,6 +116,8 @@ public class BuiltinMetricsTracerTest { @Captor private ArgumentCaptor zone; @Captor private ArgumentCaptor cluster; + private int batchElementCount = 2; + @Before public void setUp() throws Exception { // Add an interceptor to add server-timing in headers @@ -127,7 +139,7 @@ public void sendHeaders(Metadata headers) { ResponseParams params = ResponseParams.newBuilder().setZoneId(ZONE).setClusterId(CLUSTER).build(); byte[] byteArray = params.toByteArray(); - headers.put(Util.METADATA_KEY, byteArray); + headers.put(Util.LOCATION_METADATA_KEY, byteArray); super.sendHeaders(headers); } @@ -150,6 +162,22 @@ public void sendHeaders(Metadata headers) { .mutateRowSettings() .retrySettings() .setInitialRetryDelay(Duration.ofMillis(200)); + + stubSettingsBuilder + .bulkMutateRowsSettings() + .setBatchingSettings( + // Each batch has 2 mutations, batch has 1 in-flight request, disable auto flush by + // setting the delay to 1 hour. + BatchingSettings.newBuilder() + .setElementCountThreshold((long) batchElementCount) + .setRequestByteThreshold(1000L) + .setDelayThreshold(Duration.ofHours(1)) + .setFlowControlSettings( + FlowControlSettings.newBuilder() + .setMaxOutstandingElementCount((long) batchElementCount) + .setMaxOutstandingRequestBytes(1000L) + .build()) + .build()); stubSettingsBuilder.setTracerFactory(mockFactory); EnhancedBigtableStubSettings stubSettings = stubSettingsBuilder.build(); @@ -163,7 +191,7 @@ public void tearDown() { } @Test - public void testOperationLatencies() { + public void testReadRowsOperationLatencies() { when(mockFactory.newTracer(any(), any(), any())) .thenAnswer( (Answer) @@ -179,8 +207,15 @@ public void testOperationLatencies() { long elapsed = stopwatch.elapsed(TimeUnit.MILLISECONDS); verify(statsRecorderWrapper).putOperationLatencies(operationLatency.capture()); + // verify record operation is only called once + verify(statsRecorderWrapper) + .recordOperation(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); assertThat(operationLatency.getValue()).isIn(Range.closed(SERVER_LATENCY, elapsed)); + assertThat(status.getAllValues()).containsExactly("OK"); + assertThat(tableId.getAllValues()).containsExactly(TABLE_ID); + assertThat(zone.getAllValues()).containsExactly(ZONE); + assertThat(cluster.getAllValues()).containsExactly(CLUSTER); } @Test @@ -198,6 +233,10 @@ public void testGfeMetrics() { Lists.newArrayList(stub.readRowsCallable().call(Query.create(TABLE_ID))); + // Verify record attempt are called multiple times + verify(statsRecorderWrapper, times(fakeService.getAttemptCounter().get())) + .recordAttempt(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); + // The request was retried and gfe latency is only recorded in the retry attempt verify(statsRecorderWrapper).putGfeLatencies(gfeLatency.capture()); assertThat(gfeLatency.getValue()).isEqualTo(FAKE_SERVER_TIMING); @@ -206,6 +245,11 @@ public void testGfeMetrics() { verify(statsRecorderWrapper, times(fakeService.getAttemptCounter().get())) .putGfeMissingHeaders(gfeMissingHeaders.capture()); assertThat(gfeMissingHeaders.getValue()).isEqualTo(1); + + assertThat(status.getAllValues()).containsExactly("UNAVAILABLE", "OK"); + assertThat(tableId.getAllValues()).containsExactly(TABLE_ID, TABLE_ID); + assertThat(zone.getAllValues()).containsExactly("global", ZONE); + assertThat(cluster.getAllValues()).containsExactly("unspecified", CLUSTER); } @Test @@ -255,6 +299,8 @@ public void onComplete() { verify(statsRecorderWrapper).putApplicationLatencies(applicationLatency.capture()); verify(statsRecorderWrapper).putOperationLatencies(operationLatency.capture()); + verify(statsRecorderWrapper) + .recordOperation(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); assertThat(counter.get()).isEqualTo(fakeService.getResponseCounter().get()); assertThat(applicationLatency.getValue()).isAtLeast(APPLICATION_LATENCY * counter.get()); @@ -287,6 +333,8 @@ public void testReadRowsApplicationLatencyWithManualFlowControl() throws Excepti verify(statsRecorderWrapper).putApplicationLatencies(applicationLatency.capture()); verify(statsRecorderWrapper).putOperationLatencies(operationLatency.capture()); + verify(statsRecorderWrapper) + .recordOperation(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); // For manual flow control, the last application latency shouldn't count, because at that point // the server already sent back all the responses. @@ -324,7 +372,7 @@ public void testRetryCount() { } @Test - public void testMutateRowAttempts() { + public void testMutateRowAttemptsTagValues() { when(mockFactory.newTracer(any(), any(), any())) .thenReturn( new BuiltinMetricsTracer( @@ -343,6 +391,84 @@ public void testMutateRowAttempts() { assertThat(zone.getAllValues()).containsExactly("global", "global", ZONE); assertThat(cluster.getAllValues()).containsExactly("unspecified", "unspecified", CLUSTER); assertThat(status.getAllValues()).containsExactly("UNAVAILABLE", "UNAVAILABLE", "OK"); + assertThat(tableId.getAllValues()).containsExactly(TABLE_ID, TABLE_ID, TABLE_ID); + } + + @Test + public void testReadRowsAttemptsTagValues() { + when(mockFactory.newTracer(any(), any(), any())) + .thenReturn( + new BuiltinMetricsTracer( + OperationType.ServerStreaming, + SpanName.of("Bigtable", "ReadRows"), + statsRecorderWrapper)); + + Lists.newArrayList(stub.readRowsCallable().call(Query.create("fake-table")).iterator()); + + // Set a timeout to reduce flakiness of this test. BasicRetryingFuture will set + // attempt succeeded and set the response which will call complete() in AbstractFuture which + // calls releaseWaiters(). onOperationComplete() is called in TracerFinisher which will be + // called after the mutateRow call is returned. So there's a race between when the call returns + // and when the record() is called in onOperationCompletion(). + verify(statsRecorderWrapper, timeout(50).times(fakeService.getAttemptCounter().get())) + .recordAttempt(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); + assertThat(zone.getAllValues()).containsExactly("global", ZONE); + assertThat(cluster.getAllValues()).containsExactly("unspecified", CLUSTER); + assertThat(status.getAllValues()).containsExactly("UNAVAILABLE", "OK"); + } + + @Test + public void testClientBlockingLatencies() throws InterruptedException { + when(mockFactory.newTracer(any(), any(), any())) + .thenReturn( + new BuiltinMetricsTracer( + OperationType.Unary, SpanName.of("Bigtable", "MutateRows"), statsRecorderWrapper)); + try (Batcher batcher = stub.newMutateRowsBatcher(TABLE_ID, null)) { + for (int i = 0; i < 6; i++) { + batcher.add(RowMutationEntry.create("key").setCell("f", "q", "v")); + } + + int expectedNumRequests = 6 / batchElementCount; + ArgumentCaptor throttledTime = ArgumentCaptor.forClass(Long.class); + verify(statsRecorderWrapper, times(expectedNumRequests)) + .putBatchRequestThrottled(throttledTime.capture()); + + // Adding the first 2 elements should not get throttled since the batch is empty + assertThat(throttledTime.getAllValues().get(0)).isEqualTo(0); + // After the first request is sent, batcher will block on add because of the server latency. + // Blocking latency should be around server latency. + assertThat(throttledTime.getAllValues().get(1)).isAtLeast(SERVER_LATENCY - 10); + assertThat(throttledTime.getAllValues().get(2)).isAtLeast(SERVER_LATENCY - 10); + } + } + + @Test + public void testPermanentFailure() { + when(mockFactory.newTracer(any(), any(), any())) + .thenReturn( + new BuiltinMetricsTracer( + OperationType.ServerStreaming, + SpanName.of("Bigtable", "ReadRows"), + statsRecorderWrapper)); + + try { + Lists.newArrayList(stub.readRowsCallable().call(Query.create(BAD_TABLE_ID)).iterator()); + Assert.fail("Request should throw not found error"); + } catch (NotFoundException e) { + } + + ArgumentCaptor attemptLatency = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor operationLatency = ArgumentCaptor.forClass(Long.class); + + verify(statsRecorderWrapper, timeout(50)).putAttemptLatencies(attemptLatency.capture()); + verify(statsRecorderWrapper, timeout(50)).putOperationLatencies(operationLatency.capture()); + verify(statsRecorderWrapper, timeout(50)) + .recordAttempt(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); + + assertThat(status.getValue()).isEqualTo("NOT_FOUND"); + assertThat(tableId.getValue()).isEqualTo(BAD_TABLE_ID); + assertThat(cluster.getValue()).isEqualTo("unspecified"); + assertThat(zone.getValue()).isEqualTo("global"); } private static class FakeService extends BigtableGrpc.BigtableImplBase { @@ -375,6 +501,10 @@ static List createFakeResponse() { @Override public void readRows( ReadRowsRequest request, StreamObserver responseObserver) { + if (request.getTableName().contains(BAD_TABLE_ID)) { + responseObserver.onError(new StatusRuntimeException(Status.NOT_FOUND)); + return; + } final AtomicBoolean done = new AtomicBoolean(); final ServerCallStreamObserver target = (ServerCallStreamObserver) responseObserver; @@ -413,6 +543,17 @@ public void mutateRow( responseObserver.onCompleted(); } + @Override + public void mutateRows( + MutateRowsRequest request, StreamObserver responseObserver) { + try { + Thread.sleep(SERVER_LATENCY); + } catch (InterruptedException e) { + } + responseObserver.onNext(MutateRowsResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } + public AtomicInteger getAttemptCounter() { return attemptCounter; } diff --git a/grpc-google-cloud-bigtable-admin-v2/pom.xml b/grpc-google-cloud-bigtable-admin-v2/pom.xml index 308e5b3fb5f1..a553dcceeaa9 100644 --- a/grpc-google-cloud-bigtable-admin-v2/pom.xml +++ b/grpc-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.15.0 + 2.15.1 grpc-google-cloud-bigtable-admin-v2 GRPC library for grpc-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 2.15.0 + 2.15.1 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.15.0 + 2.15.1 pom import com.google.cloud google-cloud-bigtable-bom - 2.15.0 + 2.15.1 pom import diff --git a/grpc-google-cloud-bigtable-v2/pom.xml b/grpc-google-cloud-bigtable-v2/pom.xml index d6b1e51acec8..1ea54edfed80 100644 --- a/grpc-google-cloud-bigtable-v2/pom.xml +++ b/grpc-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.15.0 + 2.15.1 grpc-google-cloud-bigtable-v2 GRPC library for grpc-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 2.15.0 + 2.15.1 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.15.0 + 2.15.1 pom import com.google.cloud google-cloud-bigtable-bom - 2.15.0 + 2.15.1 pom import diff --git a/pom.xml b/pom.xml index a64d00d3735b..f934707d2ce1 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ google-cloud-bigtable-parent pom - 2.15.0 + 2.15.1 Google Cloud Bigtable Parent https://github.com/googleapis/java-bigtable @@ -153,27 +153,27 @@ com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.15.0 + 2.15.1 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.15.0 + 2.15.1 com.google.api.grpc grpc-google-cloud-bigtable-v2 - 2.15.0 + 2.15.1 com.google.api.grpc grpc-google-cloud-bigtable-admin-v2 - 2.15.0 + 2.15.1 com.google.cloud google-cloud-bigtable - 2.15.0 + 2.15.1 diff --git a/proto-google-cloud-bigtable-admin-v2/pom.xml b/proto-google-cloud-bigtable-admin-v2/pom.xml index c71bc4f23ba9..43cff3037c4d 100644 --- a/proto-google-cloud-bigtable-admin-v2/pom.xml +++ b/proto-google-cloud-bigtable-admin-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-admin-v2 - 2.15.0 + 2.15.1 proto-google-cloud-bigtable-admin-v2 PROTO library for proto-google-cloud-bigtable-admin-v2 com.google.cloud google-cloud-bigtable-parent - 2.15.0 + 2.15.1 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.15.0 + 2.15.1 pom import com.google.cloud google-cloud-bigtable-bom - 2.15.0 + 2.15.1 pom import diff --git a/proto-google-cloud-bigtable-v2/pom.xml b/proto-google-cloud-bigtable-v2/pom.xml index 56c5aa9b2cb6..b5c60b057c0c 100644 --- a/proto-google-cloud-bigtable-v2/pom.xml +++ b/proto-google-cloud-bigtable-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-bigtable-v2 - 2.15.0 + 2.15.1 proto-google-cloud-bigtable-v2 PROTO library for proto-google-cloud-bigtable-v2 com.google.cloud google-cloud-bigtable-parent - 2.15.0 + 2.15.1 @@ -18,14 +18,14 @@ com.google.cloud google-cloud-bigtable-deps-bom - 2.15.0 + 2.15.1 pom import com.google.cloud google-cloud-bigtable-bom - 2.15.0 + 2.15.1 pom import diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index c56163f327e9..ff58cb43bfc8 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -29,7 +29,7 @@ com.google.cloud google-cloud-bigtable - 2.14.1 + 2.15.0 diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index b9bb862a0e3a..bc8a2b80a401 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -29,7 +29,7 @@ com.google.cloud libraries-bom - 26.1.3 + 26.1.4 pom import @@ -89,7 +89,7 @@ org.graalvm.buildtools junit-platform-native - 0.9.16 + 0.9.17 test @@ -99,7 +99,7 @@ org.graalvm.buildtools native-maven-plugin - 0.9.16 + 0.9.17 true com.example.bigtable.NativeImageBigtableSample diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 3a23e246107d..cc589431f6f7 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-bigtable - 2.15.0 + 2.15.1 diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 0facce3f6e5b..55e0d9f8d2b3 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -30,7 +30,7 @@ com.google.cloud libraries-bom - 26.1.3 + 26.1.4 pom import diff --git a/versions.txt b/versions.txt index 4cc3a8e52fc5..9118fd205d2e 100644 --- a/versions.txt +++ b/versions.txt @@ -1,10 +1,10 @@ # Format: # module:released-version:current-version -google-cloud-bigtable:2.15.0:2.15.0 -grpc-google-cloud-bigtable-admin-v2:2.15.0:2.15.0 -grpc-google-cloud-bigtable-v2:2.15.0:2.15.0 -proto-google-cloud-bigtable-admin-v2:2.15.0:2.15.0 -proto-google-cloud-bigtable-v2:2.15.0:2.15.0 -google-cloud-bigtable-emulator:0.152.0:0.152.0 -google-cloud-bigtable-emulator-core:2.15.0:2.15.0 +google-cloud-bigtable:2.15.1:2.15.1 +grpc-google-cloud-bigtable-admin-v2:2.15.1:2.15.1 +grpc-google-cloud-bigtable-v2:2.15.1:2.15.1 +proto-google-cloud-bigtable-admin-v2:2.15.1:2.15.1 +proto-google-cloud-bigtable-v2:2.15.1:2.15.1 +google-cloud-bigtable-emulator:0.152.1:0.152.1 +google-cloud-bigtable-emulator-core:2.15.1:2.15.1