fix: retry on http 400 failedPrecondition

googleapiclient/http.py

@@ -97,10 +97,10 @@ def _should_retry_response(resp_status, content):
     if resp_status == _TOO_MANY_REQUESTS:
         return True
 
-    # For 403 errors, we have to check for the `reason` in the response to
+    # For 403 and 400 errors, we have to check for the `reason` in the response to
     # determine if we should retry.
-    if resp_status == http_client.FORBIDDEN:
-        # If there's no details about the 403 type, don't retry.
+    if resp_status in [http_client.FORBIDDEN, http_client.BAD_REQUEST]:
+        # If there's no details about the error, don't retry.
         if not content:
             return False
 
@@ -137,11 +137,21 @@ def _should_retry_response(resp_status, content):
             LOGGER.warning("Invalid JSON content from response: %s", content)
             return False
 
-        LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)
-
-        # Only retry on rate limit related failures.
-        if reason in ("userRateLimitExceeded", "rateLimitExceeded"):
-            return True
+        RETRYABLE_INFO = {
+            http_client.FORBIDDEN: {
+                "reasons": ("userRateLimitExceeded", "rateLimitExceeded"),
+                "message": 'Encountered 403 Forbidden with reason "%s"',
+            },
+            http_client.BAD_REQUEST: {
+                "reasons": ("failedPrecondition", "preconditionFailed"),
+                "message": 'Encountered 400 Bad Request with reason "%s"',
+            },
+        }
+        if resp_status in RETRYABLE_INFO:
+            info = RETRYABLE_INFO[resp_status]
+            LOGGER.warning(info["message"], reason)
+            if reason in info["reasons"]:
+                return True
 
     # Everything else is a success or non-retriable so break.
     return False

tests/test_http.py

@@ -878,6 +878,20 @@ def test_media_io_base_download_unknown_media_size(self):
  }
 }"""
 
+FAILED_PRECONDITION_RESPONSE = """{
+ "error": {
+  "errors": [
+   {
+    "domain": "global",
+    "reason": "failedPrecondition",
+    "message": "Precondition Failed"
+   }
+  ],
+  "code": 400,
+  "message": "Precondition Failed"
+ }
+}"""
+
 
 NOT_CONFIGURED_RESPONSE = """{
  "error": {
@@ -1056,6 +1070,34 @@ def test_no_retry_succeeds(self):
 
         self.assertEqual(0, len(sleeptimes))
 
+    def test_retry_400_failed_precondition(self):
+        num_retries = 2
+        resp_seq = [
+            ({"status": "400"}, FAILED_PRECONDITION_RESPONSE),
+            ({"status": "200"}, "{}")
+        ]
+        http = HttpMockSequence(resp_seq)
+        model = JsonModel()
+        uri = "https://www.googleapis.com/someapi/v1/collection/?foo=bar"
+        method = "POST"
+        request = HttpRequest(
+            http,
+            model.response,
+            uri,
+            method=method,
+            body="{}",
+            headers={"content-type": "application/json"},
+        )
+
+        sleeptimes = []
+        request._sleep = lambda x: sleeptimes.append(x)
+        request._rand = lambda: 10
+
+        request.execute(num_retries=num_retries)
+
+        self.assertEqual(1, len(sleeptimes))
+        self.assertEqual(10 * 2 ** 1, sleeptimes[0])
+
     def test_no_retry_fails_fast(self):
         http = HttpMockSequence([({"status": "500"}, ""), ({"status": "200"}, "{}")])
         model = JsonModel()