Add request retry with backoff when encountring 429 errors (#21511)

* Add request retry with backoff when encountring 429 errors

* Fix lint errors

---------

Co-authored-by: Josh Holtz <me@joshholtz.com>

spaceship/lib/spaceship/connect_api/api_client.rb

@@ -156,7 +156,13 @@ def initialize(msg)
         end
       end
 
-      def with_asc_retry(tries = 5, &_block)
+      class TooManyRequestsError < StandardError
+        def initialize(msg)
+          super
+        end
+      end
+
+      def with_asc_retry(tries = 5, backoff = 1, &_block)
         response = yield
 
         status = response.status if response
@@ -166,6 +172,10 @@ def with_asc_retry(tries = 5, &_block)
           raise TimeoutRetryError, msg
         end
 
+        if status == 429
+          raise TooManyRequestsError, "Too many requests, backing off #{backoff} seconds"
+        end
+
         return response
       rescue UnauthorizedAccessError => error
         tries -= 1
@@ -186,6 +196,14 @@ def with_asc_retry(tries = 5, &_block)
         else
           retry
         end
+      rescue TooManyRequestsError => error
+        if backoff > 3600
+          raise TooManyRequestsError, "Too many requests, giving up after backing off for > 3600 seconds."
+        end
+        puts(error) if Spaceship::Globals.verbose?
+        Kernel.sleep(backoff)
+        backoff *= 2
+        retry
       end
 
       def handle_response(response)

spaceship/spec/connect_api/api_client_spec.rb

@@ -126,6 +126,31 @@ def stub_client_request(uri, status, body)
         client.get('')
       end.to raise_error(Spaceship::AccessForbiddenError)
     end
+
+    describe 'with_retry' do
+      it 'sleeps on 429' do
+        stub_request(:get, client.hostname).
+          to_return(status: 429).then.
+          to_return(status: 200, body: "")
+
+        expect(Kernel).to receive(:sleep).once.with(1)
+        expect(client).to receive(:handle_response).once
+        expect(client).to receive(:request).twice.and_call_original
+        expect do
+          client.get('')
+        end.to_not(raise_error)
+      end
+
+      it 'sleeps until limit is reached on 429' do
+        body = JSON.generate({ "errors": [{ "title": "The request rate limit has been reached.", "details": "We've received too many requests for this API. Please wait and try again or slow down your request rate." }] })
+        stub_client_request(client.hostname, 429, body)
+
+        expect(Kernel).to receive(:sleep).exactly(12).times
+        expect do
+          client.get('')
+        end.to raise_error(Spaceship::ConnectAPI::APIClient::TooManyRequestsError, "Too many requests, giving up after backing off for > 3600 seconds.")
+      end
+    end
   end
 
   describe "#handle_error" do