.github/workflows/lint.yaml

@@ -15,24 +15,24 @@ jobs:
     steps:
       - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # v3
         with:
-          go-version: 1.19
+          go-version: 1.23.1
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3
       - name: Lint
         uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 # v3.4.0
         with:
-          version: v1.54
+          version: v1.64.6
           args: --timeout=3m
 
   lint-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # v3
         with:
-          go-version: 1.19
+          go-version: 1.23.1
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3
       - name: Lint
         uses: golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5 # v3.4.0
         with:
-          version: v1.54
+          version: v1.64.6
           args: --timeout=3m
           working-directory: github_ratelimit/github_ratelimit_test

.github/workflows/test.yaml

@@ -8,24 +8,29 @@ on:
   pull_request:
     branches:
       - main
+  schedule:
+    - cron: '0 0 * * 0'  # Runs at 00:00 UTC every Sunday
 
 jobs:
   build_and_test:
     runs-on: ubuntu-latest
     env:
-      TEST_DIR: github_ratelimit/github_ratelimit_test
+      TEST_DIR: e2e_test
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/setup-go@c4a742cab115ed795e34d4513e2cf7d472deb55f # v3
         with:
-          go-version: 1.19
+          go-version: 1.23.1
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3
       - name: Verify dependencies
         run: go mod verify
       - name: Build
         run: go build -v ./...
       - name: Vet
         run: go vet -v ./...
-      - name: Vet-Test
+      - name: Test
+        run: go test -v -count=1 -shuffle=on -timeout=30m -race ./...
+      - name: E2E-Vet
         run: cd "$TEST_DIR" && go vet -v ./...
       - name: Test
         run: cd "$TEST_DIR" && go test -v -count=1 -shuffle=on -timeout=30m -race ./...

README.md

@@ -2,50 +2,116 @@
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/gofri/go-github-ratelimit)](https://goreportcard.com/report/github.com/gofri/go-github-ratelimit)
 
-Package `go-github-ratelimit` provides an http.RoundTripper implementation that handles [secondary rate limit](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#about-secondary-rate-limits) for the GitHub API.  
-The RoundTripper waits for the secondary rate limit to finish in a blocking mode and then issues/retries requests.
+Package `go-github-ratelimit` providesa middleware (http.RoundTripper) that handles both [Primary Rate Limit](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?#about-primary-rate-limits) and [Secondary Rate Limit](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?#about-secondary-rate-limits) for the GitHub API.
+
+* Primary rate limits are handled by returning a detailed error.  
+* Secondary rate limits are handled by waiting in blocking mode (sleep) and then issuing/retrying requests.  
+* There is support for callbacks to be triggered when rate limits are detected/exceeded/etc. - see below.  
+
+The module can be used with any HTTP client communicating with GitHub API. It is designed to have low overhead during good path.    
+It is meant to complement [go-github](https://github.com/google/go-github), but there is no association between this repository and the go-github repository nor Google.
+
+## Recommended: Pagination Handling
+
+If you like this package, please check out [go-github-pagination](https://github.com/gofri/go-github-pagination).  
+It supports pagination out of the box, and plays well with the rate limit round-tripper.  
+It is best to stack the pagination round-tripper on top of the rate limit round-tripper.  
+
 
-`go-github-ratelimit` can be used with any HTTP client communicating with GitHub API.  
-It is meant to complement [go-github](https://github.com/google/go-github), but there is no association between this repository and the go-github repository nor Google.  
-
 ## Installation
 
-```go get github.com/gofri/go-github-ratelimit```
+```go get github.com/gofri/go-github-ratelimit/v2```
 
 ## Usage Example (with [go-github](https://github.com/google/go-github))
 
 ```go
-import "github.com/google/go-github/v58/github"
-import "github.com/gofri/go-github-ratelimit/github_ratelimit"
+import "github.com/google/go-github/v68/github"
+import "github.com/gofri/go-github-ratelimit/v2/github_ratelimit"
 
 func main() {
-  rateLimiter, err := github_ratelimit.NewRateLimitWaiterClient(nil)
+  // use the plain ratelimiter, without options / callbacks / underlying http.RoundTripper.
+  rateLimiter, err := github_ratelimit.New(nil)
   if err != nil {
     panic(err)
   }
   client := github.NewClient(rateLimiter).WithAuthToken("your personal access token")
 
+  // disable go-github's built-in rate limiting
+  ctx := context.WithValue(context.Background(), github.BypassRateLimitCheck)
+
   // now use the client as you please
 }
 ```
 
 ## Client Options
 
-The RoundTripper accepts a set of options to configure its behavior and set callbacks. nil callbacks are treated as no-op.  
-The options are:
+Both RoundTrippers support a set of options to configure their behavior and set callbacks.  
+nil callbacks are treated as no-op.  
+
+### Primary Rate Limit Options:
+
+- `WithLimitDetectedCallback(callback)`: the callback is triggered when any primary rate limit is detected.
+- `WithRequestPreventedCallback(callback)`: the callback is triggered when a request is prevented due to an active rate limit.
+- `WithLimitResetCallback(callback)`: the callback is triggered when the rate limit is reset (deactived).
+- `WithUnknownCategoryCallback`: the callback is triggered when the rate limit category in the response is unknown. note: please open an issue if it happens.
+- `WithSharedState(state)`: share state between multiple clients (e.g., for a single user running concurrently).
+- `WithBypassLimit()`: bypass the rate limit mechanism, i.e., do not prevent requests when a rate limit is active.
+
+### Secondary Rate Limit Options:
 
 - `WithLimitDetectedCallback(callback)`: the callback is triggered before a sleep.
 - `WithSingleSleepLimit(duration, callback)`: limit the sleep duration for a single secondary rate limit & trigger a callback when the limit is exceeded.
 - `WithTotalSleepLimit(duration, callback)`: limit the accumulated sleep duration for all secondary rate limits & trigger a callback when the limit is exceeded.
-
-_Note_: to detect secondary rate limits without sleeping, use `WithSingleSleepLimit(0, your_callback_or_nil)`.
+- `WithNoSleep(callback)`: disable sleep for secondary rate limits & trigger a callback upon any secondary rate limit.
 
 ## Per-Request Options
 
 Use `WithOverrideConfig(opts...)` to override the configuration for a specific request (using the request context).  
 Per-request overrides may be useful for special cases of user requests,
 as well as fine-grained policy control (e.g., for a sophisticated pagination mechanism).
 
+## Advanced Example
+
+```go
+import "github.com/google/go-github/v68/github"
+import "github.com/gofri/go-github-ratelimit/v2/github_ratelimit"
+import "github.com/gofri/go-github-pagination/githubpagination"
+
+func main() {
+  // Set up the rate limiter with some options.
+  rateLimiter, err := github_ratelimit.New(nil,
+    github_ratelimit.WithLimitDetectedCallback(func(category string, limit int, remaining int, reset int) {
+      fmt.Printf("Primary rate limit detected: %s, %d/%d, reset in %d seconds\n", category, remaining, limit, reset)
+    }),
+    github_ratelimit.WithRequestPreventedCallback(func(category string, limit int, remaining int, reset int) {
+      fmt.Printf("Request prevented due to primary rate limit: %s, %d/%d, reset in %d seconds\n", category, remaining, limit, reset)
+    }),
+  )
+  if err != nil {
+    panic(err)
+  }
+
+  paginator := githubpagination.NewClient(nil,
+    githubpagination.WithPerPage(100), // default to 100 results per page
+  )
+
+  client := github.NewClient(rateLimiter).WithAuthToken("your personal access token")
+
+  // now use the client as you please
+}
+```
+
+## Migration (V1 => V2)
+
+The migraiton from v1 to v2 is relatively straight-forward once you check out the examples.  
+Please open an issue if you have any trouble -  
+I'd be glad to help and add documetation per need.
+
+## Github Rate Limit References
+
+- [Primary Rate Limit](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?#about-primary-rate-limits)
+- [Secondary Rate Limit](https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?#about-secondary-rate-limits)
+
 ## License
 
 This package is distributed under the MIT license found in the LICENSE file.  

e2e_test/e2e_test.go

@@ -0,0 +1,82 @@
+package e2e_test
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_ratelimit_test"
+	"github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit"
+	"github.com/google/go-github/v64/github"
+)
+
+type orgLister struct {
+}
+
+func (o *orgLister) GetOrgName() string {
+	return "org"
+}
+
+func (o *orgLister) RoundTrip(r *http.Request) (*http.Response, error) {
+	org := github.Organization{
+		Login: github.String(o.GetOrgName()),
+	}
+
+	body, err := json.Marshal([]*github.Organization{&org})
+	if err != nil {
+		return nil, err
+	}
+
+	return &http.Response{
+		Body:       io.NopCloser(bytes.NewReader(body)),
+		Header:     http.Header{},
+		StatusCode: http.StatusOK,
+	}, nil
+}
+
+// TestGoGithubClient is a test that uses the go-github client.
+func TestGoGithubClientCompatability(t *testing.T) {
+	t.Parallel()
+	const every = 5 * time.Second
+	const sleep = 1 * time.Second
+
+	print := func(context *github_secondary_ratelimit.CallbackContext) {
+		t.Logf("Secondary rate limit reached! Sleeping for %.2f seconds [%v --> %v]",
+			time.Until(*context.ResetTime).Seconds(), time.Now(), *context.ResetTime)
+	}
+
+	orgLister := &orgLister{}
+	options := github_ratelimit_test.RateLimitInjecterOptions{
+		Every:             every,
+		InjectionDuration: sleep,
+	}
+
+	i := github_ratelimit_test.SetupInjecterWithOptions(t, options, orgLister)
+	rateLimiter := github_ratelimit_test.NewSecondaryClient(i, github_secondary_ratelimit.WithLimitDetectedCallback(print))
+
+	client := github.NewClient(rateLimiter)
+	orgs, resp, err := client.Organizations.List(context.Background(), "", nil)
+	if err != nil {
+		t.Fatalf("unexpected error response: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("unexpected status code: %v", resp.StatusCode)
+	}
+
+	if len(orgs) != 1 {
+		t.Fatalf("unexpected number of orgs: %v", len(orgs))
+	}
+
+	if orgs[0].GetLogin() != orgLister.GetOrgName() {
+		t.Fatalf("unexpected org name: %v", orgs[0].GetLogin())
+	}
+
+	// TODO add tests for:
+	// - WithSingleSleepLimit(0, ...) => expect AbuseError
+	// - WithSingleSleepLimit(>0, ...) => expect sleeping
+}

e2e_test/go.mod

@@ -0,0 +1,12 @@
+module github.com/gofri/go-github-ratelimit-e2e
+
+replace github.com/gofri/go-github-ratelimit => ../
+
+go 1.23.1
+
+require (
+	github.com/gofri/go-github-ratelimit v0.0.0-00010101000000-000000000000
+	github.com/google/go-github/v64 v64.0.0
+)
+
+require github.com/google/go-querystring v1.1.0 // indirect

github_ratelimit/github_ratelimit_test/go.sum → e2e_test/go.sum

@@ -1,9 +1,8 @@
-github.com/gofri/go-github-ratelimit v1.1.0 h1:ijQ2bcv5pjZXNil5FiwglCg8wc9s8EgjTmNkqjw8nuk=
-github.com/gofri/go-github-ratelimit v1.1.0/go.mod h1:OnCi5gV+hAG/LMR7llGhU7yHt44se9sYgKPnafoL7RY=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-github/v58 v58.0.0 h1:Una7GGERlF/37XfkPwpzYJe0Vp4dt2k1kCjlxwjIvzw=
-github.com/google/go-github/v58 v58.0.0/go.mod h1:k4hxDKEfoWpSqFlc8LTpGd9fu2KrV1YAa6Hi6FmDNY4=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-github/v64 v64.0.0 h1:4G61sozmY3eiPAjjoOHponXDBONm+utovTKbyUb2Qdg=
+github.com/google/go-github/v64 v64.0.0/go.mod h1:xB3vqMQNdHzilXBiO2I+M7iEFtHf+DP/omBOv6tQzVo=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

github_ratelimit/combined_ratelimit_test.go

@@ -0,0 +1,63 @@
+package github_ratelimit
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_primary_ratelimit"
+	"github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_ratelimit_test"
+	"github.com/gofri/go-github-ratelimit/v2/github_ratelimit/github_secondary_ratelimit"
+)
+
+func TestCombinedRateLimiter(t *testing.T) {
+	t.Parallel()
+
+	everySecondary := 300 * time.Millisecond
+	everyPrimary := 500 * time.Millisecond
+	sleepTime := 100 * time.Millisecond
+
+	injecter := github_ratelimit_test.SetupSecondaryInjecter(t, everySecondary, sleepTime).(*github_ratelimit_test.RateLimitInjecter)
+	injecter.Base = github_ratelimit_test.SetupPrimaryInjecter(t,
+		everyPrimary, sleepTime,
+		github_primary_ratelimit.ResourceCategoryCore,
+	)
+
+	primaryCalled := false
+	secondaryCalled := false
+	c := &http.Client{
+		Transport: New(injecter,
+			github_primary_ratelimit.WithLimitDetectedCallback(func(context *github_primary_ratelimit.CallbackContext) {
+				t.Logf("primary rate limit reached!")
+				primaryCalled = true
+			}),
+			github_secondary_ratelimit.WithLimitDetectedCallback(func(context *github_secondary_ratelimit.CallbackContext) {
+				t.Logf("secondary rate limit reached!")
+				secondaryCalled = true
+			}),
+			github_secondary_ratelimit.WithNoSleep(nil),
+		),
+	}
+
+	_, err := c.Get("/initial-no-called")
+	if err != nil {
+		t.Fatalf("expecting first request to succeed, got %v", err)
+	}
+	if primaryCalled || secondaryCalled {
+		t.Fatalf("expecting primary=false and secondary=false, got primary=%v, secondary=%v", primaryCalled, secondaryCalled)
+	}
+
+	// wait until secondary rate limit is triggered
+	injecter.WaitForNextInjection()
+	_, err = c.Get("/only-secondary")
+	if primaryCalled || !secondaryCalled {
+		t.Fatalf("expecting primary=false, and secondary=true, got primary=%v, secondary=%v. err: %v", primaryCalled, secondaryCalled, err)
+	}
+
+	// wait until primary rate limit is triggered
+	injecter.Base.(*github_ratelimit_test.RateLimitInjecter).WaitForNextInjection()
+	_, err = c.Get("/only-primary")
+	if !primaryCalled {
+		t.Fatalf("expecting primary=true, got primary=%v. err: %v", primaryCalled, err)
+	}
+}