From 1521f620af2ccff1544f84c38d3efb34c515dc23 Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Tue, 2 May 2023 03:56:00 +0000 Subject: [PATCH 1/4] fix(httpreplay): add ignore-header flag --- httpreplay/cmd/httpr/httpr.go | 32 ++++++++++++++++++++---- httpreplay/cmd/httpr/integration_test.go | 13 +++++++++- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/httpreplay/cmd/httpr/httpr.go b/httpreplay/cmd/httpr/httpr.go index c0588e95755..530756c79ff 100644 --- a/httpreplay/cmd/httpr/httpr.go +++ b/httpreplay/cmd/httpr/httpr.go @@ -29,20 +29,24 @@ import ( "net/http" "os" "os/signal" + "strings" "cloud.google.com/go/httpreplay/internal/proxy" "github.com/google/martian/v3/martianhttp" ) var ( - port = flag.Int("port", 8080, "port of the proxy") - controlPort = flag.Int("control-port", 8181, "port for controlling the proxy") - record = flag.String("record", "", "record traffic and save to filename") - replay = flag.String("replay", "", "read filename and replay traffic") - debugHeaders = flag.Bool("debug-headers", false, "log header mismatches") + port = flag.Int("port", 8080, "port of the proxy") + controlPort = flag.Int("control-port", 8181, "port for controlling the proxy") + record = flag.String("record", "", "record traffic and save to filename") + replay = flag.String("replay", "", "read filename and replay traffic") + debugHeaders = flag.Bool("debug-headers", false, "log header mismatches") + ignoreHeaders repeatedString ) func main() { + flag.Var(&ignoreHeaders, "ignore-header", "header key(s) to ignore when matching") + flag.Parse() if *record == "" && *replay == "" { log.Fatal("provide either -record or -replay") @@ -63,6 +67,9 @@ func main() { log.Fatal(err) } proxy.DebugHeaders = *debugHeaders + for _, key := range ignoreHeaders { + pr.IgnoreHeader(key) + } // Expose handlers on the control port. mux := http.NewServeMux() @@ -108,3 +115,18 @@ func handleInitial(pr *proxy.Proxy) http.HandlerFunc { } } } + +type repeatedString []string + +func (i *repeatedString) String() string { + v := make([]string, 0) + if i != nil { + v = *i + } + return strings.Join(v, ",") +} + +func (i *repeatedString) Set(value string) error { + *i = append(*i, value) + return nil +} diff --git a/httpreplay/cmd/httpr/integration_test.go b/httpreplay/cmd/httpr/integration_test.go index 0b1915d48b5..888cfefaf5a 100644 --- a/httpreplay/cmd/httpr/integration_test.go +++ b/httpreplay/cmd/httpr/integration_test.go @@ -143,7 +143,15 @@ func start(modeFlag, filename string) (*exec.Cmd, *http.Transport, string, error if err != nil { return nil, nil, "", err } - cmd := exec.Command("./httpr", "-port", pport, "-control-port", cport, modeFlag, filename, "-debug-headers") + cmd := exec.Command("./httpr", + "-port", pport, + "-control-port", cport, + modeFlag, + filename, + "-debug-headers", + "-ignore-header", "X-Goog-Api-Client", + "-ignore-header", "X-Goog-Gcs-Idempotency-Token", + ) if err := cmd.Start(); err != nil { return nil, nil, "", err } @@ -204,6 +212,9 @@ func proxyTransport(pport, cport string) (*http.Transport, error) { } func getBucketInfo(ctx context.Context, hc *http.Client) (string, error) { + ctx, cc := context.WithTimeout(ctx, 10*time.Second) + defer cc() + client, err := storage.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { return "", err From 0855862824f17d92d884eea26fe9949ed85083c5 Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Tue, 2 May 2023 04:00:27 +0000 Subject: [PATCH 2/4] fix TestIntegration_(RecordAndReplay|HTTPR) --- httpreplay/httpreplay_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/httpreplay/httpreplay_test.go b/httpreplay/httpreplay_test.go index 3b3252b7e78..555b4835a0a 100644 --- a/httpreplay/httpreplay_test.go +++ b/httpreplay/httpreplay_test.go @@ -68,6 +68,9 @@ func TestIntegration_RecordAndReplay(t *testing.T) { if err != nil { t.Fatal(err) } + rec.RemoveRequestHeaders("X-Goog-Api-Client") + rec.RemoveRequestHeaders("X-Goog-Gcs-Idempotency-Token") + hc, err := rec.Client(ctx, option.WithTokenSource( testutil.TokenSource(ctx, storage.ScopeFullControl))) if err != nil { From 2cc20b3a08e61bddcbf32014e2b7a0e1a98449c5 Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Tue, 2 May 2023 04:05:19 +0000 Subject: [PATCH 3/4] add more timeouts --- httpreplay/httpreplay_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/httpreplay/httpreplay_test.go b/httpreplay/httpreplay_test.go index 555b4835a0a..3b16a4393ac 100644 --- a/httpreplay/httpreplay_test.go +++ b/httpreplay/httpreplay_test.go @@ -156,7 +156,9 @@ func setup(ctx context.Context) (cleanup func(), err error) { // TODO(jba): test errors func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) { - ctx := context.Background() + ctx, cc := context.WithTimeout(context.Background(), 10*time.Second) + defer cc() + client, err := storage.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { t.Fatal(err) @@ -191,7 +193,9 @@ func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) { } func testReadCRC(t *testing.T, hc *http.Client, mode string) { - ctx := context.Background() + ctx, cc := context.WithTimeout(context.Background(), 10*time.Second) + defer cc() + client, err := storage.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { t.Fatalf("%s: %v", mode, err) From a12e55c11ca946dfa6f93bfef25da10eba5ce2ec Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Tue, 2 May 2023 04:08:07 +0000 Subject: [PATCH 4/4] add ignoreheader on replay too --- httpreplay/httpreplay_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/httpreplay/httpreplay_test.go b/httpreplay/httpreplay_test.go index 3b16a4393ac..d5fd3cc3935 100644 --- a/httpreplay/httpreplay_test.go +++ b/httpreplay/httpreplay_test.go @@ -87,6 +87,8 @@ func TestIntegration_RecordAndReplay(t *testing.T) { if err != nil { t.Fatal(err) } + rep.IgnoreHeader("X-Goog-Api-Client") + rep.IgnoreHeader("X-Goog-Gcs-Idempotency-Token") defer rep.Close() hc, err = rep.Client(ctx) if err != nil {