Added Logpush Output Options support

cloudflare · Dec 19, 2023 · fbe7244 · fbe7244
1 parent 8df5e95
commit fbe7244
Show file tree

Hide file tree

Showing 2 changed files with 144 additions and 45 deletions.
diff --git a/logpush.go b/logpush.go
@@ -12,22 +12,23 @@ import (
 
 // LogpushJob describes a Logpush job.
 type LogpushJob struct {
-	ID                       int                `json:"id,omitempty"`
-	Dataset                  string             `json:"dataset"`
-	Enabled                  bool               `json:"enabled"`
-	Kind                     string             `json:"kind,omitempty"`
-	Name                     string             `json:"name"`
-	LogpullOptions           string             `json:"logpull_options"`
-	DestinationConf          string             `json:"destination_conf"`
-	OwnershipChallenge       string             `json:"ownership_challenge,omitempty"`
-	LastComplete             *time.Time         `json:"last_complete,omitempty"`
-	LastError                *time.Time         `json:"last_error,omitempty"`
-	ErrorMessage             string             `json:"error_message,omitempty"`
-	Frequency                string             `json:"frequency,omitempty"`
-	Filter                   *LogpushJobFilters `json:"filter,omitempty"`
-	MaxUploadBytes           int                `json:"max_upload_bytes,omitempty"`
-	MaxUploadRecords         int                `json:"max_upload_records,omitempty"`
-	MaxUploadIntervalSeconds int                `json:"max_upload_interval_seconds,omitempty"`
+	ID                       int                   `json:"id,omitempty"`
+	Dataset                  string                `json:"dataset"`
+	Enabled                  bool                  `json:"enabled"`
+	Kind                     string                `json:"kind,omitempty"`
+	Name                     string                `json:"name"`
+	LogpullOptions           string                `json:"logpull_options,omitempty"`
+	OutputOptions            *LogpushOutputOptions `json:"output_options,omitempty"`
+	DestinationConf          string                `json:"destination_conf"`
+	OwnershipChallenge       string                `json:"ownership_challenge,omitempty"`
+	LastComplete             *time.Time            `json:"last_complete,omitempty"`
+	LastError                *time.Time            `json:"last_error,omitempty"`
+	ErrorMessage             string                `json:"error_message,omitempty"`
+	Frequency                string                `json:"frequency,omitempty"`
+	Filter                   *LogpushJobFilters    `json:"filter,omitempty"`
+	MaxUploadBytes           int                   `json:"max_upload_bytes,omitempty"`
+	MaxUploadRecords         int                   `json:"max_upload_records,omitempty"`
+	MaxUploadIntervalSeconds int                   `json:"max_upload_interval_seconds,omitempty"`
 }
 
 type LogpushJobFilters struct {
@@ -63,6 +64,21 @@ type LogpushJobFilter struct {
 	Value    interface{} `json:"value,omitempty"`
 }
 
+type LogpushOutputOptions struct {
+	FieldNames      []string `json:"field_names"`
+	OutputType      string   `json:"output_type,omitempty"`
+	BatchPrefix     string   `json:"batch_prefix,omitempty"`
+	BatchSuffix     string   `json:"batch_suffix,omitempty"`
+	RecordPrefix    string   `json:"record_prefix,omitempty"`
+	RecordSuffix    string   `json:"record_suffix,omitempty"`
+	RecordTemplate  string   `json:"record_template,omitempty"`
+	RecordDelimiter string   `json:"record_delimiter,omitempty"`
+	FieldDelimiter  string   `json:"field_delimiter,omitempty"`
+	TimestampFormat string   `json:"timestamp_format,omitempty"`
+	SampleRate      float64  `json:"sample_rate,omitempty"`
+	CVE202144228    bool     `json:"CVE-2021-44228"`
+}
+
 // LogpushJobsResponse is the API response, containing an array of Logpush Jobs.
 type LogpushJobsResponse struct {
 	Response
@@ -323,19 +339,20 @@ func (filter *LogpushJobFilter) Validate() error {
 }
 
 type CreateLogpushJobParams struct {
-	Dataset                  string             `json:"dataset"`
-	Enabled                  bool               `json:"enabled"`
-	Kind                     string             `json:"kind,omitempty"`
-	Name                     string             `json:"name"`
-	LogpullOptions           string             `json:"logpull_options"`
-	DestinationConf          string             `json:"destination_conf"`
-	OwnershipChallenge       string             `json:"ownership_challenge,omitempty"`
-	ErrorMessage             string             `json:"error_message,omitempty"`
-	Frequency                string             `json:"frequency,omitempty"`
-	Filter                   *LogpushJobFilters `json:"filter,omitempty"`
-	MaxUploadBytes           int                `json:"max_upload_bytes,omitempty"`
-	MaxUploadRecords         int                `json:"max_upload_records,omitempty"`
-	MaxUploadIntervalSeconds int                `json:"max_upload_interval_seconds,omitempty"`
+	Dataset                  string                `json:"dataset"`
+	Enabled                  bool                  `json:"enabled"`
+	Kind                     string                `json:"kind,omitempty"`
+	Name                     string                `json:"name"`
+	LogpullOptions           string                `json:"logpull_options,omitempty"`
+	OutputOptions            *LogpushOutputOptions `json:"output_options,omitempty"`
+	DestinationConf          string                `json:"destination_conf"`
+	OwnershipChallenge       string                `json:"ownership_challenge,omitempty"`
+	ErrorMessage             string                `json:"error_message,omitempty"`
+	Frequency                string                `json:"frequency,omitempty"`
+	Filter                   *LogpushJobFilters    `json:"filter,omitempty"`
+	MaxUploadBytes           int                   `json:"max_upload_bytes,omitempty"`
+	MaxUploadRecords         int                   `json:"max_upload_records,omitempty"`
+	MaxUploadIntervalSeconds int                   `json:"max_upload_interval_seconds,omitempty"`
 }
 
 type ListLogpushJobsParams struct{}
@@ -349,22 +366,23 @@ type GetLogpushFieldsParams struct {
 }
 
 type UpdateLogpushJobParams struct {
-	ID                       int                `json:"-"`
-	Dataset                  string             `json:"dataset"`
-	Enabled                  bool               `json:"enabled"`
-	Kind                     string             `json:"kind,omitempty"`
-	Name                     string             `json:"name"`
-	LogpullOptions           string             `json:"logpull_options"`
-	DestinationConf          string             `json:"destination_conf"`
-	OwnershipChallenge       string             `json:"ownership_challenge,omitempty"`
-	LastComplete             *time.Time         `json:"last_complete,omitempty"`
-	LastError                *time.Time         `json:"last_error,omitempty"`
-	ErrorMessage             string             `json:"error_message,omitempty"`
-	Frequency                string             `json:"frequency,omitempty"`
-	Filter                   *LogpushJobFilters `json:"filter,omitempty"`
-	MaxUploadBytes           int                `json:"max_upload_bytes,omitempty"`
-	MaxUploadRecords         int                `json:"max_upload_records,omitempty"`
-	MaxUploadIntervalSeconds int                `json:"max_upload_interval_seconds,omitempty"`
+	ID                       int                   `json:"-"`
+	Dataset                  string                `json:"dataset"`
+	Enabled                  bool                  `json:"enabled"`
+	Kind                     string                `json:"kind,omitempty"`
+	Name                     string                `json:"name"`
+	LogpullOptions           string                `json:"logpull_options,omitempty"`
+	OutputOptions            *LogpushOutputOptions `json:"output_options,omitempty"`
+	DestinationConf          string                `json:"destination_conf"`
+	OwnershipChallenge       string                `json:"ownership_challenge,omitempty"`
+	LastComplete             *time.Time            `json:"last_complete,omitempty"`
+	LastError                *time.Time            `json:"last_error,omitempty"`
+	ErrorMessage             string                `json:"error_message,omitempty"`
+	Frequency                string                `json:"frequency,omitempty"`
+	Filter                   *LogpushJobFilters    `json:"filter,omitempty"`
+	MaxUploadBytes           int                   `json:"max_upload_bytes,omitempty"`
+	MaxUploadRecords         int                   `json:"max_upload_records,omitempty"`
+	MaxUploadIntervalSeconds int                   `json:"max_upload_interval_seconds,omitempty"`
 }
 
 type ValidateLogpushOwnershipChallengeParams struct {

diff --git a/logpush_test.go b/logpush_test.go
@@ -32,6 +32,28 @@ const (
 	"frequency": "high",
 	"max_upload_bytes": 5000000
   }
+`
+	serverLogpushJobWithOutputOptionsDescription = `{
+	"id": %d,
+	"dataset": "http_requests",
+	"kind": "",
+	"enabled": false,
+	"name": "example.com",
+	"output_options": {
+		"field_names":[
+			"RayID",
+			"ClientIP",
+			"EdgeStartTimestamp"
+		],
+		"timestamp_format": "rfc3339"
+	},
+	"destination_conf": "s3://mybucket/logs?region=us-west-2",
+	"last_complete": "%[2]s",
+	"last_error": "%[2]s",
+	"error_message": "test",
+	"frequency": "high",
+	"max_upload_bytes": 5000000
+  }
 `
 	serverEdgeLogpushJobDescription = `{
 	"id": %d,
@@ -76,6 +98,26 @@ var (
 		Frequency:       "high",
 		MaxUploadBytes:  5000000,
 	}
+	expectedLogpushJobWithOutputOptionsStruct = LogpushJob{
+		ID:      jobID,
+		Dataset: "http_requests",
+		Enabled: false,
+		Name:    "example.com",
+		OutputOptions: &LogpushOutputOptions{
+			FieldNames: []string{
+				"RayID",
+				"ClientIP",
+				"EdgeStartTimestamp",
+			},
+			TimestampFormat: "rfc3339",
+		},
+		DestinationConf: "s3://mybucket/logs?region=us-west-2",
+		LastComplete:    &testLogpushTimestamp,
+		LastError:       &testLogpushTimestamp,
+		ErrorMessage:    "test",
+		Frequency:       "high",
+		MaxUploadBytes:  5000000,
+	}
 	expectedEdgeLogpushJobStruct = LogpushJob{
 		ID:              jobID,
 		Dataset:         "http_requests",
@@ -138,6 +180,10 @@ func TestGetLogpushJob(t *testing.T) {
 			result: serverLogpushJobDescription,
 			want:   expectedLogpushJobStruct,
 		},
+		"core logpush job with output options": {
+			result: serverLogpushJobWithOutputOptionsDescription,
+			want:   expectedLogpushJobWithOutputOptionsStruct,
+		},
 		"edge logpush job": {
 			result: serverEdgeLogpushJobDescription,
 			want:   expectedEdgeLogpushJobStruct,
@@ -198,6 +244,41 @@ func TestCreateLogpushJob(t *testing.T) {
 			result: serverLogpushJobDescription,
 			want:   expectedLogpushJobStruct,
 		},
+		"core logpush job with output options": {
+			newJob: CreateLogpushJobParams{
+				Dataset: "http_requests",
+				Enabled: false,
+				Name:    "example.com",
+				OutputOptions: &LogpushOutputOptions{
+					FieldNames: []string{
+						"RayID",
+						"ClientIP",
+						"EdgeStartTimestamp",
+					},
+					TimestampFormat: "rfc3339",
+				},
+				DestinationConf:  "s3://mybucket/logs?region=us-west-2",
+				MaxUploadRecords: 1000,
+			},
+			payload: `{
+				"dataset": "http_requests",
+				"enabled":false,
+				"name":"example.com",
+				"output_options": {
+					"field_names":[
+						"RayID",
+						"ClientIP",
+						"EdgeStartTimestamp"
+					],
+					"timestamp_format": "rfc3339",
+					"CVE-2021-44228": false
+				},
+				"destination_conf":"s3://mybucket/logs?region=us-west-2",
+				"max_upload_records": 1000
+			}`,
+			result: serverLogpushJobWithOutputOptionsDescription,
+			want:   expectedLogpushJobWithOutputOptionsStruct,
+		},
 		"edge logpush job": {
 			newJob: CreateLogpushJobParams{
 				Dataset:         "http_requests",