Skip to content

Commit

Permalink
staging put: stream file uploads instead of loading all to memory
Browse files Browse the repository at this point in the history
os.ReadFile reads all of the content of the file into a byte array in
memory, which can cause memory consumption pressure for users. Instead,
an os.File instance is itself a byte reader, and we can provide the file
directly to http.NewRequest so it can read the file in chunks and upload
it as a stream, thus not holding the whole file in memory.
  • Loading branch information
mdibaiee committed Mar 28, 2024
1 parent d70ab7c commit 6e608ad
Showing 1 changed file with 3 additions and 4 deletions.
7 changes: 3 additions & 4 deletions connection.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package dbsql

import (
"bytes"
"context"
"database/sql/driver"
"encoding/json"
Expand Down Expand Up @@ -433,13 +432,13 @@ func (c *conn) handleStagingPut(ctx context.Context, presignedUrl string, header
}
client := &http.Client{}

dat, err := os.ReadFile(localFile)

dat, err := os.Open(localFile)
if err != nil {
return dbsqlerrint.NewDriverError(ctx, "error reading local file", err)
}
defer dat.Close()

req, _ := http.NewRequest("PUT", presignedUrl, bytes.NewReader(dat))
req, _ := http.NewRequest("PUT", presignedUrl, dat)

for k, v := range headers {
req.Header.Set(k, v)
Expand Down

0 comments on commit 6e608ad

Please sign in to comment.