use fetchup to download

.github/workflows/test-linux.yml

@@ -21,7 +21,7 @@ jobs:
 
       - uses: actions/setup-go@v4
         with:
-          go-version: 1.18
+          go-version: 1.20.1
 
       - uses: actions/checkout@v3
 

browser_test.go

@@ -285,7 +285,7 @@ func TestBinarySize(t *testing.T) {
 	stat, err := os.Stat("tmp/translator")
 	g.E(err)
 
-	g.Lte(float64(stat.Size())/1024/1024, 10.6) // mb
+	g.Lte(float64(stat.Size())/1024/1024, 11) // mb
 }
 
 func TestBrowserCookies(t *testing.T) {

cspell.json

@@ -38,6 +38,7 @@
         "enctype",
         "evenodd",
         "excludesfile",
+        "fetchup",
         "fontconfig",
         "Fullscreen",
         "Geolocation",
@@ -97,12 +98,12 @@
         "srgb",
         "staticcheck",
         "stdlib",
+        "termux",
         "tlid",
         "touchend",
         "touchstart",
         "tracebackancestors",
         "trimpath",
-        "termux",
         "Typedarray",
         "tzdata",
         "Unserializable",

go.mod

@@ -3,8 +3,9 @@ module github.com/go-rod/rod
 go 1.16
 
 require (
+	github.com/ysmood/fetchup v0.2.1
 	github.com/ysmood/goob v0.4.0
-	github.com/ysmood/got v0.33.0
+	github.com/ysmood/got v0.33.2
 	github.com/ysmood/gotrace v0.6.0
 	github.com/ysmood/gson v0.7.3
 	github.com/ysmood/leakless v0.8.0

go.sum

@@ -1,10 +1,12 @@
+github.com/ysmood/fetchup v0.2.1 h1:n/NgIx92KOXFiKAhK3d+LlKpl8JuSjh5U27ULmHKtag=
+github.com/ysmood/fetchup v0.2.1/go.mod h1:94ROLWpn5fmCD4LPlcZ+LOE/iE/kRTU3kL+0ue/V+Os=
 github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
 github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
 github.com/ysmood/gop v0.0.1 h1:EGxcy28xzMfjT+TlLf8JQd1w/sTz0XPrbHazLLs84BA=
 github.com/ysmood/gop v0.0.1/go.mod h1:JenWOSCGucsXYHdw1Cw4a0fmLnux7jXBDdlBGjh8V/g=
 github.com/ysmood/got v0.32.0/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
-github.com/ysmood/got v0.33.0 h1:E0XJ5/mI2wdCCjXkqJ2d96GdOEYFwe2ou0FqHJf6zr4=
-github.com/ysmood/got v0.33.0/go.mod h1:P3C/Wwttv4uq/tpovaH+c8ANmHePyFPxEbNzdxcEGDU=
+github.com/ysmood/got v0.33.2 h1:mz0PaCMzR//YBtDDkDf6z0O09SfotXBHzw3zLrrS2sw=
+github.com/ysmood/got v0.33.2/go.mod h1:P3C/Wwttv4uq/tpovaH+c8ANmHePyFPxEbNzdxcEGDU=
 github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
 github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
 github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=

go.work.sum

@@ -1,13 +1,4 @@
-github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
-github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
-github.com/ysmood/gop v0.0.1 h1:EGxcy28xzMfjT+TlLf8JQd1w/sTz0XPrbHazLLs84BA=
-github.com/ysmood/gop v0.0.1/go.mod h1:JenWOSCGucsXYHdw1Cw4a0fmLnux7jXBDdlBGjh8V/g=
-github.com/ysmood/got v0.32.0/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
-github.com/ysmood/got v0.33.0 h1:E0XJ5/mI2wdCCjXkqJ2d96GdOEYFwe2ou0FqHJf6zr4=
-github.com/ysmood/got v0.33.0/go.mod h1:P3C/Wwttv4uq/tpovaH+c8ANmHePyFPxEbNzdxcEGDU=
-github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
-github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
-github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
-github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
-github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak=
-github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
+github.com/ysmood/fetchup v0.2.1 h1:n/NgIx92KOXFiKAhK3d+LlKpl8JuSjh5U27ULmHKtag=
+github.com/ysmood/fetchup v0.2.1/go.mod h1:94ROLWpn5fmCD4LPlcZ+LOE/iE/kRTU3kL+0ue/V+Os=
+github.com/ysmood/got v0.33.2 h1:mz0PaCMzR//YBtDDkDf6z0O09SfotXBHzw3zLrrS2sw=
+github.com/ysmood/got v0.33.2/go.mod h1:P3C/Wwttv4uq/tpovaH+c8ANmHePyFPxEbNzdxcEGDU=

lib/launcher/browser.go

@@ -3,24 +3,19 @@ package launcher
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
-	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
-	"sync"
 
 	"github.com/go-rod/rod/lib/defaults"
 	"github.com/go-rod/rod/lib/utils"
+	"github.com/ysmood/fetchup"
 	"github.com/ysmood/leakless"
 )
 
@@ -88,20 +83,17 @@ type Browser struct {
 	// Revision of the browser to use
 	Revision int
 
-	// Dir to download browser.
-	Dir string
+	// RootDir to download different browser versions.
+	RootDir string
 
 	// Log to print output
 	Logger utils.Logger
 
 	// LockPort a tcp port to prevent race downloading. Default is 2968 .
 	LockPort int
 
-	// Proxy to use (http/socks5). Default is nil.
-	proxyURL *url.URL
-
-	// IgnoreCerts skips proxy certificate validation
-	IgnoreCerts bool
+	// HTTPClient to download the browser
+	HTTPClient *http.Client
 }
 
 // NewBrowser with default values
@@ -110,155 +102,50 @@ func NewBrowser() *Browser {
 		Context:  context.Background(),
 		Revision: RevisionDefault,
 		Hosts:    []Host{HostGoogle, HostNPM, HostPlaywright},
-		Dir:      DefaultBrowserDir,
+		RootDir:  DefaultBrowserDir,
 		Logger:   log.New(os.Stdout, "[launcher.Browser]", log.LstdFlags),
 		LockPort: defaults.LockPort,
 	}
 }
 
-// Destination of the downloaded browser executable
-func (lc *Browser) Destination() string {
+// Dir to download the browser
+func (lc *Browser) Dir() string {
+	return filepath.Join(lc.RootDir, fmt.Sprintf("chromium-%d", lc.Revision))
+}
+
+// BinPath to download the browser executable
+func (lc *Browser) BinPath() string {
 	bin := map[string]string{
-		"darwin":  fmt.Sprintf("chromium-%d/chrome-mac/Chromium.app/Contents/MacOS/Chromium", lc.Revision),
-		"linux":   fmt.Sprintf("chromium-%d/chrome-linux/chrome", lc.Revision),
-		"windows": fmt.Sprintf("chromium-%d/chrome-win/chrome.exe", lc.Revision),
+		"darwin":  "Chromium.app/Contents/MacOS/Chromium",
+		"linux":   "chrome",
+		"windows": "chrome.exe",
 	}[runtime.GOOS]
 
-	return filepath.Join(lc.Dir, bin)
+	return filepath.Join(lc.Dir(), filepath.FromSlash(bin))
 }
 
 // Download browser from the fastest host. It will race downloading a TCP packet from each host and use the fastest host.
-func (lc *Browser) Download() (err error) {
-	defer func() {
-		if e := recover(); e != nil {
-			err = e.(error)
-		}
-	}()
-
-	u, err := lc.fastestHost()
-	utils.E(err)
-
-	if u == "" {
-		panic(fmt.Errorf("Can't find a browser binary for your OS, the doc might help https://go-rod.github.io/#/compatibility?id=os"))
-	}
-
-	return lc.download(lc.Context, u)
-}
-
-// Proxy sets the proxy for chrome download
-func (lc *Browser) Proxy(URL string) error {
-	proxyURL, err := url.Parse(URL)
-	if err != nil {
-		return err
-	}
-	lc.proxyURL = proxyURL
-	return err
-}
-
-func (lc *Browser) fastestHost() (fastest string, err error) {
-	lc.Logger.Println("try to find the fastest host to download the browser binary")
-
-	setURL := sync.Once{}
-	ctx, cancel := context.WithCancel(lc.Context)
-	defer cancel()
-
-	wg := sync.WaitGroup{}
+func (lc *Browser) Download() error {
+	us := []string{}
 	for _, host := range lc.Hosts {
-		u := host(lc.Revision)
-
-		lc.Logger.Println("check", u)
-		wg.Add(1)
-
-		go func() {
-			defer func() {
-				err := recover()
-				if err != nil {
-					lc.Logger.Println("check result:", err)
-				}
-				wg.Done()
-			}()
-
-			q, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
-			utils.E(err)
-
-			res, err := lc.httpClient().Do(q)
-			utils.E(err)
-			defer func() { _ = res.Body.Close() }()
-
-			if res.StatusCode == http.StatusOK {
-				buf := make([]byte, 64*1024) // a TCP packet won't be larger than 64KB
-				_, err = res.Body.Read(buf)
-				utils.E(err)
-
-				setURL.Do(func() {
-					fastest = u
-					cancel()
-				})
-			}
-		}()
+		us = append(us, host(lc.Revision))
 	}
-	wg.Wait()
-
-	return
-}
-
-func (lc *Browser) download(ctx context.Context, u string) error {
-	lc.Logger.Println("Download:", u)
 
-	zipPath := filepath.Join(lc.Dir, fmt.Sprintf("chromium-%d.zip", lc.Revision))
+	dir := lc.Dir()
 
-	err := utils.Mkdir(lc.Dir)
-	utils.E(err)
-
-	zipFile, err := os.Create(zipPath)
-	utils.E(err)
-
-	q, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
-	utils.E(err)
-
-	res, err := lc.httpClient().Do(q)
-	utils.E(err)
-	defer func() { _ = res.Body.Close() }()
-
-	size, _ := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
-
-	if res.StatusCode >= 400 || size < 1024*1024 {
-		b, err := ioutil.ReadAll(res.Body)
-		utils.E(err)
-		err = errors.New("failed to download the browser")
-		return fmt.Errorf("%w: %d %s", err, res.StatusCode, string(b))
+	fu := fetchup.New(dir, us...)
+	fu.Ctx = lc.Context
+	fu.Logger = lc.Logger
+	if lc.HTTPClient != nil {
+		fu.HttpClient = lc.HTTPClient
 	}
 
-	progress := &progresser{
-		size:   int(size),
-		logger: lc.Logger,
+	err := fu.Fetch()
+	if err != nil {
+		return fmt.Errorf("Can't find a browser binary for your OS, the doc might help https://go-rod.github.io/#/compatibility?id=os : %w", err)
 	}
 
-	_, err = io.Copy(io.MultiWriter(progress, zipFile), res.Body)
-	utils.E(err)
-
-	err = zipFile.Close()
-	utils.E(err)
-
-	unzipPath := filepath.Join(lc.Dir, fmt.Sprintf("chromium-%d", lc.Revision))
-	_ = os.RemoveAll(unzipPath)
-	utils.E(unzip(lc.Logger, zipPath, unzipPath))
-	return os.Remove(zipPath)
-}
-
-func (lc *Browser) httpClient() *http.Client {
-	transport := &http.Transport{
-		DisableKeepAlives: true,
-	}
-	if lc.IgnoreCerts {
-		transport.TLSClientConfig = &tls.Config{
-			InsecureSkipVerify: true,
-		}
-	}
-	if lc.proxyURL != nil {
-		transport.Proxy = http.ProxyURL(lc.proxyURL)
-	}
-	return &http.Client{Transport: transport}
+	return fetchup.StripFirstDir(dir)
 }
 
 // Get is a smart helper to get the browser executable path.
@@ -267,10 +154,13 @@ func (lc *Browser) Get() (string, error) {
 	defer leakless.LockPort(lc.LockPort)()
 
 	if lc.Validate() == nil {
-		return lc.Destination(), nil
+		return lc.BinPath(), nil
 	}
 
-	return lc.Destination(), lc.Download()
+	// Try to cleanup before downloading
+	_ = os.RemoveAll(lc.Dir())
+
+	return lc.BinPath(), lc.Download()
 }
 
 // MustGet is similar with Get
@@ -283,12 +173,12 @@ func (lc *Browser) MustGet() string {
 // Validate returns nil if the browser executable valid.
 // If the executable is malformed it will return error.
 func (lc *Browser) Validate() error {
-	_, err := os.Stat(lc.Destination())
+	_, err := os.Stat(lc.BinPath())
 	if err != nil {
 		return err
 	}
 
-	cmd := exec.Command(lc.Destination(), "--headless", "--no-sandbox",
+	cmd := exec.Command(lc.BinPath(), "--headless", "--no-sandbox",
 		"--disable-gpu", "--dump-dom", "about:blank")
 	b, err := cmd.CombinedOutput()
 	if err != nil {

lib/launcher/launcher.go

@@ -303,7 +303,6 @@ func (l *Launcher) RemoteDebuggingPort(port int) *Launcher {
 
 // Proxy for the browser
 func (l *Launcher) Proxy(host string) *Launcher {
-	_ = l.browser.Proxy(host)
 	return l.Set(flags.ProxyServer, host)
 }