use fetchup to download

cspell.json

@@ -38,6 +38,7 @@
         "enctype",
         "evenodd",
         "excludesfile",
+        "fetchup",
         "fontconfig",
         "Fullscreen",
         "Geolocation",
@@ -97,12 +98,12 @@
         "srgb",
         "staticcheck",
         "stdlib",
+        "termux",
         "tlid",
         "touchend",
         "touchstart",
         "tracebackancestors",
         "trimpath",
-        "termux",
         "Typedarray",
         "tzdata",
         "Unserializable",

go.mod

@@ -3,6 +3,7 @@ module github.com/go-rod/rod
 go 1.16
 
 require (
+	github.com/ysmood/fetchup v0.2.0
 	github.com/ysmood/goob v0.4.0
 	github.com/ysmood/got v0.33.0
 	github.com/ysmood/gotrace v0.6.0

go.sum

@@ -1,3 +1,5 @@
+github.com/ysmood/fetchup v0.2.0 h1:dHNF2D4nfiGq0doxsXtr02WezZs7k7yzBciu9UzIc44=
+github.com/ysmood/fetchup v0.2.0/go.mod h1:b5DtW4kZ7L5UokuXfT4QKEZBCubUQXKcwbX2PMNFzRg=
 github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
 github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
 github.com/ysmood/gop v0.0.1 h1:EGxcy28xzMfjT+TlLf8JQd1w/sTz0XPrbHazLLs84BA=

lib/launcher/browser.go

@@ -6,21 +6,18 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
-	"sync"
 
 	"github.com/go-rod/rod/lib/defaults"
 	"github.com/go-rod/rod/lib/utils"
+	"github.com/ysmood/fetchup"
 	"github.com/ysmood/leakless"
 )
 
@@ -119,30 +116,34 @@ func NewBrowser() *Browser {
 // Destination of the downloaded browser executable
 func (lc *Browser) Destination() string {
 	bin := map[string]string{
-		"darwin":  fmt.Sprintf("chromium-%d/chrome-mac/Chromium.app/Contents/MacOS/Chromium", lc.Revision),
-		"linux":   fmt.Sprintf("chromium-%d/chrome-linux/chrome", lc.Revision),
-		"windows": fmt.Sprintf("chromium-%d/chrome-win/chrome.exe", lc.Revision),
+		"darwin":  fmt.Sprintf("chromium-%d/Chromium.app/Contents/MacOS/Chromium", lc.Revision),
+		"linux":   fmt.Sprintf("chromium-%d/chrome", lc.Revision),
+		"windows": fmt.Sprintf("chromium-%d/chrome.exe", lc.Revision),
 	}[runtime.GOOS]
 
 	return filepath.Join(lc.Dir, bin)
 }
 
 // Download browser from the fastest host. It will race downloading a TCP packet from each host and use the fastest host.
-func (lc *Browser) Download() (err error) {
-	defer func() {
-		if e := recover(); e != nil {
-			err = e.(error)
-		}
-	}()
+func (lc *Browser) Download() error {
+	us := []string{}
+	for _, host := range lc.Hosts {
+		us = append(us, host(lc.Revision))
+	}
 
-	u, err := lc.fastestHost()
-	utils.E(err)
+	unzipPath := filepath.Join(lc.Dir, fmt.Sprintf("chromium-%d", lc.Revision))
+	_ = os.RemoveAll(unzipPath)
+
+	fu := fetchup.New(unzipPath, us...)
+	fu.Ctx = lc.Context
+	fu.HttpClient = lc.httpClient()
 
-	if u == "" {
-		panic(fmt.Errorf("Can't find a browser binary for your OS, the doc might help https://go-rod.github.io/#/compatibility?id=os"))
+	err := fu.Fetch()
+	if err != nil {
+		return fmt.Errorf("Can't find a browser binary for your OS, the doc might help https://go-rod.github.io/#/compatibility?id=os : %w", err)
 	}
 
-	return lc.download(lc.Context, u)
+	return fetchup.StripFirstDir(unzipPath)
 }
 
 // Proxy sets the proxy for chrome download
@@ -155,97 +156,6 @@ func (lc *Browser) Proxy(URL string) error {
 	return err
 }
 
-func (lc *Browser) fastestHost() (fastest string, err error) {
-	lc.Logger.Println("try to find the fastest host to download the browser binary")
-
-	setURL := sync.Once{}
-	ctx, cancel := context.WithCancel(lc.Context)
-	defer cancel()
-
-	wg := sync.WaitGroup{}
-	for _, host := range lc.Hosts {
-		u := host(lc.Revision)
-
-		lc.Logger.Println("check", u)
-		wg.Add(1)
-
-		go func() {
-			defer func() {
-				err := recover()
-				if err != nil {
-					lc.Logger.Println("check result:", err)
-				}
-				wg.Done()
-			}()
-
-			q, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
-			utils.E(err)
-
-			res, err := lc.httpClient().Do(q)
-			utils.E(err)
-			defer func() { _ = res.Body.Close() }()
-
-			if res.StatusCode == http.StatusOK {
-				buf := make([]byte, 64*1024) // a TCP packet won't be larger than 64KB
-				_, err = res.Body.Read(buf)
-				utils.E(err)
-
-				setURL.Do(func() {
-					fastest = u
-					cancel()
-				})
-			}
-		}()
-	}
-	wg.Wait()
-
-	return
-}
-
-func (lc *Browser) download(ctx context.Context, u string) error {
-	lc.Logger.Println("Download:", u)
-
-	zipPath := filepath.Join(lc.Dir, fmt.Sprintf("chromium-%d.zip", lc.Revision))
-
-	err := utils.Mkdir(lc.Dir)
-	utils.E(err)
-
-	zipFile, err := os.Create(zipPath)
-	utils.E(err)
-
-	q, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
-	utils.E(err)
-
-	res, err := lc.httpClient().Do(q)
-	utils.E(err)
-	defer func() { _ = res.Body.Close() }()
-
-	size, _ := strconv.ParseInt(res.Header.Get("Content-Length"), 10, 64)
-
-	if res.StatusCode >= 400 || size < 1024*1024 {
-		b, err := ioutil.ReadAll(res.Body)
-		utils.E(err)
-		err = errors.New("failed to download the browser")
-		return fmt.Errorf("%w: %d %s", err, res.StatusCode, string(b))
-	}
-
-	progress := &progresser{
-		size:   int(size),
-		logger: lc.Logger,
-	}
-
-	_, err = io.Copy(io.MultiWriter(progress, zipFile), res.Body)
-	utils.E(err)
-
-	err = zipFile.Close()
-	utils.E(err)
-
-	unzipPath := filepath.Join(lc.Dir, fmt.Sprintf("chromium-%d", lc.Revision))
-	_ = os.RemoveAll(unzipPath)
-	utils.E(unzip(lc.Logger, zipPath, unzipPath))
-	return os.Remove(zipPath)
-}
-
 func (lc *Browser) httpClient() *http.Client {
 	transport := &http.Transport{
 		DisableKeepAlives: true,

lib/launcher/launcher_test.go

@@ -1,26 +1,17 @@
 package launcher_test
 
 import (
-	"archive/zip"
-	"bytes"
-	"context"
 	"crypto"
 	"crypto/x509"
 	"encoding/pem"
 	"flag"
-	"fmt"
-	"io"
 	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"net/http/httputil"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/go-rod/rod/lib/defaults"
 	"github.com/go-rod/rod/lib/launcher"
@@ -39,65 +30,6 @@ func TestDownloadHosts(t *testing.T) {
 	g.Has(launcher.HostPlaywright(launcher.RevisionDefault), "https://playwright.azureedge.net/")
 }
 
-func TestDownload(t *testing.T) {
-	g := setup(t)
-
-	s := g.Serve()
-	s.Mux.HandleFunc("/fast/", func(rw http.ResponseWriter, r *http.Request) {
-		buf := bytes.NewBuffer(nil)
-		zw := zip.NewWriter(buf)
-
-		// folder "to"
-		h := &zip.FileHeader{Name: "to/"}
-		h.SetMode(0755)
-		_, err := zw.CreateHeader(h)
-		g.E(err)
-
-		// file "file.txt"
-		w, err := zw.CreateHeader(&zip.FileHeader{Name: "to/file.txt"})
-		g.E(err)
-		b := []byte(g.RandStr(2 * 1024 * 1024))
-		g.E(w.Write(b))
-
-		g.E(zw.Close())
-
-		rw.Header().Add("Content-Length", fmt.Sprintf("%d", buf.Len()))
-		_, _ = io.Copy(rw, buf)
-	})
-	s.Mux.HandleFunc("/slow/", func(rw http.ResponseWriter, r *http.Request) {
-		t := time.NewTimer(3 * time.Second)
-		select {
-		case <-t.C:
-		case <-r.Context().Done():
-			t.Stop()
-		}
-	})
-
-	b, cancel := newBrowser()
-	b.Logger = utils.LoggerQuiet
-	defer cancel()
-
-	b.Hosts = []launcher.Host{launcher.HostTest(s.URL("/slow")), launcher.HostTest(s.URL("/fast"))}
-	b.Dir = filepath.Join("tmp", "browser-from-mirror", g.RandStr(16))
-	g.E(b.Download())
-	g.Nil(os.Stat(b.Dir))
-
-	// download chrome with a proxy
-	// should fail with self signed certificate
-	p := httptest.NewTLSServer(&httputil.ReverseProxy{Director: func(_ *http.Request) {}})
-	defer p.Close()
-	// invalid proxy URL should trigger an error
-	err := b.Proxy(`invalid.escaping%%2`)
-	g.Eq(err.Error(), `parse "invalid.escaping%%2": invalid URL escape "%%2"`)
-
-	g.E(b.Proxy(p.URL))
-	g.NotNil(b.Download())
-	// should instead be successful with ignore certificate
-	b.IgnoreCerts = true
-	g.E(b.Download())
-	g.Nil(os.Stat(b.Dir))
-}
-
 func TestBrowserGet(t *testing.T) {
 	g := setup(t)
 
@@ -239,16 +171,6 @@ func TestLaunchErr(t *testing.T) {
 	}
 }
 
-func newBrowser() (*launcher.Browser, func()) {
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	b := launcher.NewBrowser()
-	if !testing.Verbose() {
-		b.Logger = utils.LoggerQuiet
-	}
-	b.Context = ctx
-	return b, cancel
-}
-
 var testProfileDir = flag.Bool("test-profile-dir", false, "set it to test profile dir")
 
 func TestProfileDir(t *testing.T) {

lib/launcher/private_test.go

@@ -57,12 +57,6 @@ func TestToWS(t *testing.T) {
 	g.Eq("ws", toWS(*u).Scheme)
 }
 
-func TestUnzip(t *testing.T) {
-	g := setup(t)
-
-	g.Err(unzip(utils.LoggerQuiet, "", ""))
-}
-
 func TestLaunchOptions(t *testing.T) {
 	g := setup(t)
 
@@ -150,16 +144,6 @@ func TestLaunchErrs(t *testing.T) {
 	g.Err(err)
 }
 
-func TestProgresser(t *testing.T) {
-	g := setup(t)
-
-	p := progresser{size: 100, logger: utils.LoggerQuiet}
-
-	g.E(p.Write(make([]byte, 100)))
-	g.E(p.Write(make([]byte, 100)))
-	g.E(p.Write(make([]byte, 100)))
-}
-
 func TestURLParserErr(t *testing.T) {
 	g := setup(t)
 
@@ -176,11 +160,10 @@ func TestURLParserErr(t *testing.T) {
 
 func TestBrowserDownloadErr(t *testing.T) {
 	g := setup(t)
-
-	r := g.Serve().Route("/", "", "")
 	b := NewBrowser()
 	b.Logger = utils.LoggerQuiet
-	g.Has(b.download(g.Context(), r.URL()).Error(), "failed to download the browser: 200")
+	b.Hosts = []Host{}
+	g.Err(b.Download())
 }
 
 func TestTestOpen(_ *testing.T) {