Add NewBufferedWatcher()

.circleci/config.yml

@@ -23,7 +23,8 @@ jobs:
           command: |
             uname -a
             go version
-            go test -parallel 1 -race ./...
+            FSNOTIFY_BUFFER=4096 go test -parallel 1 -race ./...
+                                 go test -parallel 1 -race ./...
 
   # iOS
   ios:
@@ -48,7 +49,8 @@ jobs:
             export PATH=$PATH:/usr/local/Cellar/go/*/bin
             uname -a
             go version
-            go test -parallel 1 -race ./...
+            FSNOTIFY_BUFFER=4096 go test -parallel 1 -race ./...
+                                 go test -parallel 1 -race ./...
 
   # This is just Linux x86_64; also need to get a Go with GOOS=android, but
   # there aren't any pre-built versions of that on the Go site. Idk, disable for
@@ -76,5 +78,6 @@ jobs:
   #           uname -a
   #           export PATH=/usr/local/go/bin:$PATH
   #           go version
-  #           go test -parallel 1 -race ./...
+  #           FSNOTIFY_BUFFER=4096 go test -parallel 1 -race ./...
+  #                                go test -parallel 1 -race ./...
   #

.cirrus.yml

@@ -9,4 +9,5 @@ freebsd_task:
       # run tests as user "cirrus" instead of root
     - pw useradd cirrus -m
     - chown -R cirrus:cirrus .
-    - sudo -u cirrus go test -parallel 1 -race ./...
+    - FSNOTIFY_BUFFER=4096 sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./...
+    -                      sudo --preserve-env=FSNOTIFY_BUFFER -u cirrus go test -parallel 1 -race ./...

.github/workflows/test.yml

@@ -6,71 +6,74 @@ on:
     branches: ['main', 'aix']
 
 jobs:
-  # Test Windows and Linux with the latest Go version and the oldest we support.
-  test:
+  linux:
     strategy:
       fail-fast: false
       matrix:
-        os:
-          - ubuntu-latest
-          - windows-latest
-        go:
-          - '1.17'
-          - '1.21'
+        os: ['ubuntu-latest']
+        go: ['1.17', '1.21']
     runs-on: ${{ matrix.os }}
     steps:
-      - name: checkout
-        uses: actions/checkout@v3
-
-      - name: setup Go
-        uses: actions/setup-go@v4
+      - uses: 'actions/checkout@v3'
+      - uses: 'actions/setup-go@v4'
         with:
           go-version: ${{ matrix.go }}
+      - name: test
+        run: |
+          FSNOTIFY_BUFFER=4096 go test -parallel 1 -race ./...
+                               go test -parallel 1 -race ./...
 
+  windows:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['windows-latest']
+        go: ['1.17', '1.21']
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: 'actions/checkout@v3'
+      - uses: 'actions/setup-go@v4'
+        with:
+          go-version: ${{ matrix.go }}
       - name: test
         run: |
           go test -parallel 1 -race ./...
+          set "FSNOTIFY_BUFFER=4096"
+          go test -parallel 1 -race ./...
 
   # Test gccgo
-  testgcc:
-    runs-on: ubuntu-22.04
-    name: test (ubuntu-22.04, gccgo 12.1)
+  gcc:
+    runs-on: 'ubuntu-22.04'
+    name:    'test (ubuntu-22.04, gccgo 12.1)'
     steps:
-      - name: checkout
-        uses: actions/checkout@v3
-
+      - uses: 'actions/checkout@v3'
       - name: test
         run: |
           sudo apt-get -y install gccgo-12
-          go-12 test -parallel 1 ./...
+          FSNOTIFY_BUFFER=4096 go-12 test -parallel 1 ./...
+                               go-12 test -parallel 1 ./...
 
   # Test only the latest Go version on macOS; we use the macOS builders for BSD
   # and illumos, and GitHub doesn't allow many of them to run concurrently. If
   # it works on Windows and Linux with Go 1.17, then it probably does on macOS
   # too.
-  testMacOS:
+  macos:
     name: test
     strategy:
       fail-fast: false
       matrix:
-        os:
-          - macos-11
-          - macos-13
-        go:
-          - '1.21'
+        os: ['macos-11', 'macos-13']
+        go: ['1.21']
     runs-on: ${{ matrix.os }}
     steps:
-      - name: checkout
-        uses: actions/checkout@v3
-
-      - name: setup Go
-        uses: actions/setup-go@v4
+      - uses: 'actions/checkout@v3'
+      - uses: 'actions/setup-go@v4'
         with:
           go-version: ${{ matrix.go }}
-
       - name: test
         run: |
-          go test -parallel 1 -race ./...
+          FSNOTIFY_BUFFER=4096 go test -parallel 1 -race ./...
+                               go test -parallel 1 -race ./...
 
   # OpenBSD; no -race as the VM doesn't include the comp set.
   #
@@ -79,57 +82,64 @@ jobs:
   #       so should probably look into that first. Go 1.19 is supposed to have a
   #       much faster race detector, so maybe waiting until we have that is
   #       enough.
-  testOpenBSD:
-    runs-on: macos-12
-    name: test (openbsd, 1.17)
+  openbsd:
+    runs-on: 'macos-12'
+    timeout-minutes: 30
+    name: 'test (openbsd, 1.17)'
     steps:
-      - uses: actions/checkout@v3
-      - name: test (openbsd, 1.17)
-        id: test
-        uses: vmactions/openbsd-vm@v0
+      - uses: 'actions/checkout@v3'
+      - name: 'test (openbsd, 1.17)'
+        id:   'openbsd'
+        uses: 'vmactions/openbsd-vm@v0'
         with:
           prepare: pkg_add go
           run: |
             useradd -mG wheel action
-            su action -c 'go test -parallel 1 ./...'
+            FSNOTIFY_BUFFER=4096 su action -c 'go test -parallel 1 ./...'
+                                 su action -c 'go test -parallel 1 ./...'
 
   # NetBSD
-  testNetBSD:
+  netbsd:
     runs-on: macos-12
+    timeout-minutes: 30
     name: test (netbsd, 1.20)
     steps:
-      - uses: actions/checkout@v3
-      - name: test (netbsd, 1.20)
-        id: test
-        uses: vmactions/netbsd-vm@v0
+      - uses: 'actions/checkout@v3'
+      - name: 'test (netbsd, 1.20)'
+        id:   'netbsd'
+        uses: 'vmactions/netbsd-vm@v0'
         with:
           prepare: pkg_add go
           # TODO: no -race for the same reason as OpenBSD (the timing; it does run).
           run: |
             useradd -mG wheel action
-            su action -c 'go120 test -parallel 1 ./...'
+            FSNOTIFY_BUFFER=4096 su action -c 'go120 test -parallel 1 ./...'
+                                 su action -c 'go120 test -parallel 1 ./...'
 
   # illumos
-  testillumos:
+  illumos:
     runs-on: macos-12
+    timeout-minutes: 30
     name: test (illumos, 1.19)
     steps:
-    - uses: actions/checkout@v3
-    - name: test (illumos, 1.19)
-      id: test
-      uses: papertigers/illumos-vm@r38
+    - uses: 'actions/checkout@v3'
+    - name: 'test (illumos, 1.19)'
+      id:   'illumos'
+      uses: 'papertigers/illumos-vm@r38'
       with:
         prepare: |
           pkg install go-119
         run: |
           useradd action
           export GOCACHE=/tmp/go-cache
           export GOPATH=/tmp/go-path
-          su action -c '/opt/ooce/go-1.19/bin/go test -parallel 1 ./...'
+          FSNOTIFY_BUFFER=4096 su action -c '/opt/ooce/go-1.19/bin/go test -parallel 1 ./...'
+                               su action -c '/opt/ooce/go-1.19/bin/go test -parallel 1 ./...'
 
   # Older Debian 6, for old Linux kernels.
-  testDebian6:
+  debian6:
     runs-on: macos-12
+    timeout-minutes: 30
     name: test (debian6, 1.19)
     strategy:
       fail-fast: false
@@ -149,16 +159,18 @@ jobs:
         with:
           go-version: '1.19'
 
-      - name: test (debian6, 1.19)
-        id: test
+      - name: 'test (debian6, 1.19)'
+        id:   'debian6'
         run: |
           cp -f .github/workflows/Vagrantfile.debian6 Vagrantfile
           export GOOS=linux
           export GOARCH=amd64
           for p in $(go list ./...); do
-            go test -c -o ${p//\//-}.test $p
+            FSNOTIFY_BUFFER=4096 go test -c -o ${p//\//-}.test $p
+                                 go test -c -o ${p//\//-}.test $p
           done
           vagrant up
           for t in *.test; do
-            vagrant ssh -c "/vagrant/$t -test.parallel 1"
+            FSNOTIFY_BUFFER=4096 vagrant ssh -c "/vagrant/$t -test.parallel 1"
+                                 vagrant ssh -c "/vagrant/$t -test.parallel 1"
           done

CHANGELOG.md

@@ -8,12 +8,17 @@ This version of fsnotify needs Go 1.17.
 
 - illumos: add FEN backend to support illumos and Solaris. ([#371])
 
+- all: add `NewBufferedWatcher()` to use a buffered channel, which can be useful
+  in cases where you can't control the kernel buffer and receive a large number
+  of events in bursts. ([#550], [#572])
+
 - all: add `AddWith()`, which is identical to `Add()` but allows passing
   options. ([#521])
 
-- windows: allow setting the buffer size with `fsnotify.WithBufferSize()`; the
-  default of 64K is the highest value that works on all platforms and is enough
-  for most purposes, but in some cases a highest buffer is needed. ([#521])
+- windows: allow setting the ReadDirectoryChangesW() buffer size with
+  `fsnotify.WithBufferSize()`; the default of 64K is the highest value that
+  works on all platforms and is enough for most purposes, but in some cases a
+  highest buffer is needed. ([#521])
 
 ### Changes and fixes
 
@@ -57,7 +62,6 @@ This version of fsnotify needs Go 1.17.
   Google AppEngine forbids usage of the unsafe package so the inotify backend
   won't compile there.
 
-
 [#371]: https://github.com/fsnotify/fsnotify/pull/371
 [#516]: https://github.com/fsnotify/fsnotify/pull/516
 [#518]: https://github.com/fsnotify/fsnotify/pull/518
@@ -67,6 +71,8 @@ This version of fsnotify needs Go 1.17.
 [#526]: https://github.com/fsnotify/fsnotify/pull/526
 [#528]: https://github.com/fsnotify/fsnotify/pull/528
 [#537]: https://github.com/fsnotify/fsnotify/pull/537
+[#550]: https://github.com/fsnotify/fsnotify/pull/550
+[#572]: https://github.com/fsnotify/fsnotify/pull/572
 
 1.6.0 - 2022-10-13
 -------------------

backend_fen.go

@@ -77,10 +77,10 @@ import (
 // Sometimes it will send events for all times, sometimes it will send no
 // events, and often only for some files. 
 //
-// The default buffer size is 64K, which is the largest value that is guaranteed
-// to work with SMB filesystems. If you have many events in quick succession
-// this may not be enough, and you will have to use [WithBufferSize] to increase
-// the value.
+// The default ReadDirectoryChangesW() buffer size is 64K, which is the largest
+// value that is guaranteed to work with SMB filesystems. If you have many
+// events in quick succession this may not be enough, and you will have to use
+// [WithBufferSize] to increase the value.
 type Watcher struct {
 	// Events sends the filesystem change events.
 	//
@@ -139,8 +139,19 @@ type Watcher struct {
 
 // NewWatcher creates a new Watcher.
 func NewWatcher() (*Watcher, error) {
+	return NewBufferedWatcher(0)
+}
+
+// NewBufferedWatcher creates a new Watcher with a buffered [Events] channel.
+//
+// The main use-case for this is situations with a very large number of events
+// where the kernel buffer size can't be increased (e.g. due to lack of
+// permissions). An unbuffered Watcher will perform better for almost all use
+// cases, and whenever possible you will be better off increasing the kernel
+// buffers instead of adding a large userspace buffer.
+func NewBufferedWatcher(sz uint) (*Watcher, error) {
 	w := &Watcher{
-		Events:  make(chan Event),
+		Events:  make(chan Event, sz),
 		Errors:  make(chan error),
 		dirs:    make(map[string]struct{}),
 		watches: make(map[string]struct{}),

backend_inotify.go

@@ -80,10 +80,10 @@ import (
 // Sometimes it will send events for all times, sometimes it will send no
 // events, and often only for some files. 
 //
-// The default buffer size is 64K, which is the largest value that is guaranteed
-// to work with SMB filesystems. If you have many events in quick succession
-// this may not be enough, and you will have to use [WithBufferSize] to increase
-// the value.
+// The default ReadDirectoryChangesW() buffer size is 64K, which is the largest
+// value that is guaranteed to work with SMB filesystems. If you have many
+// events in quick succession this may not be enough, and you will have to use
+// [WithBufferSize] to increase the value.
 type Watcher struct {
 	// Events sends the filesystem change events.
 	//
@@ -238,6 +238,17 @@ func (w *watches) updatePath(path string, f func(*watch) (*watch, error)) error
 
 // NewWatcher creates a new Watcher.
 func NewWatcher() (*Watcher, error) {
+	return NewBufferedWatcher(0)
+}
+
+// NewBufferedWatcher creates a new Watcher with a buffered [Events] channel.
+//
+// The main use-case for this is situations with a very large number of events
+// where the kernel buffer size can't be increased (e.g. due to lack of
+// permissions). An unbuffered Watcher will perform better for almost all use
+// cases, and whenever possible you will be better off increasing the kernel
+// buffers instead of adding a large userspace buffer.
+func NewBufferedWatcher(sz uint) (*Watcher, error) {
 	// Need to set nonblocking mode for SetDeadline to work, otherwise blocking
 	// I/O operations won't terminate on close.
 	fd, errno := unix.InotifyInit1(unix.IN_CLOEXEC | unix.IN_NONBLOCK)
@@ -249,7 +260,7 @@ func NewWatcher() (*Watcher, error) {
 		fd:          fd,
 		inotifyFile: os.NewFile(uintptr(fd), ""),
 		watches:     newWatches(),
-		Events:      make(chan Event),
+		Events:      make(chan Event, sz),
 		Errors:      make(chan error),
 		done:        make(chan struct{}),
 		doneResp:    make(chan struct{}),