Skip to content

Commit 565c30e

Browse files
bepbep
committedDec 16, 2024·
js: Fix js.Batch for multihost setups
Note that this is an unreleased feature. Fixes #13151
·
v0.146.5v0.140.0
1 parent 48dd6a9 commit 565c30e

File tree

8 files changed

+190
-71
lines changed

8 files changed

+190
-71
lines changed
 

Diff for: ‎deps/deps.go

+7
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/gohugoio/hugo/helpers"
2525
"github.com/gohugoio/hugo/hugofs"
2626
"github.com/gohugoio/hugo/identity"
27+
"github.com/gohugoio/hugo/internal/js"
2728
"github.com/gohugoio/hugo/internal/warpc"
2829
"github.com/gohugoio/hugo/media"
2930
"github.com/gohugoio/hugo/resources/page"
@@ -105,6 +106,12 @@ type Deps struct {
105106
// TODO(bep) rethink this re. a plugin setup, but this will have to do for now.
106107
WasmDispatchers *warpc.Dispatchers
107108

109+
// The JS batcher client.
110+
JSBatcherClient js.BatcherClient
111+
112+
// The JS batcher client.
113+
// JSBatcherClient *esbuild.BatcherClient
114+
108115
isClosed bool
109116

110117
*globalErrHandler

Diff for: ‎hugolib/paths/paths.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func New(fs *hugofs.Fs, cfg config.AllProvider) (*Paths, error) {
6767
var multihostTargetBasePaths []string
6868
if cfg.IsMultihost() && len(cfg.Languages()) > 1 {
6969
for _, l := range cfg.Languages() {
70-
multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang)
70+
multihostTargetBasePaths = append(multihostTargetBasePaths, hpaths.ToSlashPreserveLeading(l.Lang))
7171
}
7272
}
7373

Diff for: ‎hugolib/site.go

+7-1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ import (
4242
"github.com/gohugoio/hugo/deps"
4343
"github.com/gohugoio/hugo/hugolib/doctree"
4444
"github.com/gohugoio/hugo/hugolib/pagesfromdata"
45+
"github.com/gohugoio/hugo/internal/js/esbuild"
4546
"github.com/gohugoio/hugo/internal/warpc"
4647
"github.com/gohugoio/hugo/langs/i18n"
4748
"github.com/gohugoio/hugo/modules"
@@ -205,6 +206,12 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
205206
return nil, err
206207
}
207208

209+
batcherClient, err := esbuild.NewBatcherClient(firstSiteDeps)
210+
if err != nil {
211+
return nil, err
212+
}
213+
firstSiteDeps.JSBatcherClient = batcherClient
214+
208215
confm := cfg.Configs
209216
if err := confm.Validate(logger); err != nil {
210217
return nil, err
@@ -313,7 +320,6 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
313320
return li.Lang < lj.Lang
314321
})
315322

316-
var err error
317323
h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
318324
if err == nil && h == nil {
319325
panic("hugo: newHugoSitesNew returned nil error and nil HugoSites")

Diff for: ‎internal/js/api.go

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
// Copyright 2024 The Hugo Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
package js
15+
16+
import (
17+
"context"
18+
19+
"github.com/gohugoio/hugo/common/maps"
20+
"github.com/gohugoio/hugo/resources/resource"
21+
)
22+
23+
// BatcherClient is used to do JS batch operations.
24+
type BatcherClient interface {
25+
New(id string) (Batcher, error)
26+
Store() *maps.Cache[string, Batcher]
27+
}
28+
29+
// BatchPackage holds a group of JavaScript resources.
30+
type BatchPackage interface {
31+
Groups() map[string]resource.Resources
32+
}
33+
34+
// Batcher is used to build JavaScript packages.
35+
type Batcher interface {
36+
Build(context.Context) (BatchPackage, error)
37+
Config(ctx context.Context) OptionsSetter
38+
Group(ctx context.Context, id string) BatcherGroup
39+
}
40+
41+
// BatcherGroup is a group of scripts and instances.
42+
type BatcherGroup interface {
43+
Instance(sid, iid string) OptionsSetter
44+
Runner(id string) OptionsSetter
45+
Script(id string) OptionsSetter
46+
}
47+
48+
// OptionsSetter is used to set options for a batch, script or instance.
49+
type OptionsSetter interface {
50+
SetOptions(map[string]any) string
51+
}

Diff for: ‎internal/js/esbuild/batch.go

+56-49
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
_ "embed"
2121
"encoding/json"
2222
"fmt"
23+
"io"
2324
"path"
2425
"path/filepath"
2526
"reflect"
@@ -34,19 +35,20 @@ import (
3435
"github.com/gohugoio/hugo/common/maps"
3536
"github.com/gohugoio/hugo/common/paths"
3637
"github.com/gohugoio/hugo/deps"
38+
"github.com/gohugoio/hugo/helpers"
3739
"github.com/gohugoio/hugo/identity"
40+
"github.com/gohugoio/hugo/internal/js"
3841
"github.com/gohugoio/hugo/lazy"
3942
"github.com/gohugoio/hugo/media"
4043
"github.com/gohugoio/hugo/resources"
4144
"github.com/gohugoio/hugo/resources/resource"
4245
"github.com/gohugoio/hugo/resources/resource_factories/create"
4346
"github.com/gohugoio/hugo/tpl"
4447
"github.com/mitchellh/mapstructure"
45-
"github.com/spf13/afero"
4648
"github.com/spf13/cast"
4749
)
4850

49-
var _ Batcher = (*batcher)(nil)
51+
var _ js.Batcher = (*batcher)(nil)
5052

5153
const (
5254
NsBatch = "_hugo-js-batch"
@@ -58,7 +60,7 @@ const (
5860
//go:embed batch-esm-runner.gotmpl
5961
var runnerTemplateStr string
6062

61-
var _ BatchPackage = (*Package)(nil)
63+
var _ js.BatchPackage = (*Package)(nil)
6264

6365
var _ buildToucher = (*optsHolder[scriptOptions])(nil)
6466

@@ -67,16 +69,17 @@ var (
6769
_ isBuiltOrTouchedProvider = (*scriptGroup)(nil)
6870
)
6971

70-
func NewBatcherClient(deps *deps.Deps) (*BatcherClient, error) {
72+
func NewBatcherClient(deps *deps.Deps) (js.BatcherClient, error) {
7173
c := &BatcherClient{
7274
d: deps,
7375
buildClient: NewBuildClient(deps.BaseFs.Assets, deps.ResourceSpec),
7476
createClient: create.New(deps.ResourceSpec),
75-
bundlesCache: maps.NewCache[string, BatchPackage](),
77+
batcherStore: maps.NewCache[string, js.Batcher](),
78+
bundlesStore: maps.NewCache[string, js.BatchPackage](),
7679
}
7780

7881
deps.BuildEndListeners.Add(func(...any) bool {
79-
c.bundlesCache.Reset()
82+
c.bundlesStore.Reset()
8083
return false
8184
})
8285

@@ -125,7 +128,7 @@ func (o *opts[K, C]) Reset() {
125128
o.h.resetCounter++
126129
}
127130

128-
func (o *opts[K, C]) Get(id uint32) OptionsSetter {
131+
func (o *opts[K, C]) Get(id uint32) js.OptionsSetter {
129132
var b *optsHolder[C]
130133
o.once.Do(func() {
131134
b = o.h
@@ -184,18 +187,6 @@ func newOpts[K any, C optionsCompiler[C]](key K, optionsID string, defaults defa
184187
}
185188
}
186189

187-
// BatchPackage holds a group of JavaScript resources.
188-
type BatchPackage interface {
189-
Groups() map[string]resource.Resources
190-
}
191-
192-
// Batcher is used to build JavaScript packages.
193-
type Batcher interface {
194-
Build(context.Context) (BatchPackage, error)
195-
Config(ctx context.Context) OptionsSetter
196-
Group(ctx context.Context, id string) BatcherGroup
197-
}
198-
199190
// BatcherClient is a client for building JavaScript packages.
200191
type BatcherClient struct {
201192
d *deps.Deps
@@ -206,12 +197,13 @@ type BatcherClient struct {
206197
createClient *create.Client
207198
buildClient *BuildClient
208199

209-
bundlesCache *maps.Cache[string, BatchPackage]
200+
batcherStore *maps.Cache[string, js.Batcher]
201+
bundlesStore *maps.Cache[string, js.BatchPackage]
210202
}
211203

212204
// New creates a new Batcher with the given ID.
213205
// This will be typically created once and reused across rebuilds.
214-
func (c *BatcherClient) New(id string) (Batcher, error) {
206+
func (c *BatcherClient) New(id string) (js.Batcher, error) {
215207
var initErr error
216208
c.once.Do(func() {
217209
// We should fix the initialization order here (or use the Go template package directly), but we need to wait
@@ -288,6 +280,10 @@ func (c *BatcherClient) New(id string) (Batcher, error) {
288280
return b, nil
289281
}
290282

283+
func (c *BatcherClient) Store() *maps.Cache[string, js.Batcher] {
284+
return c.batcherStore
285+
}
286+
291287
func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTemplateContext) (resource.Resource, string, error) {
292288
var buf bytes.Buffer
293289

@@ -304,18 +300,6 @@ func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTempla
304300
return r, s, nil
305301
}
306302

307-
// BatcherGroup is a group of scripts and instances.
308-
type BatcherGroup interface {
309-
Instance(sid, iid string) OptionsSetter
310-
Runner(id string) OptionsSetter
311-
Script(id string) OptionsSetter
312-
}
313-
314-
// OptionsSetter is used to set options for a batch, script or instance.
315-
type OptionsSetter interface {
316-
SetOptions(map[string]any) string
317-
}
318-
319303
// Package holds a group of JavaScript resources.
320304
type Package struct {
321305
id string
@@ -353,9 +337,9 @@ type batcher struct {
353337
}
354338

355339
// Build builds the batch if not already built or if it's stale.
356-
func (b *batcher) Build(ctx context.Context) (BatchPackage, error) {
340+
func (b *batcher) Build(ctx context.Context) (js.BatchPackage, error) {
357341
key := dynacache.CleanKey(b.id + ".js")
358-
p, err := b.client.bundlesCache.GetOrCreate(key, func() (BatchPackage, error) {
342+
p, err := b.client.bundlesStore.GetOrCreate(key, func() (js.BatchPackage, error) {
359343
return b.build(ctx)
360344
})
361345
if err != nil {
@@ -364,11 +348,11 @@ func (b *batcher) Build(ctx context.Context) (BatchPackage, error) {
364348
return p, nil
365349
}
366350

367-
func (b *batcher) Config(ctx context.Context) OptionsSetter {
351+
func (b *batcher) Config(ctx context.Context) js.OptionsSetter {
368352
return b.configOptions.Get(b.buildCount)
369353
}
370354

371-
func (b *batcher) Group(ctx context.Context, id string) BatcherGroup {
355+
func (b *batcher) Group(ctx context.Context, id string) js.BatcherGroup {
372356
if err := ValidateBatchID(id, false); err != nil {
373357
panic(err)
374358
}
@@ -419,7 +403,7 @@ func (b *batcher) isStale() bool {
419403
return false
420404
}
421405

422-
func (b *batcher) build(ctx context.Context) (BatchPackage, error) {
406+
func (b *batcher) build(ctx context.Context) (js.BatchPackage, error) {
423407
b.mu.Lock()
424408
defer b.mu.Unlock()
425409
defer func() {
@@ -463,6 +447,8 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) {
463447
pathGroup: maps.NewCache[string, string](),
464448
}
465449

450+
multihostBasePaths := b.client.d.ResourceSpec.MultihostTargetBasePaths
451+
466452
// Entry points passed to ESBuid.
467453
var entryPoints []string
468454
addResource := func(group, pth string, r resource.Resource, isResult bool) {
@@ -701,15 +687,36 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) {
701687

702688
if !handled {
703689
// Copy to destination.
704-
p := strings.TrimPrefix(o.Path, outDir)
705-
targetFilename := filepath.Join(b.id, p)
706-
fs := b.client.d.BaseFs.PublishFs
707-
if err := fs.MkdirAll(filepath.Dir(targetFilename), 0o777); err != nil {
708-
return nil, fmt.Errorf("failed to create dir %q: %w", targetFilename, err)
690+
// In a multihost setup, we will have multiple targets.
691+
var targetFilenames []string
692+
if len(multihostBasePaths) > 0 {
693+
for _, base := range multihostBasePaths {
694+
p := strings.TrimPrefix(o.Path, outDir)
695+
targetFilename := filepath.Join(base, b.id, p)
696+
targetFilenames = append(targetFilenames, targetFilename)
697+
}
698+
} else {
699+
p := strings.TrimPrefix(o.Path, outDir)
700+
targetFilename := filepath.Join(b.id, p)
701+
targetFilenames = append(targetFilenames, targetFilename)
709702
}
710703

711-
if err := afero.WriteFile(fs, targetFilename, o.Contents, 0o666); err != nil {
712-
return nil, fmt.Errorf("failed to write to %q: %w", targetFilename, err)
704+
fs := b.client.d.BaseFs.PublishFs
705+
706+
if err := func() error {
707+
fw, err := helpers.OpenFilesForWriting(fs, targetFilenames...)
708+
if err != nil {
709+
return err
710+
}
711+
defer fw.Close()
712+
713+
fr := bytes.NewReader(o.Contents)
714+
715+
_, err = io.Copy(fw, fr)
716+
717+
return err
718+
}(); err != nil {
719+
return nil, fmt.Errorf("failed to copy to %q: %w", targetFilenames, err)
713720
}
714721
}
715722
}
@@ -845,7 +852,7 @@ type optionsGetSetter[K, C any] interface {
845852
Key() K
846853
Reset()
847854

848-
Get(uint32) OptionsSetter
855+
Get(uint32) js.OptionsSetter
849856
isStale() bool
850857
currPrev() (map[string]any, map[string]any)
851858
}
@@ -975,7 +982,7 @@ func (b *scriptGroup) IdentifierBase() string {
975982
return b.id
976983
}
977984

978-
func (s *scriptGroup) Instance(sid, id string) OptionsSetter {
985+
func (s *scriptGroup) Instance(sid, id string) js.OptionsSetter {
979986
if err := ValidateBatchID(sid, false); err != nil {
980987
panic(err)
981988
}
@@ -1014,7 +1021,7 @@ func (g *scriptGroup) Reset() {
10141021
}
10151022
}
10161023

1017-
func (s *scriptGroup) Runner(id string) OptionsSetter {
1024+
func (s *scriptGroup) Runner(id string) js.OptionsSetter {
10181025
if err := ValidateBatchID(id, false); err != nil {
10191026
panic(err)
10201027
}
@@ -1043,7 +1050,7 @@ func (s *scriptGroup) Runner(id string) OptionsSetter {
10431050
return s.runnersOptions[sid].Get(s.b.buildCount)
10441051
}
10451052

1046-
func (s *scriptGroup) Script(id string) OptionsSetter {
1053+
func (s *scriptGroup) Script(id string) js.OptionsSetter {
10471054
if err := ValidateBatchID(id, false); err != nil {
10481055
panic(err)
10491056
}

Diff for: ‎internal/js/esbuild/batch_integration_test.go

+63
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,69 @@ func TestBatchEditScriptParam(t *testing.T) {
184184
b.AssertFileContent("public/mybatch/mygroup.js", "param-p1-main-edited")
185185
}
186186

187+
func TestBatchMultiHost(t *testing.T) {
188+
files := `
189+
-- hugo.toml --
190+
disableKinds = ["taxonomy", "term", "section"]
191+
[languages]
192+
[languages.en]
193+
weight = 1
194+
baseURL = "https://example.com/en"
195+
[languages.fr]
196+
weight = 2
197+
baseURL = "https://example.com/fr"
198+
disableLiveReload = true
199+
-- assets/js/styles.css --
200+
body {
201+
background-color: red;
202+
}
203+
-- assets/js/main.js --
204+
import * as foo from 'mylib';
205+
console.log("Hello, Main!");
206+
-- assets/js/runner.js --
207+
console.log("Hello, Runner!");
208+
-- node_modules/mylib/index.js --
209+
console.log("Hello, My Lib!");
210+
-- layouts/index.html --
211+
Home.
212+
{{ $batch := (js.Batch "mybatch") }}
213+
{{ with $batch.Config }}
214+
{{ .SetOptions (dict
215+
"params" (dict "id" "config")
216+
"sourceMap" ""
217+
)
218+
}}
219+
{{ end }}
220+
{{ with (templates.Defer (dict "key" "global")) }}
221+
Defer:
222+
{{ $batch := (js.Batch "mybatch") }}
223+
{{ range $k, $v := $batch.Build.Groups }}
224+
{{ range $kk, $vv := . -}}
225+
{{ $k }}: {{ .RelPermalink }}
226+
{{ end }}
227+
{{ end -}}
228+
{{ end }}
229+
{{ $batch := (js.Batch "mybatch") }}
230+
{{ with $batch.Group "mygroup" }}
231+
{{ with .Runner "run" }}
232+
{{ .SetOptions (dict "resource" (resources.Get "js/runner.js")) }}
233+
{{ end }}
234+
{{ with .Script "main" }}
235+
{{ .SetOptions (dict "resource" (resources.Get "js/main.js") "params" (dict "p1" "param-p1-main" )) }}
236+
{{ end }}
237+
{{ with .Instance "main" "i1" }}
238+
{{ .SetOptions (dict "params" (dict "title" "Instance 1")) }}
239+
{{ end }}
240+
{{ end }}
241+
242+
243+
`
244+
b := hugolib.Test(t, files, hugolib.TestOptWithOSFs())
245+
b.AssertPublishDir(
246+
"en/mybatch/chunk-TOZKWCDE.js", "en/mybatch/mygroup.js ",
247+
"fr/mybatch/mygroup.js", "fr/mybatch/chunk-TOZKWCDE.js")
248+
}
249+
187250
func TestBatchRenameBundledScript(t *testing.T) {
188251
files := jsBatchFilesTemplate
189252
b := hugolib.TestRunning(t, files, hugolib.TestOptWithOSFs())

Diff for: ‎resources/resource.go

-7
Original file line numberDiff line numberDiff line change
@@ -141,13 +141,6 @@ func (fd *ResourceSourceDescriptor) init(r *Spec) error {
141141
}
142142

143143
fd.TargetPath = paths.ToSlashPreserveLeading(fd.TargetPath)
144-
for i, base := range fd.TargetBasePaths {
145-
dir := paths.ToSlashPreserveLeading(base)
146-
if dir == "/" {
147-
dir = ""
148-
}
149-
fd.TargetBasePaths[i] = dir
150-
}
151144

152145
if fd.NameNormalized == "" {
153146
fd.NameNormalized = fd.TargetPath

Diff for: ‎tpl/js/js.go

+5-13
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ package js
1717
import (
1818
"errors"
1919

20-
"github.com/gohugoio/hugo/common/maps"
2120
"github.com/gohugoio/hugo/deps"
21+
"github.com/gohugoio/hugo/internal/js"
2222
"github.com/gohugoio/hugo/internal/js/esbuild"
2323
"github.com/gohugoio/hugo/resources"
2424
"github.com/gohugoio/hugo/resources/resource"
@@ -34,16 +34,9 @@ func New(d *deps.Deps) (*Namespace, error) {
3434
return &Namespace{}, nil
3535
}
3636

37-
batcherClient, err := esbuild.NewBatcherClient(d)
38-
if err != nil {
39-
return nil, err
40-
}
41-
4237
return &Namespace{
4338
d: d,
4439
jsTransformClient: jstransform.New(d.BaseFs.Assets, d.ResourceSpec),
45-
jsBatcherClient: batcherClient,
46-
jsBatcherStore: maps.NewCache[string, esbuild.Batcher](),
4740
createClient: create.New(d.ResourceSpec),
4841
babelClient: babel.New(d.ResourceSpec),
4942
}, nil
@@ -56,8 +49,6 @@ type Namespace struct {
5649
jsTransformClient *jstransform.Client
5750
createClient *create.Client
5851
babelClient *babel.Client
59-
jsBatcherClient *esbuild.BatcherClient
60-
jsBatcherStore *maps.Cache[string, esbuild.Batcher]
6152
}
6253

6354
// Build processes the given Resource with ESBuild.
@@ -90,12 +81,13 @@ func (ns *Namespace) Build(args ...any) (resource.Resource, error) {
9081
// Repeated calls with the same ID will return the same Batcher.
9182
// The ID will be used to name the root directory of the batch.
9283
// Forward slashes in the ID is allowed.
93-
func (ns *Namespace) Batch(id string) (esbuild.Batcher, error) {
84+
func (ns *Namespace) Batch(id string) (js.Batcher, error) {
9485
if err := esbuild.ValidateBatchID(id, true); err != nil {
9586
return nil, err
9687
}
97-
b, err := ns.jsBatcherStore.GetOrCreate(id, func() (esbuild.Batcher, error) {
98-
return ns.jsBatcherClient.New(id)
88+
89+
b, err := ns.d.JSBatcherClient.Store().GetOrCreate(id, func() (js.Batcher, error) {
90+
return ns.d.JSBatcherClient.New(id)
9991
})
10092
if err != nil {
10193
return nil, err

0 commit comments

Comments
 (0)
Please sign in to comment.