Skip to content

Commit

Permalink
commands: Fix data race
Browse files Browse the repository at this point in the history 
By wrapping all use of the shared config in a lock.

Updates #10953
  • Loading branch information
@bep
bep committed May 19, 2023
1 parent c371171 commit 0a51dfa
Show file tree
Hide file tree
Showing 3 changed files with 213 additions and 153 deletions.
13 changes: 3 additions & 10 deletions commands/commandeer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,18 +79,12 @@ func Execute(args []string) error {
}

type commonConfig struct {
mu sync.Mutex
mu *sync.Mutex
configs *allconfig.Configs
cfg config.Provider
fs *hugofs.Fs
}

func (c *commonConfig) getFs() *hugofs.Fs {
c.mu.Lock()
defer c.mu.Unlock()
return c.fs
}

// This is the root command.
type rootCommand struct {
Printf func(format string, v ...interface{})
Expand Down Expand Up @@ -181,6 +175,7 @@ func (r *rootCommand) ConfigFromConfig(key int32, oldConf *commonConfig) (*commo
}

return &commonConfig{
mu: oldConf.mu,
configs: configs,
cfg: oldConf.cfg,
fs: fs,
Expand Down Expand Up @@ -295,6 +290,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo
}

commonConfig := &commonConfig{
mu: &sync.Mutex{},
configs: configs,
cfg: cfg,
fs: fs,
Expand All @@ -309,9 +305,6 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo

func (r *rootCommand) HugFromConfig(conf *commonConfig) (*hugolib.HugoSites, error) {
h, _, err := r.hugoSites.GetOrCreate(r.configVersionID.Load(), func(key int32) (*hugolib.HugoSites, error) {
conf.mu.Lock()
defer conf.mu.Unlock()

depsCfg := deps.DepsCfg{Configs: conf.configs, Fs: conf.fs, Logger: r.logger}
return hugolib.NewHugoSites(depsCfg)
})
Expand Down
147 changes: 91 additions & 56 deletions commands/hugobuilder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ import (
type hugoBuilder struct {
r *rootCommand

cunfMu sync.Mutex
conf_ *commonConfig
confmu sync.Mutex
conf *commonConfig

// May be nil.
s *serverCommand
Expand All @@ -74,16 +74,17 @@ type hugoBuilder struct {
errState hugoBuilderErrState
}

func (c *hugoBuilder) conf() *commonConfig {
c.cunfMu.Lock()
defer c.cunfMu.Unlock()
return c.conf_
func (c *hugoBuilder) withConfE(fn func(conf *commonConfig) error) error {
c.confmu.Lock()
defer c.confmu.Unlock()
return fn(c.conf)
}

func (c *hugoBuilder) setConf(conf *commonConfig) {
c.cunfMu.Lock()
defer c.cunfMu.Unlock()
c.conf_ = conf
func (c *hugoBuilder) withConf(fn func(conf *commonConfig)) {
c.confmu.Lock()
defer c.confmu.Unlock()
fn(c.conf)

}

type hugoBuilderErrState struct {
Expand Down Expand Up @@ -357,7 +358,10 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa

// Identifies changes to config (config.toml) files.
configSet := make(map[string]bool)
configFiles := c.conf().configs.LoadingInfo.ConfigFiles
var configFiles []string
c.withConf(func(conf *commonConfig) {
configFiles = conf.configs.LoadingInfo.ConfigFiles
})

c.r.logger.Println("Watching for config changes in", strings.Join(configFiles, ", "))
for _, configFile := range configFiles {
Expand Down Expand Up @@ -466,14 +470,18 @@ func (c *hugoBuilder) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint
fs := &countingStatFs{Fs: sourceFs.Fs}

syncer := fsync.NewSyncer()
syncer.NoTimes = c.conf().configs.Base.NoTimes
syncer.NoChmod = c.conf().configs.Base.NoChmod
syncer.ChmodFilter = chmodFilter
c.withConf(func(conf *commonConfig) {
syncer.NoTimes = conf.configs.Base.NoTimes
syncer.NoChmod = conf.configs.Base.NoChmod
syncer.ChmodFilter = chmodFilter

syncer.DestFs = conf.fs.PublishDirStatic
// Now that we are using a unionFs for the static directories
// We can effectively clean the publishDir on initial sync
syncer.Delete = conf.configs.Base.CleanDestinationDir
})

syncer.SrcFs = fs
syncer.DestFs = c.conf().fs.PublishDirStatic
// Now that we are using a unionFs for the static directories
// We can effectively clean the publishDir on initial sync
syncer.Delete = c.conf().configs.Base.CleanDestinationDir

if syncer.Delete {
c.r.logger.Infoln("removing all files from destination that don't exist in static dirs")
Expand Down Expand Up @@ -518,9 +526,11 @@ func (c *hugoBuilder) doWithPublishDirs(f func(sourceFs *filesystems.SourceFiles
}
if lang == "" {
// Not multihost
for _, l := range c.conf().configs.Languages {
langCount[l.Lang] = cnt
}
c.withConf(func(conf *commonConfig) {
for _, l := range conf.configs.Languages {
langCount[l.Lang] = cnt
}
})
} else {
langCount[lang] = cnt
}
Expand Down Expand Up @@ -562,7 +572,11 @@ func (c *hugoBuilder) fullBuild(noBuildLock bool) error {
// Do not copy static files and build sites in parallel if cleanDestinationDir is enabled.
// This flag deletes all static resources in /public folder that are missing in /static,
// and it does so at the end of copyStatic() call.
if c.conf().configs.Base.CleanDestinationDir {
var cleanDestinationDir bool
c.withConf(func(conf *commonConfig) {
cleanDestinationDir = conf.configs.Base.CleanDestinationDir
})
if cleanDestinationDir {
if err := copyStaticFunc(); err != nil {
return err
}
Expand Down Expand Up @@ -692,17 +706,18 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
}

if ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename {
configFiles := c.conf().configs.LoadingInfo.ConfigFiles
for _, configFile := range configFiles {
counter := 0
for watcher.Add(configFile) != nil {
counter++
if counter >= 100 {
break
c.withConf(func(conf *commonConfig) {
for _, configFile := range conf.configs.LoadingInfo.ConfigFiles {
counter := 0
for watcher.Add(configFile) != nil {
counter++
if counter >= 100 {
break
}
time.Sleep(100 * time.Millisecond)
}
time.Sleep(100 * time.Millisecond)
}
}
})
}

// Config file(s) changed. Need full rebuild.
Expand Down Expand Up @@ -834,9 +849,11 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
// recursively add new directories to watch list
// When mkdir -p is used, only the top directory triggers an event (at least on OSX)
if ev.Op&fsnotify.Create == fsnotify.Create {
if s, err := c.conf().fs.Source.Stat(ev.Name); err == nil && s.Mode().IsDir() {
_ = helpers.SymbolicWalk(c.conf().fs.Source, ev.Name, walkAdder)
}
c.withConf(func(conf *commonConfig) {
if s, err := conf.fs.Source.Stat(ev.Name); err == nil && s.Mode().IsDir() {
_ = helpers.SymbolicWalk(conf.fs.Source, ev.Name, walkAdder)
}
})
}

if staticSyncer.isStatic(h, ev.Name) {
Expand Down Expand Up @@ -942,10 +959,16 @@ func (c *hugoBuilder) handleEvents(watcher *watcher.Batcher,
}

func (c *hugoBuilder) hugo() (*hugolib.HugoSites, error) {
h, err := c.r.HugFromConfig(c.conf())
if err != nil {
var h *hugolib.HugoSites
if err := c.withConfE(func(conf *commonConfig) error {
var err error
h, err = c.r.HugFromConfig(conf)
return err

}); err != nil {
return nil, err
}

if c.s != nil {
// A running server, register the media types.
for _, s := range h.Sites {
Expand All @@ -956,7 +979,10 @@ func (c *hugoBuilder) hugo() (*hugolib.HugoSites, error) {
}

func (c *hugoBuilder) hugoTry() *hugolib.HugoSites {
h, _ := c.r.HugFromConfig(c.conf())
var h *hugolib.HugoSites
c.withConf(func(conf *commonConfig) {
h, _ = c.r.HugFromConfig(conf)
})
return h
}

Expand All @@ -977,7 +1003,7 @@ func (c *hugoBuilder) loadConfig(cd *simplecobra.Commandeer, running bool) error
return err
}

c.setConf(conf)
c.conf = conf
if c.onConfigLoaded != nil {
if err := c.onConfigLoaded(false); err != nil {
return err
Expand Down Expand Up @@ -1014,34 +1040,43 @@ func (c *hugoBuilder) rebuildSites(events []fsnotify.Event) error {
return err
}
if c.fastRenderMode {
// Make sure we always render the home pages
for _, l := range c.conf().configs.Languages {
langPath := h.GetLangSubDir(l.Lang)
if langPath != "" {
langPath = langPath + "/"
c.withConf(func(conf *commonConfig) {
// Make sure we always render the home pages
for _, l := range conf.configs.Languages {
langPath := h.GetLangSubDir(l.Lang)
if langPath != "" {
langPath = langPath + "/"
}
home := h.PrependBasePath("/"+langPath, false)
visited[home] = true
}
home := h.PrependBasePath("/"+langPath, false)
visited[home] = true
}
})
}
return h.Build(hugolib.BuildCfg{NoBuildLock: true, RecentlyVisited: visited, ErrRecovery: c.errState.wasErr()}, events...)
}

func (c *hugoBuilder) reloadConfig() error {
c.r.Reset()
c.r.configVersionID.Add(1)
oldConf := c.conf()
conf, err := c.r.ConfigFromConfig(c.r.configVersionID.Load(), c.conf())
if err != nil {
return err
}
sameLen := len(oldConf.configs.Languages) == len(conf.configs.Languages)
if !sameLen {
if oldConf.configs.IsMultihost || conf.configs.IsMultihost {
return errors.New("multihost change detected, please restart server")

if err := c.withConfE(func(conf *commonConfig) error {
oldConf := conf
newConf, err := c.r.ConfigFromConfig(c.r.configVersionID.Load(), conf)
if err != nil {
return err
}
sameLen := len(oldConf.configs.Languages) == len(newConf.configs.Languages)
if !sameLen {
if oldConf.configs.IsMultihost || newConf.configs.IsMultihost {
return errors.New("multihost change detected, please restart server")
}
}
c.conf = newConf
return nil
}); err != nil {
return err
}
c.setConf(conf)

if c.onConfigLoaded != nil {
if err := c.onConfigLoaded(true); err != nil {
return err
Expand Down

0 comments on commit 0a51dfa

Please sign in to comment.