lsp: Poll workspace state to detect state changes (#954)

* WIP: attempt to use fsnotify reliably

Signed-off-by: Charlie Egan <charlie@styra.com>

* Revert "WIP: attempt to use fsnotify reliably"

This reverts commit f0bce94.

I am going to use a simple polling approach.

* lsp: Poll workspace state to detect state changes

Fixes #857

I will add some notes to the issue too, but in short I found that it
wasn't possible to catch new files added to new directories without some
polling-based delay. So I have opted for the keep it simple approach for
now.

Signed-off-by: Charlie Egan <charlie@styra.com>

---------

Signed-off-by: Charlie Egan <charlie@styra.com>

Author: charlieegan3

cmd/languageserver.go

@@ -44,6 +44,7 @@ func init() {
 			go ls.StartHoverWorker(ctx)
 			go ls.StartCommandWorker(ctx)
 			go ls.StartConfigWorker(ctx)
+			go ls.StartWorkspaceStateWorker(ctx)
 
 			sigChan := make(chan os.Signal, 1)
 			signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)

internal/lsp/cache/cache.go

@@ -330,20 +330,20 @@ func (c *Cache) Delete(fileURI string) {
 	c.ignoredFileContentsMu.Unlock()
 }
 
-func UpdateCacheForURIFromDisk(cache *Cache, fileURI, path string) (string, error) {
+func UpdateCacheForURIFromDisk(cache *Cache, fileURI, path string) (bool, string, error) {
 	content, err := os.ReadFile(path)
 	if err != nil {
-		return "", fmt.Errorf("failed to read file: %w", err)
+		return false, "", fmt.Errorf("failed to read file: %w", err)
 	}
 
 	currentContent := string(content)
 
 	cachedContent, ok := cache.GetFileContents(fileURI)
 	if ok && cachedContent == currentContent {
-		return cachedContent, nil
+		return false, cachedContent, nil
 	}
 
 	cache.SetFileContents(fileURI, currentContent)
 
-	return currentContent, nil
+	return true, currentContent, nil
 }

internal/lsp/server.go

@@ -196,7 +196,8 @@ func (l *LanguageServer) StartDiagnosticsWorker(ctx context.Context) {
 			return
 		case evt := <-l.diagnosticRequestFile:
 			// if file has been deleted, clear diagnostics in the client
-			if evt.Reason == "textDocument/didDelete" {
+			if evt.Reason == "textDocument/didDelete" ||
+				evt.Reason == "internal/workspaceStateWorker/missingFile" {
 				err := l.sendFileDiagnostics(ctx, evt.URI)
 				if err != nil {
 					l.logError(fmt.Errorf("failed to send diagnostic: %w", err))
@@ -460,6 +461,72 @@ func (l *LanguageServer) StartCommandWorker(ctx context.Context) {
 	}
 }
 
+// StartWorkspaceStateWorker will poll for changes to the workspaces state that
+// are not sent from the client. For example, when a file a is removed from the
+// workspace after changing branch.
+func (l *LanguageServer) StartWorkspaceStateWorker(ctx context.Context) {
+	timer := time.NewTicker(2 * time.Second)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			// first clear files that are missing from the workspaceDir
+			for fileURI := range l.cache.GetAllFiles() {
+				filePath := uri.ToPath(l.clientIdentifier, fileURI)
+
+				_, err := os.Stat(filePath)
+				if !os.IsNotExist(err) {
+					// if the file is not missing, we have no work to do
+					continue
+				}
+
+				// if the diagnostics for the file are empty, or missing
+				// then we do not need to send anything to the client and can
+				// remove the file from the cache.
+				diagnostics, ok := l.cache.GetFileDiagnostics(fileURI)
+				if !ok || len(diagnostics) == 0 {
+					l.cache.Delete(fileURI)
+
+					continue
+				}
+
+				// if there are diagnostics, we need to clear them and send a
+				// notification to the client.
+				l.cache.SetFileDiagnostics(fileURI, []types.Diagnostic{})
+				l.diagnosticRequestFile <- fileUpdateEvent{
+					URI:    fileURI,
+					Reason: "internal/workspaceStateWorker/missingFile",
+				}
+			}
+
+			// for this next operation, the workspace root must be set as it's
+			// used to scan for new files.
+			if l.workspaceRootURI == "" {
+				continue
+			}
+
+			// next, check if there are any new files that are not ignored and
+			// need to be loaded. We get new only so that files being worked
+			// on are not loaded from disk during editing.
+			changedOrNewURIs, err := l.loadWorkspaceContents(ctx, true)
+			if err != nil {
+				l.logError(fmt.Errorf("failed to refresh workspace contents: %w", err))
+
+				continue
+			}
+
+			for _, uri := range changedOrNewURIs {
+				l.diagnosticRequestFile <- fileUpdateEvent{
+					URI:    uri,
+					Reason: "internal/workspaceStateWorker/changedOrNewFile",
+				}
+			}
+		}
+	}
+}
+
 func (l *LanguageServer) fixEditParams(
 	label string,
 	fix fixes.Fix,
@@ -1271,7 +1338,7 @@ func (l *LanguageServer) handleWorkspaceDidCreateFiles(
 	}
 
 	for _, createOp := range params.Files {
-		_, err = cache.UpdateCacheForURIFromDisk(
+		_, _, err = cache.UpdateCacheForURIFromDisk(
 			l.cache,
 			uri.FromPath(l.clientIdentifier, createOp.URI),
 			uri.ToPath(l.clientIdentifier, createOp.URI),
@@ -1335,7 +1402,7 @@ func (l *LanguageServer) handleWorkspaceDidRenameFiles(
 			continue
 		}
 
-		content, err := cache.UpdateCacheForURIFromDisk(
+		_, content, err := cache.UpdateCacheForURIFromDisk(
 			l.cache,
 			uri.FromPath(l.clientIdentifier, renameOp.NewURI),
 			uri.ToPath(l.clientIdentifier, renameOp.NewURI),
@@ -1475,7 +1542,7 @@ func (l *LanguageServer) handleInitialize(
 			l.configWatcher.Watch(configFile.Name())
 		}
 
-		err = l.loadWorkspaceContents(ctx)
+		_, err = l.loadWorkspaceContents(ctx, false)
 		if err != nil {
 			return nil, fmt.Errorf("failed to load workspace contents: %w", err)
 		}
@@ -1486,9 +1553,11 @@ func (l *LanguageServer) handleInitialize(
 	return result, nil
 }
 
-func (l *LanguageServer) loadWorkspaceContents(ctx context.Context) error {
+func (l *LanguageServer) loadWorkspaceContents(ctx context.Context, newOnly bool) ([]string, error) {
 	workspaceRootPath := uri.ToPath(l.clientIdentifier, l.workspaceRootURI)
 
+	changedOrNewURIs := make([]string, 0)
+
 	err := filepath.WalkDir(workspaceRootPath, func(path string, d os.DirEntry, err error) error {
 		if err != nil {
 			return fmt.Errorf("failed to walk workspace dir %q: %w", path, err)
@@ -1505,23 +1574,36 @@ func (l *LanguageServer) loadWorkspaceContents(ctx context.Context) error {
 			return nil
 		}
 
-		_, err = cache.UpdateCacheForURIFromDisk(l.cache, fileURI, path)
+		// if the caller has requested only new files, then we can exit early
+		if _, ok := l.cache.GetModule(fileURI); newOnly && ok {
+			return nil
+		}
+
+		changed, _, err := cache.UpdateCacheForURIFromDisk(l.cache, fileURI, path)
 		if err != nil {
 			return fmt.Errorf("failed to update cache for uri %q: %w", path, err)
 		}
 
+		// there is no need to update the parse if the file contents
+		// was not changed in the above operation.
+		if !changed {
+			return nil
+		}
+
 		_, err = updateParse(ctx, l.cache, l.regoStore, fileURI)
 		if err != nil {
 			return fmt.Errorf("failed to update parse: %w", err)
 		}
 
+		changedOrNewURIs = append(changedOrNewURIs, fileURI)
+
 		return nil
 	})
 	if err != nil {
-		return fmt.Errorf("failed to walk workspace dir %q: %w", workspaceRootPath, err)
+		return nil, fmt.Errorf("failed to walk workspace dir %q: %w", workspaceRootPath, err)
 	}
 
-	return nil
+	return changedOrNewURIs, nil
 }
 
 func (l *LanguageServer) handleInitialized(