internal/lsp: fix deadlocks loading lots of files at once

The first deadlock involved differing mutex acquisition order in two
code paths:

1. loadParseTypecheck() holds the "mcache" mutex then eventually
   acquires the "handleMu" file mutex.
2. (*goFile).invalidateContent() acquires the "handleMu" mutex first and
   then the "mcache" mutex.

Fix by changing the acquisition order in invalidateContent().

The second deadlock involved the file watcher. The two code paths
involved were:

1. (*goFile).GetPackages() holds the view mutex and eventually gets
   (*WatchMap).Watch, which acquires the watcher mutex.
2. (*session).openOverlay acquires the watcher mutex as it triggers a
   file's callbacks, and then the callback
   "(*goFile).invalidateContent" acquires the vie mutex.

Fix by not holding the watcher mutex as we invoke the callbacks.

Author: muirdm

internal/lsp/cache/view.go

@@ -225,15 +225,17 @@ func (v *view) SetContent(ctx context.Context, uri span.URI, content []byte) err
 // invalidateContent invalidates the content of a Go file,
 // including any position and type information that depends on it.
 func (f *goFile) invalidateContent(ctx context.Context) {
-	f.handleMu.Lock()
-	defer f.handleMu.Unlock()
-
+	// Mutex acquisition order here is important. It must match the order
+	// in loadParseTypecheck to avoid deadlocks.
 	f.view.mcache.mu.Lock()
 	defer f.view.mcache.mu.Unlock()
 
 	f.view.pcache.mu.Lock()
 	defer f.view.pcache.mu.Unlock()
 
+	f.handleMu.Lock()
+	defer f.handleMu.Unlock()
+
 	f.invalidateAST(ctx)
 	f.handle = nil
 }

internal/lsp/cache/watcher.go

@@ -48,9 +48,15 @@ func (w *WatchMap) Watch(key interface{}, callback func()) func() {
 }
 
 func (w *WatchMap) Notify(key interface{}) {
+	// Make a copy of the watcher callbacks so we don't need to hold
+	// the mutex during the callbacks (to avoid deadlocks).
 	w.mu.Lock()
-	defer w.mu.Unlock()
-	for _, entry := range w.watchers[key] {
+	entries := w.watchers[key]
+	entriesCopy := make([]watcher, len(entries))
+	copy(entriesCopy, entries)
+	w.mu.Unlock()
+
+	for _, entry := range entriesCopy {
 		entry.callback()
 	}
 }