release-19.2: server: add metric for number of disk stalls detected

pkg/server/node.go

@@ -44,7 +44,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/tracing"
 	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/cockroachdb/logtags"
-	"github.com/opentracing/opentracing-go"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 )
 
@@ -78,6 +78,13 @@ var (
 		Measurement: "Batch KV Requests",
 		Unit:        metric.Unit_COUNT,
 	}
+
+	metaDiskStalls = metric.Metadata{
+		Name:        "engine.stalls",
+		Help:        "Number of disk stalls detected on this node",
+		Measurement: "Disk stalls detected",
+		Unit:        metric.Unit_COUNT,
+	}
 )
 
 // Cluster settings.
@@ -105,16 +112,18 @@ var (
 )
 
 type nodeMetrics struct {
-	Latency *metric.Histogram
-	Success *metric.Counter
-	Err     *metric.Counter
+	Latency    *metric.Histogram
+	Success    *metric.Counter
+	Err        *metric.Counter
+	DiskStalls *metric.Counter
 }
 
 func makeNodeMetrics(reg *metric.Registry, histogramWindow time.Duration) nodeMetrics {
 	nm := nodeMetrics{
-		Latency: metric.NewLatency(metaExecLatency, histogramWindow),
-		Success: metric.NewCounter(metaExecSuccess),
-		Err:     metric.NewCounter(metaExecError),
+		Latency:    metric.NewLatency(metaExecLatency, histogramWindow),
+		Success:    metric.NewCounter(metaExecSuccess),
+		Err:        metric.NewCounter(metaExecError),
+		DiskStalls: metric.NewCounter(metaDiskStalls),
 	}
 	reg.AddMetricStruct(nm)
 	return nm

pkg/server/server_engine_health.go → pkg/server/node_engine_health.go

@@ -18,7 +18,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/storage/engine"
 	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
-	"github.com/cockroachdb/cockroach/pkg/util/stop"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 )
 
@@ -35,8 +34,8 @@ var maxSyncDurationFatalOnExceeded = envutil.EnvOrDefaultBool("COCKROACH_ENGINE_
 // startAssertEngineHealth starts a goroutine that periodically verifies that
 // syncing the engines is possible within maxSyncDuration. If not,
 // the process is terminated (with an attempt at a descriptive message).
-func startAssertEngineHealth(ctx context.Context, stopper *stop.Stopper, engines []engine.Engine) {
-	stopper.RunWorker(ctx, func(ctx context.Context) {
+func (n *Node) startAssertEngineHealth(ctx context.Context, engines []engine.Engine) {
+	n.stopper.RunWorker(ctx, func(ctx context.Context) {
 		t := timeutil.NewTimer()
 		t.Reset(0)
 
@@ -45,8 +44,8 @@ func startAssertEngineHealth(ctx context.Context, stopper *stop.Stopper, engines
 			case <-t.C:
 				t.Read = true
 				t.Reset(10 * time.Second)
-				assertEngineHealth(ctx, engines, maxSyncDuration)
-			case <-stopper.ShouldQuiesce():
+				n.assertEngineHealth(ctx, engines, maxSyncDuration)
+			case <-n.stopper.ShouldQuiesce():
 				return
 			}
 		}
@@ -58,10 +57,13 @@ func guaranteedExitFatal(ctx context.Context, msg string, args ...interface{}) {
 	log.Shout(ctx, log.Severity_FATAL, fmt.Sprintf(msg, args...))
 }
 
-func assertEngineHealth(ctx context.Context, engines []engine.Engine, maxDuration time.Duration) {
+func (n *Node) assertEngineHealth(
+	ctx context.Context, engines []engine.Engine, maxDuration time.Duration,
+) {
 	for _, eng := range engines {
 		func() {
 			t := time.AfterFunc(maxDuration, func() {
+				n.metrics.DiskStalls.Inc(1)
 				var stats string
 				if rocks, ok := eng.(*engine.RocksDB); ok {
 					stats = "\n" + rocks.GetCompactionStats()

pkg/server/server.go

@@ -81,7 +81,7 @@ import (
 	"github.com/cockroachdb/logtags"
 	raven "github.com/getsentry/raven-go"
 	gwruntime "github.com/grpc-ecosystem/grpc-gateway/runtime"
-	"github.com/opentracing/opentracing-go"
+	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"google.golang.org/grpc"
 )
@@ -1270,7 +1270,7 @@ func (s *Server) Start(ctx context.Context) error {
 	}
 	s.stopper.AddCloser(&s.engines)
 
-	startAssertEngineHealth(ctx, s.stopper, s.engines)
+	s.node.startAssertEngineHealth(ctx, s.engines)
 
 	// Write listener info files early in the startup sequence. `listenerInfo` has a comment.
 	listenerFiles := listenerInfo{

pkg/server/status_test.go

@@ -808,7 +808,7 @@ func TestChartCatalogMetrics(t *testing.T) {
 	}
 
 	if len(undefinedMetrics) > 0 {
-		t.Fatalf(`The following metrics need are no longer present and  need to be removed
+		t.Fatalf(`The following metrics need are no longer present and need to be removed
 			from the chart catalog (pkg/ts/chart_catalog.go):%v`, undefinedMetrics)
 	}
 

pkg/ts/catalog/chart_catalog.go

@@ -578,6 +578,13 @@ var charts = []sectionDescription{
 					"exec.success",
 				},
 			},
+			{
+				Title:       "Storage Engine Stalls",
+				Downsampler: DescribeAggregator_MAX,
+				Rate:        DescribeDerivative_NON_NEGATIVE_DERIVATIVE,
+				Percentiles: false,
+				Metrics:     []string{"engine.stalls"},
+			},
 		},
 	},
 	{