fix: properly account request timeouts not as shard write errors

http/write_handler.go

@@ -3,6 +3,7 @@ package http
 import (
 	"compress/gzip"
 	"context"
+	goerrors "errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -190,7 +191,8 @@ func (h *WriteHandler) handleWrite(w http.ResponseWriter, r *http.Request) {
 	requestBytes = parsed.RawSize
 
 	if err := h.PointsWriter.WritePoints(ctx, org.ID, bucket.ID, parsed.Points); err != nil {
-		if partialErr, ok := err.(tsdb.PartialWriteError); ok {
+		var partialErr tsdb.PartialWriteError
+		if goerrors.As(err, &partialErr) {
 			h.HandleHTTPError(ctx, &errors.Error{
 				Code: errors.EUnprocessableEntity,
 				Op:   opWriteHandler,
@@ -200,6 +202,7 @@ func (h *WriteHandler) handleWrite(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
+		h.log.Error("Error handling Write", zap.Error(err))
 		h.HandleHTTPError(ctx, &errors.Error{
 			Code: errors.EInternal,
 			Op:   opWriteHandler,

tsdb/shard.go

@@ -698,8 +698,9 @@ func (s *Shard) WritePoints(ctx context.Context, points []models.Point) (rErr er
 	var writeError error
 	s.stats.writes.Observe(float64(len(points)))
 	defer func() {
-		if rErr != nil {
+		if rErr != nil && !errors.Is(rErr, context.DeadlineExceeded) && !errors.Is(rErr, context.Canceled) {
 			s.stats.writesErr.Observe(float64(len(points)))
+			s.logger.Error("writing points to engine failed", zap.Error(rErr))
 		}
 	}()
 

v1/coordinator/points_writer.go

@@ -17,9 +17,6 @@ import (
 )
 
 var (
-	// ErrTimeout is returned when a write times out.
-	ErrTimeout = errors.New("timeout")
-
 	// ErrWriteFailed is returned when no writes succeeded.
 	ErrWriteFailed = errors.New("write failed")
 )
@@ -459,26 +456,28 @@ func (w *PointsWriter) WritePointsPrivileged(
 		return err
 	}
 
+	// Set a timeout using the context object.
+	ctx, cancel := context.WithTimeout(ctx, w.WriteTimeout)
+	defer cancel()
+
 	// Write each shard in it's own goroutine and return as soon as one fails.
 	ch := make(chan error, len(shardMappings.Points))
 	for shardID, points := range shardMappings.Points {
 		go func(shard *meta.ShardInfo, database, retentionPolicy string, points []models.Point) {
 			err := w.writeToShard(ctx, shard, database, retentionPolicy, points)
 			if err == nil {
 				w.stats.pointsWriteOk.Observe(float64(len(points)))
-			} else {
+			} else if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
 				w.stats.pointsWriteErr.Observe(float64(len(points)))
+				w.Logger.Error("writing points to shard failed", zap.Error(err))
 			}
-			if err == tsdb.ErrShardDeletion {
+			if errors.Is(err, tsdb.ErrShardDeletion) {
 				err = tsdb.PartialWriteError{Reason: fmt.Sprintf("shard %d is pending deletion", shard.ID), Dropped: len(points)}
 			}
 			ch <- err
 		}(shardMappings.Shards[shardID], database, retentionPolicy, points)
 	}
 
-	timeout := time.NewTimer(w.WriteTimeout)
-	defer timeout.Stop()
-
 	if err == nil && shardMappings.Dropped() > 0 {
 		w.stats.pointsWriteDropped.Observe(float64(shardMappings.Dropped()))
 		err = tsdb.PartialWriteError{Reason: shardMappings.SummariseDropped(),
@@ -492,11 +491,10 @@ func (w *PointsWriter) WritePointsPrivileged(
 		select {
 		case <-w.closing:
 			return ErrWriteFailed
-		case <-timeout.C:
-			w.stats.timeout.Inc()
-			// return timeout error to caller
-			return ErrTimeout
 		case err := <-ch:
+			if errors.Is(err, context.Canceled) {
+				w.stats.timeout.Inc()
+			}
 			if err != nil {
 				return err
 			}