reverseproxy: Enable response interception (#1447, #2920)

It's a raw, low-level implementation for now, but it's very flexible.
More sugar-coating can be added after error handling is more developed.

modules/caddyhttp/reverseproxy/reverseproxy.go

@@ -94,6 +94,20 @@ type Handler struct {
 	// be avoided if at all possible for performance reasons.
 	BufferRequests bool `json:"buffer_requests,omitempty"`
 
+	// List of handlers and their associated matchers to evaluate
+	// after successful roundtrips. The first handler that matches
+	// the response from a backend will be invoked. The response
+	// body from the backend will not be written to the client;
+	// it is up to the handler to finish handling the response.
+	// If passive health checks are enabled, any errors from the
+	// handler chain will not affect the health status of the
+	// backend.
+	//
+	// Two new placeholders are available in this handler chain:
+	// - `{http.reverse_proxy.status_code}` The status code
+	// - `{http.reverse_proxy.status_text}` The status text
+	HandleResponse []caddyhttp.ResponseHandler `json:"handle_response,omitempty"`
+
 	Transport http.RoundTripper `json:"-"`
 	CB        CircuitBreaker    `json:"-"`
 
@@ -252,6 +266,14 @@ func (h *Handler) Provision(ctx caddy.Context) error {
 		go h.activeHealthChecker()
 	}
 
+	// set up any response routes
+	for i, rh := range h.HandleResponse {
+		err := rh.Provision(ctx)
+		if err != nil {
+			return fmt.Errorf("provisioning response handler %d: %v", i, err)
+		}
+	}
+
 	return nil
 }
 
@@ -361,13 +383,21 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
 		}
 
 		// proxy the request to that upstream
-		proxyErr = h.reverseProxy(w, r, dialInfo)
+		proxyErr = h.reverseProxy(w, r, dialInfo, next)
 		if proxyErr == nil || proxyErr == context.Canceled {
 			// context.Canceled happens when the downstream client
 			// cancels the request, which is not our failure
 			return nil
 		}
 
+		// if the roundtrip was successful, don't retry the request or
+		// ding the health status of the upstream (an error can still
+		// occur after the roundtrip if, for example, a response handler
+		// after the roundtrip returns an error)
+		if succ, ok := proxyErr.(roundtripSucceeded); ok {
+			return succ.error
+		}
+
 		// remember this failure (if enabled)
 		h.countFailure(upstream)
 
@@ -456,7 +486,7 @@ func (h Handler) prepareRequest(req *http.Request) error {
 // reverseProxy performs a round-trip to the given backend and processes the response with the client.
 // (This method is mostly the beginning of what was borrowed from the net/http/httputil package in the
 // Go standard library which was used as the foundation.)
-func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, di DialInfo) error {
+func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, di DialInfo, next caddyhttp.Handler) error {
 	di.Upstream.Host.CountRequest(1)
 	defer di.Upstream.Host.CountRequest(-1)
 
@@ -471,16 +501,14 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, di Dia
 	logger := h.logger.With(
 		zap.String("upstream", di.Upstream.String()),
 		zap.Object("request", caddyhttp.LoggableHTTPRequest{Request: req}),
-		zap.Duration("duration", duration),
+		zap.Duration("duration", duration))
-	)
 	if err != nil {
 		logger.Debug("upstream roundtrip", zap.Error(err))
 		return err
 	}
 	logger.Debug("upstream roundtrip",
 		zap.Object("headers", caddyhttp.LoggableHTTPHeader(res.Header)),
-		zap.Int("status", res.StatusCode),
+		zap.Int("status", res.StatusCode))
-	)
 
 	// update circuit breaker on current conditions
 	if di.Upstream.cb != nil {
@@ -503,6 +531,25 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, di Dia
 		}
 	}
 
+	for i, rh := range h.HandleResponse {
+		if len(rh.Routes) == 0 {
+			continue
+		}
+		if rh.Match != nil && !rh.Match.Match(res.StatusCode, res.Header) {
+			continue
+		}
+		res.Body.Close()
+		repl := req.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
+		repl.Set("http.reverse_proxy.status_code", res.StatusCode)
+		repl.Set("http.reverse_proxy.status_text", res.Status)
+		h.logger.Debug("handling response", zap.Int("handler", i))
+		if routeErr := rh.Routes.Compile(next).ServeHTTP(rw, req); routeErr != nil {
+			// wrap error in roundtripSucceeded so caller knows that
+			// the roundtrip was successful and to not retry
+			return roundtripSucceeded{routeErr}
+		}
+	}
+
 	// Deal with 101 Switching Protocols responses: (WebSocket, h2c, etc)
 	if res.StatusCode == http.StatusSwitchingProtocols {
 		h.handleUpgradeResponse(rw, req, res)
@@ -537,15 +584,6 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, di Dia
 		rw.Header().Add("Trailer", strings.Join(trailerKeys, ", "))
 	}
 
-	// TODO: there should be an option to return an error if the response
-	// matches some criteria; would solve https://github.com/caddyserver/caddy/issues/1447
-	// by allowing the backend to determine whether this server should treat
-	// a 400+ status code as an error -- but we might need to be careful that
-	// we do not affect the health status of the backend... still looking into
-	// that; if we need to avoid that, we should return a particular error type
-	// that the caller of this function checks for and only applies health
-	// status changes if the error is not this special type
-
 	rw.WriteHeader(res.StatusCode)
 
 	err = h.copyResponse(rw, res.Body, h.flushInterval(req, res))
@@ -782,6 +820,10 @@ type TLSTransport interface {
 	EnableTLS(base *TLSConfig) error
 }
 
+// roundtripSucceeded is an error type that is returned if the
+// roundtrip succeeded, but an error occurred after-the-fact.
+type roundtripSucceeded struct{ error }
+
 var bufPool = sync.Pool{
 	New: func() interface{} {
 		return new(bytes.Buffer)

modules/caddyhttp/subroute.go

@@ -80,6 +80,36 @@ func (sr *Subroute) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handl
 	return err
 }
 
+// ResponseHandler pairs a response matcher with a route list.
+// It is useful for executing handler routes based on the
+// properties of an HTTP response that has not been written
+// out to the client yet.
+//
+// To use this type, provision it at module load time, then
+// when ready to use, match the response against its matcher;
+// if it matches (or doesn't have a matcher), invoke the routes
+// by calling `rh.Routes.Compile(next).ServeHTTP(rw, req)` (or
+// similar).
+type ResponseHandler struct {
+	// The response matcher for this handler. If empty/nil,
+	// it always matches.
+	Match *ResponseMatcher `json:"match,omitempty"`
+
+	// The list of HTTP routes to execute.
+	Routes RouteList `json:"routes,omitempty"`
+}
+
+// Provision sets up the routse in rh.
+func (rh *ResponseHandler) Provision(ctx caddy.Context) error {
+	if rh.Routes != nil {
+		err := rh.Routes.Provision(ctx)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Interface guards
 var (
 	_ caddy.Provisioner = (*Subroute)(nil)