From b4cab78bec6e5817a2196c297d8653ca4900bb00 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 13 Jan 2016 00:29:22 -0700 Subject: [PATCH 01/52] Starting transition to Go 1.6 (http2 compatibility) I've built this on Go 1.6 beta 1 and made some changes to be more compatible. Namely, I removed the use of the /x/net/http2 package and let net/http enable h2 by default; updated the way h2 is disabled (if the user requires it); moved TLS_FALLBACK_SCSV to the front of the cipher suites list (all values not accepted by http2 must go after those allowed by it); removed the NextProto default of http/1.1; set the http.Server.TLSConfig value to the TLS config used by the listener (we left it nil before, but this prevents automatic enabling of h2). It is very likely there is more to do, but at least already Caddy uses HTTP/2 when built with Go 1.6. --- caddy/setup/tls.go | 3 ++- server/server.go | 11 +++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/caddy/setup/tls.go b/caddy/setup/tls.go index 5b6c086e9..cf45278ca 100644 --- a/caddy/setup/tls.go +++ b/caddy/setup/tls.go @@ -95,7 +95,8 @@ func SetDefaultTLSParams(c *server.Config) { } // Not a cipher suite, but still important for mitigating protocol downgrade attacks - c.TLS.Ciphers = append(c.TLS.Ciphers, tls.TLS_FALLBACK_SCSV) + // (prepend since having it at end breaks http2 due to non-h2-approved suites before it) + c.TLS.Ciphers = append([]uint16{tls.TLS_FALLBACK_SCSV}, c.TLS.Ciphers...) // Set default protocol min and max versions - must balance compatibility and security if c.TLS.ProtocolMinVersion == 0 { diff --git a/server/server.go b/server/server.go index 4fe12b369..5794c167d 100644 --- a/server/server.go +++ b/server/server.go @@ -15,8 +15,6 @@ import ( "runtime" "sync" "time" - - "golang.org/x/net/http2" ) // Server represents an instance of a server, which serves @@ -179,9 +177,8 @@ func (s *Server) serve(ln ListenerFile) error { // called just before the listener announces itself on the network // and should only be called when the server is just starting up. func (s *Server) setup() error { - if s.HTTP2 { - // TODO: This call may not be necessary after HTTP/2 is merged into std lib - http2.ConfigureServer(s.Server, nil) + if !s.HTTP2 { + s.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) } // Execute startup functions now @@ -206,9 +203,6 @@ func (s *Server) setup() error { // client authentication, and our custom Server type. func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error { config := cloneTLSConfig(s.TLSConfig) - if config.NextProtos == nil { - config.NextProtos = []string{"http/1.1"} - } // Here we diverge from the stdlib a bit by loading multiple certs/key pairs // then we map the server names to their certs @@ -236,6 +230,7 @@ func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error { defer close(s.startChan) return err } + s.TLSConfig = config // Create TLS listener - note that we do not replace s.listener // with this TLS listener; tls.listener is unexported and does From 47079c3d24219336ba0d1ad62468fd8bb07a061a Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 13 Jan 2016 00:32:46 -0700 Subject: [PATCH 02/52] PoC: on-demand TLS Implements "on-demand TLS" as I call it, which means obtaining TLS certificates on-the-fly during TLS handshakes if a certificate for the requested hostname is not already available. Only the first request for a new hostname will experience higher latency; subsequent requests will get the new certificates right out of memory. Code still needs lots of cleanup but the feature is basically working. --- caddy/caddy.go | 5 +- caddy/letsencrypt/handshake.go | 99 ++++++++++++++++++++++++++++++++ caddy/letsencrypt/letsencrypt.go | 89 +++++++++++++++++----------- server/server.go | 70 +++++++++++++++++++--- 4 files changed, 218 insertions(+), 45 deletions(-) create mode 100644 caddy/letsencrypt/handshake.go diff --git a/caddy/caddy.go b/caddy/caddy.go index 734e984d1..600abe668 100644 --- a/caddy/caddy.go +++ b/caddy/caddy.go @@ -191,8 +191,9 @@ func startServers(groupings bindingGroup) error { if err != nil { return err } - s.HTTP2 = HTTP2 // TODO: This setting is temporary - s.ReqCallback = letsencrypt.RequestCallback // ensures we can solve ACME challenges while running + s.HTTP2 = HTTP2 // TODO: This setting is temporary + s.ReqCallback = letsencrypt.RequestCallback // ensures we can solve ACME challenges while running + s.SNICallback = letsencrypt.GetCertificateDuringHandshake // TLS on demand -- awesome! var ln server.ListenerFile if IsRestart() { diff --git a/caddy/letsencrypt/handshake.go b/caddy/letsencrypt/handshake.go new file mode 100644 index 000000000..690eb0767 --- /dev/null +++ b/caddy/letsencrypt/handshake.go @@ -0,0 +1,99 @@ +package letsencrypt + +import ( + "crypto/tls" + "errors" + "strings" + "sync" + + "github.com/mholt/caddy/server" +) + +// GetCertificateDuringHandshake is a function that gets a certificate during a TLS handshake. +// It first checks an in-memory cache in case the cert was requested before, then tries to load +// a certificate in the storage folder from disk. If it can't find an existing certificate, it +// will try to obtain one using ACME, which will then be stored on disk and cached in memory. +// +// This function is safe for use by multiple concurrent goroutines. +func GetCertificateDuringHandshake(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + // Utility function to help us load a cert from disk and put it in the cache if successful + loadCertFromDisk := func(domain string) *tls.Certificate { + cert, err := tls.LoadX509KeyPair(storage.SiteCertFile(domain), storage.SiteKeyFile(domain)) + if err == nil { + certCacheMu.Lock() + if len(certCache) < 10000 { // limit size of cache to prevent a ridiculous, unusual kind of attack + certCache[domain] = &cert + } + certCacheMu.Unlock() + return &cert + } + return nil + } + + // First check our in-memory cache to see if we've already loaded it + certCacheMu.RLock() + cert := server.GetCertificateFromCache(clientHello, certCache) + certCacheMu.RUnlock() + if cert != nil { + return cert, nil + } + + // Then check to see if we already have one on disk; if we do, add it to cache and use it + name := strings.ToLower(clientHello.ServerName) + cert = loadCertFromDisk(name) + if cert != nil { + return cert, nil + } + + // Only option left is to get one from LE, but the name has to qualify first + if !HostQualifies(name) { + return nil, nil + } + + // By this point, we need to obtain one from the CA. We must protect this process + // from happening concurrently, so synchronize. + obtainCertWaitGroupsMutex.Lock() + wg, ok := obtainCertWaitGroups[name] + if ok { + // lucky us -- another goroutine is already obtaining the certificate. + // wait for it to finish obtaining the cert and then we'll use it. + obtainCertWaitGroupsMutex.Unlock() + wg.Wait() + return GetCertificateDuringHandshake(clientHello) + } + + // looks like it's up to us to do all the work and obtain the cert + wg = new(sync.WaitGroup) + wg.Add(1) + obtainCertWaitGroups[name] = wg + obtainCertWaitGroupsMutex.Unlock() + + // Unblock waiters and delete waitgroup when we return + defer func() { + obtainCertWaitGroupsMutex.Lock() + wg.Done() + delete(obtainCertWaitGroups, name) + obtainCertWaitGroupsMutex.Unlock() + }() + + // obtain cert + client, err := newClientPort(DefaultEmail, AlternatePort) + if err != nil { + return nil, errors.New("error creating client: " + err.Error()) + } + err = clientObtain(client, []string{name}, false) + if err != nil { + return nil, err + } + + // load certificate into memory and return it + return loadCertFromDisk(name), nil +} + +// obtainCertWaitGroups is used to coordinate obtaining certs for each hostname. +var obtainCertWaitGroups = make(map[string]*sync.WaitGroup) +var obtainCertWaitGroupsMutex sync.Mutex + +// certCache stores certificates that have been obtained in memory. +var certCache = make(map[string]*tls.Certificate) +var certCacheMu sync.RWMutex diff --git a/caddy/letsencrypt/letsencrypt.go b/caddy/letsencrypt/letsencrypt.go index a2965a104..e721a7b13 100644 --- a/caddy/letsencrypt/letsencrypt.go +++ b/caddy/letsencrypt/letsencrypt.go @@ -6,6 +6,7 @@ package letsencrypt import ( "encoding/json" "errors" + "fmt" "io/ioutil" "net" "net/http" @@ -82,6 +83,13 @@ func Activate(configs []server.Config) ([]server.Config, error) { // keep certificates renewed and OCSP stapling updated go maintainAssets(configs, stopChan) + // TODO - experimental dynamic TLS! + for i := range configs { + if configs[i].Host == "" && configs[i].Port == "443" { + configs[i].TLS.Enabled = true + } + } + return configs, nil } @@ -127,41 +135,9 @@ func ObtainCerts(configs []server.Config, altPort string) error { continue } - Obtain: - certificate, failures := client.ObtainCertificate([]string{cfg.Host}, true, nil) - if len(failures) == 0 { - // Success - immediately save the certificate resource - err := saveCertResource(certificate) - if err != nil { - return errors.New("error saving assets for " + cfg.Host + ": " + err.Error()) - } - } else { - // Error - either try to fix it or report them it to the user and abort - var errMsg string // we'll combine all the failures into a single error message - var promptedForAgreement bool // only prompt user for agreement at most once - - for errDomain, obtainErr := range failures { - // TODO: Double-check, will obtainErr ever be nil? - if tosErr, ok := obtainErr.(acme.TOSError); ok { - // Terms of Service agreement error; we can probably deal with this - if !Agreed && !promptedForAgreement && altPort == "" { // don't prompt if server is already running - Agreed = promptUserAgreement(tosErr.Detail, true) // TODO: Use latest URL - promptedForAgreement = true - } - if Agreed || altPort != "" { - err := client.AgreeToTOS() - if err != nil { - return errors.New("error agreeing to updated terms: " + err.Error()) - } - goto Obtain - } - } - - // If user did not agree or it was any other kind of error, just append to the list of errors - errMsg += "[" + errDomain + "] failed to get certificate: " + obtainErr.Error() + "\n" - } - - return errors.New(errMsg) + err := clientObtain(client, []string{cfg.Host}, altPort == "") + if err != nil { + return err } } } @@ -447,6 +423,49 @@ func redirPlaintextHost(cfg server.Config) server.Config { } } +// clientObtain uses client to obtain a single certificate for domains in names. If +// the user is present to provide an email address, pass in true for allowPrompt, +// otherwise pass in false. If err == nil, the certificate (and key) will be saved +// to disk in the storage folder. +func clientObtain(client *acme.Client, names []string, allowPrompt bool) error { + certificate, failures := client.ObtainCertificate(names, true, nil) + if len(failures) > 0 { + // Error - either try to fix it or report them it to the user and abort + var errMsg string // we'll combine all the failures into a single error message + var promptedForAgreement bool // only prompt user for agreement at most once + + for errDomain, obtainErr := range failures { + // TODO: Double-check, will obtainErr ever be nil? + if tosErr, ok := obtainErr.(acme.TOSError); ok { + // Terms of Service agreement error; we can probably deal with this + if !Agreed && !promptedForAgreement && allowPrompt { // don't prompt if server is already running + Agreed = promptUserAgreement(tosErr.Detail, true) // TODO: Use latest URL + promptedForAgreement = true + } + if Agreed || !allowPrompt { + err := client.AgreeToTOS() + if err != nil { + return errors.New("error agreeing to updated terms: " + err.Error()) + } + return clientObtain(client, names, allowPrompt) + } + } + + // If user did not agree or it was any other kind of error, just append to the list of errors + errMsg += "[" + errDomain + "] failed to get certificate: " + obtainErr.Error() + "\n" + } + return errors.New(errMsg) + } + + // Success - immediately save the certificate resource + err := saveCertResource(certificate) + if err != nil { + return fmt.Errorf("error saving assets for %v: %v", names, err) + } + + return nil +} + // Revoke revokes the certificate for host via ACME protocol. func Revoke(host string) error { if !existingCertAndKey(host) { diff --git a/server/server.go b/server/server.go index 5794c167d..293092c6e 100644 --- a/server/server.go +++ b/server/server.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "runtime" + "strings" "sync" "time" ) @@ -33,6 +34,7 @@ type Server struct { startChan chan struct{} // used to block until server is finished starting connTimeout time.Duration // the maximum duration of a graceful shutdown ReqCallback OptionalCallback // if non-nil, is executed at the beginning of every request + SNICallback func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) } // ListenerFile represents a listener. @@ -206,17 +208,39 @@ func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error { // Here we diverge from the stdlib a bit by loading multiple certs/key pairs // then we map the server names to their certs - var err error - config.Certificates = make([]tls.Certificate, len(tlsConfigs)) - for i, tlsConfig := range tlsConfigs { - config.Certificates[i], err = tls.LoadX509KeyPair(tlsConfig.Certificate, tlsConfig.Key) - config.Certificates[i].OCSPStaple = tlsConfig.OCSPStaple + for _, tlsConfig := range tlsConfigs { + if tlsConfig.Certificate == "" || tlsConfig.Key == "" { + continue + } + cert, err := tls.LoadX509KeyPair(tlsConfig.Certificate, tlsConfig.Key) if err != nil { defer close(s.startChan) - return err + return fmt.Errorf("loading certificate and key pair: %v", err) } + cert.OCSPStaple = tlsConfig.OCSPStaple + config.Certificates = append(config.Certificates, cert) + } + if len(config.Certificates) > 0 { + config.BuildNameToCertificate() + } + + config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + // TODO: When Caddy starts, if it is to issue certs dynamically, we need + // terms agreement and an email address. make sure this is enforced at server + // start if the Caddyfile enables dynamic certificate issuance! + + // Check NameToCertificate like the std lib does in "getCertificate" (unexported, bah) + cert := GetCertificateFromCache(clientHello, config.NameToCertificate) + if cert != nil { + return cert, nil + } + + if s.SNICallback != nil { + return s.SNICallback(clientHello) + } + + return nil, nil } - config.BuildNameToCertificate() // Customize our TLS configuration config.MinVersion = tlsConfigs[0].ProtocolMinVersion @@ -225,7 +249,7 @@ func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error { config.PreferServerCipherSuites = tlsConfigs[0].PreferServerCipherSuites // TLS client authentication, if user enabled it - err = setupClientAuth(tlsConfigs, config) + err := setupClientAuth(tlsConfigs, config) if err != nil { defer close(s.startChan) return err @@ -242,6 +266,36 @@ func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error { return s.Server.Serve(ln) } +// Borrowed from the Go standard library, crypto/tls pacakge, common.go. +// It has been modified to fit this program. +// Original license: +// +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +func GetCertificateFromCache(clientHello *tls.ClientHelloInfo, cache map[string]*tls.Certificate) *tls.Certificate { + name := strings.ToLower(clientHello.ServerName) + for len(name) > 0 && name[len(name)-1] == '.' { + name = name[:len(name)-1] + } + + // exact match? great! use it + if cert, ok := cache[name]; ok { + return cert + } + + // try replacing labels in the name with wildcards until we get a match. + labels := strings.Split(name, ".") + for i := range labels { + labels[i] = "*" + candidate := strings.Join(labels, ".") + if cert, ok := cache[candidate]; ok { + return cert + } + } + return nil +} + // Stop stops the server. It blocks until the server is // totally stopped. On POSIX systems, it will wait for // connections to close (up to a max timeout of a few From b0ccab7b4a7cb93d413fa7359171569092f629e5 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 13 Jan 2016 09:24:03 -0700 Subject: [PATCH 03/52] tls: Fix failing test --- caddy/setup/tls_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caddy/setup/tls_test.go b/caddy/setup/tls_test.go index d8b2a4d95..727a7996e 100644 --- a/caddy/setup/tls_test.go +++ b/caddy/setup/tls_test.go @@ -34,6 +34,7 @@ func TestTLSParseBasic(t *testing.T) { // Cipher checks expectedCiphers := []uint16{ + tls.TLS_FALLBACK_SCSV, tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, @@ -42,7 +43,6 @@ func TestTLSParseBasic(t *testing.T) { tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, tls.TLS_RSA_WITH_AES_256_CBC_SHA, tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_FALLBACK_SCSV, } // Ensure count is correct (plus one for TLS_FALLBACK_SCSV) From f1b2637d4468f1230aeee3d1126bdd4e914e8c08 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 25 Jan 2016 20:21:08 -0700 Subject: [PATCH 04/52] letsencrypt: Enable activation on empty hosts; fix email bug --- caddy/letsencrypt/letsencrypt.go | 18 +++++++++++------- caddy/letsencrypt/letsencrypt_test.go | 14 ++++++++------ caddy/letsencrypt/user.go | 3 ++- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/caddy/letsencrypt/letsencrypt.go b/caddy/letsencrypt/letsencrypt.go index 92fa10a36..d6fb9cc37 100644 --- a/caddy/letsencrypt/letsencrypt.go +++ b/caddy/letsencrypt/letsencrypt.go @@ -131,7 +131,7 @@ func ObtainCerts(configs []server.Config, altPort string) error { } for _, cfg := range group { - if existingCertAndKey(cfg.Host) { + if cfg.Host == "" || existingCertAndKey(cfg.Host) { continue } @@ -170,8 +170,10 @@ func EnableTLS(configs []server.Config) { continue } configs[i].TLS.Enabled = true - configs[i].TLS.Certificate = storage.SiteCertFile(configs[i].Host) - configs[i].TLS.Key = storage.SiteKeyFile(configs[i].Host) + if configs[i].Host != "" { + configs[i].TLS.Certificate = storage.SiteCertFile(configs[i].Host) + configs[i].TLS.Key = storage.SiteKeyFile(configs[i].Host) + } setup.SetDefaultTLSParams(&configs[i]) } } @@ -257,13 +259,15 @@ func ConfigQualifies(cfg server.Config) bool { cfg.Port != "80" && cfg.TLS.LetsEncryptEmail != "off" && - // we get can't certs for some kinds of hostnames - HostQualifies(cfg.Host) + // we get can't certs for some kinds of hostnames, + // but we CAN get certs at request-time even if + // the hostname in the config is empty right now. + (cfg.Host == "" || HostQualifies(cfg.Host)) } // HostQualifies returns true if the hostname alone // appears eligible for automatic HTTPS. For example, -// localhost, empty hostname, and wildcard hosts are +// localhost, empty hostname, and IP addresses are // not eligible because we cannot obtain certificates // for those names. func HostQualifies(hostname string) bool { @@ -397,7 +401,7 @@ func saveCertResource(cert acme.CertificateResource) error { // be the HTTPS configuration. The returned configuration is set // to listen on port 80. func redirPlaintextHost(cfg server.Config) server.Config { - toURL := "https://" + cfg.Host + toURL := "https://{host}" // serve any host, since cfg.Host could be empty if cfg.Port != "443" && cfg.Port != "80" { toURL += ":" + cfg.Port } diff --git a/caddy/letsencrypt/letsencrypt_test.go b/caddy/letsencrypt/letsencrypt_test.go index 2cce94058..e3ac2212e 100644 --- a/caddy/letsencrypt/letsencrypt_test.go +++ b/caddy/letsencrypt/letsencrypt_test.go @@ -46,6 +46,7 @@ func TestConfigQualifies(t *testing.T) { cfg server.Config expect bool }{ + {server.Config{Host: ""}, true}, {server.Config{Host: "localhost"}, false}, {server.Config{Host: "example.com"}, true}, {server.Config{Host: "example.com", TLS: server.TLSConfig{Certificate: "cert.pem"}}, false}, @@ -105,18 +106,18 @@ func TestRedirPlaintextHost(t *testing.T) { if actual, expected := handler.Rules[0].FromPath, "/"; actual != expected { t.Errorf("Expected redirect rule to be for path '%s' but is actually for '%s'", expected, actual) } - if actual, expected := handler.Rules[0].To, "https://example.com:1234{uri}"; actual != expected { + if actual, expected := handler.Rules[0].To, "https://{host}:1234{uri}"; actual != expected { t.Errorf("Expected redirect rule to be to URL '%s' but is actually to '%s'", expected, actual) } if actual, expected := handler.Rules[0].Code, http.StatusMovedPermanently; actual != expected { t.Errorf("Expected redirect rule to have code %d but was %d", expected, actual) } - // browsers can interpret default ports with scheme, so make sure the port - // doesn't get added in explicitly for default ports. + // browsers can infer a default port from scheme, so make sure the port + // doesn't get added in explicitly for default ports like 443 for https. cfg = redirPlaintextHost(server.Config{Host: "example.com", Port: "443"}) handler, ok = cfg.Middleware["/"][0](nil).(redirect.Redirect) - if actual, expected := handler.Rules[0].To, "https://example.com{uri}"; actual != expected { + if actual, expected := handler.Rules[0].To, "https://{host}{uri}"; actual != expected { t.Errorf("(Default Port) Expected redirect rule to be to URL '%s' but is actually to '%s'", expected, actual) } } @@ -252,7 +253,7 @@ func TestMakePlaintextRedirects(t *testing.T) { func TestEnableTLS(t *testing.T) { configs := []server.Config{ - server.Config{TLS: server.TLSConfig{Managed: true}}, + server.Config{Host: "example.com", TLS: server.TLSConfig{Managed: true}}, server.Config{}, // not managed - no changes! } @@ -325,8 +326,9 @@ func TestMarkQualified(t *testing.T) { {Host: "example.com", Port: "1234"}, {Host: "example.com", Scheme: "https"}, {Host: "example.com", Port: "80", Scheme: "https"}, + {Host: ""}, } - expectedManagedCount := 4 + expectedManagedCount := 5 MarkQualified(configs) diff --git a/caddy/letsencrypt/user.go b/caddy/letsencrypt/user.go index fca50ec9c..1fac1d71d 100644 --- a/caddy/letsencrypt/user.go +++ b/caddy/letsencrypt/user.go @@ -154,10 +154,11 @@ func getEmail(cfg server.Config, skipPrompt bool) string { if err != nil { return "" } + leEmail = strings.TrimSpace(leEmail) DefaultEmail = leEmail Agreed = true } - return strings.TrimSpace(leEmail) + return leEmail } // promptUserAgreement prompts the user to agree to the agreement From d8be787f398239f3c25b888e2b2e45d15a558011 Mon Sep 17 00:00:00 2001 From: MathiasB Date: Thu, 28 Jan 2016 15:26:33 +0100 Subject: [PATCH 05/52] FastCGI: IPv6 when parsing r.RemoteAddr --- middleware/fastcgi/fastcgi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go index 517505b6d..153cae7f6 100755 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -182,7 +182,7 @@ func (h Handler) buildEnv(r *http.Request, rule Rule, fpath string) (map[string] // Separate remote IP and port; more lenient than net.SplitHostPort var ip, port string - if idx := strings.Index(r.RemoteAddr, ":"); idx > -1 { + if idx := strings.LastIndex(r.RemoteAddr, ":"); idx > -1 { ip = r.RemoteAddr[:idx] port = r.RemoteAddr[idx+1:] } else { From ac197f1694cdacd8cfe8f5aeaddf49326d5cc21e Mon Sep 17 00:00:00 2001 From: MathiasB Date: Fri, 29 Jan 2016 11:46:06 +0100 Subject: [PATCH 06/52] FastCGI: some simple tests for buildEnv More tests are needed for the other environmental variables. These tests were specifically made for testing of IP addresses. --- middleware/fastcgi/fastcgi_test.go | 64 ++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/middleware/fastcgi/fastcgi_test.go b/middleware/fastcgi/fastcgi_test.go index 69ee02f33..0d391a237 100644 --- a/middleware/fastcgi/fastcgi_test.go +++ b/middleware/fastcgi/fastcgi_test.go @@ -1,6 +1,8 @@ package fastcgi import ( + "net/http" + "net/url" "testing" ) @@ -29,3 +31,65 @@ func TestRuleParseAddress(t *testing.T) { } } + +func BuildEnvSingle(r *http.Request, rule Rule, fpath string, envExpected map[string]string, t *testing.T) { + + h := Handler{} + + env, err := h.buildEnv(r, rule, fpath) + if err != nil { + t.Error("Unexpected error:", err.Error()) + } + + for k, v := range envExpected { + if env[k] != v { + t.Errorf("Unexpected %v. Got %v, expected %v", k, env[k], v) + } + } + +} + +func TestBuildEnv(t *testing.T) { + + rule := Rule{} + url, err := url.Parse("http://localhost:2015/fgci_test.php?test=blabla") + if err != nil { + t.Error("Unexpected error:", err.Error()) + } + + r := http.Request{ + Method: "GET", + URL: url, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Host: "localhost:2015", + RemoteAddr: "[2b02:1810:4f2d:9400:70ab:f822:be8a:9093]:51688", + RequestURI: "/fgci_test.php", + } + + fpath := "/fgci_test.php" + + var envExpected = map[string]string{ + "REMOTE_ADDR": "[2b02:1810:4f2d:9400:70ab:f822:be8a:9093]", + "REMOTE_PORT": "51688", + "SERVER_PROTOCOL": "HTTP/1.1", + "QUERY_STRING": "test=blabla", + "REQUEST_METHOD": "GET", + "HTTP_HOST": "localhost:2015", + } + + // 1. Test for full canonical IPv6 address + BuildEnvSingle(&r, rule, fpath, envExpected, t) + + // 2. Test for shorthand notation of IPv6 address + r.RemoteAddr = "[::1]:51688" + envExpected["REMOTE_ADDR"] = "[::1]" + BuildEnvSingle(&r, rule, fpath, envExpected, t) + + // 3. Test for IPv4 address + r.RemoteAddr = "192.168.0.10:51688" + envExpected["REMOTE_ADDR"] = "192.168.0.10" + BuildEnvSingle(&r, rule, fpath, envExpected, t) + +} From 8d057c861477a28b96552d892cea7c9f58d3baab Mon Sep 17 00:00:00 2001 From: Den Quixote Date: Sat, 30 Jan 2016 02:20:34 +0100 Subject: [PATCH 07/52] letsencrypt: properly retrieve hostname from request. --- caddy/letsencrypt/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caddy/letsencrypt/handler.go b/caddy/letsencrypt/handler.go index e147e00c8..f1db03dd5 100644 --- a/caddy/letsencrypt/handler.go +++ b/caddy/letsencrypt/handler.go @@ -23,9 +23,9 @@ func RequestCallback(w http.ResponseWriter, r *http.Request) bool { scheme = "https" } - hostname, _, err := net.SplitHostPort(r.URL.Host) + hostname, _, err := net.SplitHostPort(r.Host) if err != nil { - hostname = r.URL.Host + hostname = r.Host } upstream, err := url.Parse(scheme + "://" + hostname + ":" + AlternatePort) From c59fd1c76ed35b8f0f767c34285767ab1026be2c Mon Sep 17 00:00:00 2001 From: MathiasB Date: Mon, 1 Feb 2016 09:39:13 +0100 Subject: [PATCH 08/52] Defined test function in TestBuildEnv --- middleware/fastcgi/fastcgi_test.go | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/middleware/fastcgi/fastcgi_test.go b/middleware/fastcgi/fastcgi_test.go index 0d391a237..1fc7446d0 100644 --- a/middleware/fastcgi/fastcgi_test.go +++ b/middleware/fastcgi/fastcgi_test.go @@ -32,25 +32,25 @@ func TestRuleParseAddress(t *testing.T) { } -func BuildEnvSingle(r *http.Request, rule Rule, fpath string, envExpected map[string]string, t *testing.T) { - - h := Handler{} - - env, err := h.buildEnv(r, rule, fpath) - if err != nil { - t.Error("Unexpected error:", err.Error()) - } - - for k, v := range envExpected { - if env[k] != v { - t.Errorf("Unexpected %v. Got %v, expected %v", k, env[k], v) - } - } - -} - func TestBuildEnv(t *testing.T) { + buildEnvSingle := func(r *http.Request, rule Rule, fpath string, envExpected map[string]string, t *testing.T) { + + h := Handler{} + + env, err := h.buildEnv(r, rule, fpath) + if err != nil { + t.Error("Unexpected error:", err.Error()) + } + + for k, v := range envExpected { + if env[k] != v { + t.Errorf("Unexpected %v. Got %v, expected %v", k, env[k], v) + } + } + + } + rule := Rule{} url, err := url.Parse("http://localhost:2015/fgci_test.php?test=blabla") if err != nil { @@ -80,16 +80,16 @@ func TestBuildEnv(t *testing.T) { } // 1. Test for full canonical IPv6 address - BuildEnvSingle(&r, rule, fpath, envExpected, t) + buildEnvSingle(&r, rule, fpath, envExpected, t) // 2. Test for shorthand notation of IPv6 address r.RemoteAddr = "[::1]:51688" envExpected["REMOTE_ADDR"] = "[::1]" - BuildEnvSingle(&r, rule, fpath, envExpected, t) + buildEnvSingle(&r, rule, fpath, envExpected, t) // 3. Test for IPv4 address r.RemoteAddr = "192.168.0.10:51688" envExpected["REMOTE_ADDR"] = "192.168.0.10" - BuildEnvSingle(&r, rule, fpath, envExpected, t) + buildEnvSingle(&r, rule, fpath, envExpected, t) } From fde9bbeb3274b1ef92a769e1eb26c6f144bb81e0 Mon Sep 17 00:00:00 2001 From: MathiasB Date: Mon, 1 Feb 2016 11:17:16 +0100 Subject: [PATCH 09/52] basicauth: fixed 'go vet' printing function value --- middleware/basicauth/basicauth_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/basicauth/basicauth_test.go b/middleware/basicauth/basicauth_test.go index aad5ed399..631aaaed9 100644 --- a/middleware/basicauth/basicauth_test.go +++ b/middleware/basicauth/basicauth_test.go @@ -139,7 +139,7 @@ md5:$apr1$l42y8rex$pOA2VJ0x/0TwaFeAF9nX61` if rule.Password, err = GetHtpasswdMatcher(filename, rule.Username, siteRoot); err != nil { t.Fatalf("GetHtpasswdMatcher(%q, %q): %v", htfh.Name(), rule.Username, err) } - t.Logf("%d. username=%q password=%v", i, rule.Username, rule.Password) + t.Logf("%d. username=%q", i, rule.Username) if !rule.Password(htpasswdPasswd) || rule.Password(htpasswdPasswd+"!") { t.Errorf("%d (%s) password does not match.", i, rule.Username) } From f4fcfa87937072f09cfe54ea7a7b59767bbffffc Mon Sep 17 00:00:00 2001 From: David Darrell Date: Thu, 4 Feb 2016 12:46:24 +0800 Subject: [PATCH 10/52] When the requested host is not found log the remote host. --- server/server.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 4fe12b369..12513e81b 100644 --- a/server/server.go +++ b/server/server.go @@ -337,6 +337,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } + // Get the remote host + remoteHost, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + remoteHost = r.RemoteAddr + } + if vh, ok := s.vhosts[host]; ok { status, _ := vh.stack.ServeHTTP(w, r) @@ -347,7 +353,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } else { w.WriteHeader(http.StatusNotFound) fmt.Fprintf(w, "No such host at %s", s.Server.Addr) - log.Printf("[INFO] %s - No such host at %s", host, s.Server.Addr) + log.Printf("[INFO] %s - No such host at %s (requested by %s)", host, s.Server.Addr, remoteHost) } } From 2acaf2fa6fa97b61ee0fa09e3eb8a5c782d6abf8 Mon Sep 17 00:00:00 2001 From: David Darrell Date: Thu, 4 Feb 2016 16:17:10 +0800 Subject: [PATCH 11/52] Move logic to split the port to only happen when the host is not found. --- server/server.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/server.go b/server/server.go index 12513e81b..e325940dc 100644 --- a/server/server.go +++ b/server/server.go @@ -337,12 +337,6 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } - // Get the remote host - remoteHost, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - remoteHost = r.RemoteAddr - } - if vh, ok := s.vhosts[host]; ok { status, _ := vh.stack.ServeHTTP(w, r) @@ -351,6 +345,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { DefaultErrorFunc(w, r, status) } } else { + // Get the remote host + remoteHost, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + remoteHost = r.RemoteAddr + } + w.WriteHeader(http.StatusNotFound) fmt.Fprintf(w, "No such host at %s", s.Server.Addr) log.Printf("[INFO] %s - No such host at %s (requested by %s)", host, s.Server.Addr, remoteHost) From fbdfc979ec992b84cd962a8c563fdf06f6a34388 Mon Sep 17 00:00:00 2001 From: Miek Gieben Date: Thu, 4 Feb 2016 11:21:44 +0000 Subject: [PATCH 12/52] Markdown: enable definition lists --- middleware/markdown/process.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/markdown/process.go b/middleware/markdown/process.go index 0a7324776..807ae47c2 100644 --- a/middleware/markdown/process.go +++ b/middleware/markdown/process.go @@ -68,7 +68,7 @@ func (md Markdown) Process(c *Config, requestPath string, b []byte, ctx middlewa } // process markdown - extns := blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_STRIKETHROUGH + extns := blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_STRIKETHROUGH | blackfriday.EXTENSION_DEFINITION_LISTS markdown = blackfriday.Markdown(markdown, c.Renderer, extns) // set it as body for template From 86f36bdb610eadc521906a6e5072d4c789b602b8 Mon Sep 17 00:00:00 2001 From: Miek Gieben Date: Thu, 4 Feb 2016 12:51:14 +0000 Subject: [PATCH 13/52] Add .Markdown directive This allows any template to use: {{.Markdown "filename"}} which will convert the markdown contents of filename to HTML and then include the HTML in the template. --- middleware/context.go | 16 ++++++++++++++++ middleware/context_test.go | 39 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/middleware/context.go b/middleware/context.go index 587642027..4b61da290 100644 --- a/middleware/context.go +++ b/middleware/context.go @@ -9,6 +9,8 @@ import ( "strings" "text/template" "time" + + "github.com/russross/blackfriday" ) // This file contains the context and functions available for @@ -190,3 +192,17 @@ func (c Context) StripExt(path string) string { func (c Context) Replace(input, find, replacement string) string { return strings.Replace(input, find, replacement, -1) } + +// Markdown returns the HTML contents of the markdown contained in filename +// (relative to the site root). +func (c Context) Markdown(filename string) (string, error) { + body, err := c.Include(filename) + if err != nil { + return "", err + } + renderer := blackfriday.HtmlRenderer(0, "", "") + extns := blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_STRIKETHROUGH | blackfriday.EXTENSION_DEFINITION_LISTS + markdown := blackfriday.Markdown([]byte(body), renderer, extns) + + return string(markdown), nil +} diff --git a/middleware/context_test.go b/middleware/context_test.go index e60bd7f13..5fb883c6f 100644 --- a/middleware/context_test.go +++ b/middleware/context_test.go @@ -92,6 +92,45 @@ func TestIncludeNotExisting(t *testing.T) { } } +func TestMarkdown(t *testing.T) { + context := getContextOrFail(t) + + inputFilename := "test_file" + absInFilePath := filepath.Join(fmt.Sprintf("%s", context.Root), inputFilename) + defer func() { + err := os.Remove(absInFilePath) + if err != nil && !os.IsNotExist(err) { + t.Fatalf("Failed to clean test file!") + } + }() + + tests := []struct { + fileContent string + expectedContent string + }{ + // Test 0 - test parsing of markdown + { + fileContent: "* str1\n* str2\n", + expectedContent: "
    \n
  • str1
    • \n
  • str2
    • \n
\n", + }, + } + + for i, test := range tests { + testPrefix := getTestPrefix(i) + + // WriteFile truncates the contentt + err := ioutil.WriteFile(absInFilePath, []byte(test.fileContent), os.ModePerm) + if err != nil { + t.Fatal(testPrefix+"Failed to create test file. Error was: %v", err) + } + + content, _ := context.Markdown(inputFilename) + if content != test.expectedContent { + t.Errorf(testPrefix+"Expected content [%s] but found [%s]. Input file was: %s", test.expectedContent, content, inputFilename) + } + } +} + func TestCookie(t *testing.T) { tests := []struct { From e72fc20c78bb90f0020a892d5e2c78b46e6f1a26 Mon Sep 17 00:00:00 2001 From: Craig Peterson Date: Wed, 11 Nov 2015 11:19:52 -0700 Subject: [PATCH 14/52] making directives externally registerable --- caddy/directives.go | 17 +++++++++++++++++ caddy/directives_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 caddy/directives_test.go diff --git a/caddy/directives.go b/caddy/directives.go index 39b54b7d6..cf7875355 100644 --- a/caddy/directives.go +++ b/caddy/directives.go @@ -68,6 +68,23 @@ var directiveOrder = []directive{ {"browse", setup.Browse}, } +// RegisterDirective adds the given directive to caddy's list of directives. +// Pass the name of a directive you want it to be placed after, +// otherwise it will be placed at the bottom of the stack. +func RegisterDirective(name string, setup SetupFunc, after string) { + dir := directive{name: name, setup: setup} + idx := len(directiveOrder) + for i := range directiveOrder { + if directiveOrder[i].name == after { + idx = i + 1 + break + } + } + newDirectives := append(directiveOrder[:idx], append([]directive{dir}, directiveOrder[idx:]...)...) + directiveOrder = newDirectives + parse.ValidDirectives[name] = struct{}{} +} + // directive ties together a directive name with its setup function. type directive struct { name string diff --git a/caddy/directives_test.go b/caddy/directives_test.go new file mode 100644 index 000000000..e37411f1c --- /dev/null +++ b/caddy/directives_test.go @@ -0,0 +1,31 @@ +package caddy + +import ( + "reflect" + "testing" +) + +func TestRegister(t *testing.T) { + directives := []directive{ + {"dummy", nil}, + {"dummy2", nil}, + } + directiveOrder = directives + RegisterDirective("foo", nil, "dummy") + if len(directiveOrder) != 3 { + t.Fatal("Should have 3 directives now") + } + getNames := func() (s []string) { + for _, d := range directiveOrder { + s = append(s, d.name) + } + return s + } + if !reflect.DeepEqual(getNames(), []string{"dummy", "foo", "dummy2"}) { + t.Fatalf("directive order doesn't match: %s", getNames()) + } + RegisterDirective("bar", nil, "ASDASD") + if !reflect.DeepEqual(getNames(), []string{"dummy", "foo", "dummy2", "bar"}) { + t.Fatalf("directive order doesn't match: %s", getNames()) + } +} From b1208d3fdfb972985c3356bd308833274f5bd373 Mon Sep 17 00:00:00 2001 From: Vadim Petrov Date: Wed, 10 Feb 2016 18:03:43 +0300 Subject: [PATCH 15/52] New function DialWithDialer to create FCGIClient with custom Dialer. --- middleware/fastcgi/fcgiclient.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/middleware/fastcgi/fcgiclient.go b/middleware/fastcgi/fcgiclient.go index 511a5219d..82580137c 100644 --- a/middleware/fastcgi/fcgiclient.go +++ b/middleware/fastcgi/fcgiclient.go @@ -169,12 +169,11 @@ type FCGIClient struct { reqID uint16 } -// Dial connects to the fcgi responder at the specified network address. +// DialWithDialer connects to the fcgi responder at the specified network address, using custom net.Dialer. // See func net.Dial for a description of the network and address parameters. -func Dial(network, address string) (fcgi *FCGIClient, err error) { +func DialWithDialer(network, address string, dialer net.Dialer) (fcgi *FCGIClient, err error) { var conn net.Conn - - conn, err = net.Dial(network, address) + conn, err = dialer.Dial(network, address) if err != nil { return } @@ -188,6 +187,12 @@ func Dial(network, address string) (fcgi *FCGIClient, err error) { return } +// Dial connects to the fcgi responder at the specified network address, using default net.Dialer. +// See func net.Dial for a description of the network and address parameters. +func Dial(network, address string) (fcgi *FCGIClient, err error) { + return DialWithDialer(network, address, net.Dialer{}) +} + // Close closes fcgi connnection func (c *FCGIClient) Close() { c.rwc.Close() From 7091a2090bf69ecee9d6ed7f1491b229da13d584 Mon Sep 17 00:00:00 2001 From: eiszfuchs Date: Wed, 10 Feb 2016 19:45:31 +0100 Subject: [PATCH 16/52] created http.Transport and tests for unix sockets --- middleware/proxy/proxy.go | 1 - middleware/proxy/proxy_test.go | 66 ++++++++++++++++++++++++++++++++ middleware/proxy/reverseproxy.go | 31 +++++++++++++-- middleware/proxy/upstream.go | 3 +- 4 files changed, 96 insertions(+), 5 deletions(-) diff --git a/middleware/proxy/proxy.go b/middleware/proxy/proxy.go index 3efcf6030..7be8af2ad 100644 --- a/middleware/proxy/proxy.go +++ b/middleware/proxy/proxy.go @@ -63,7 +63,6 @@ var tryDuration = 60 * time.Second // ServeHTTP satisfies the middleware.Handler interface. func (p Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { - for _, upstream := range p.Upstreams { if middleware.Path(r.URL.Path).Matches(upstream.From()) && upstream.IsAllowedPath(r.URL.Path) { var replacer middleware.Replacer diff --git a/middleware/proxy/proxy_test.go b/middleware/proxy/proxy_test.go index 18e2034b6..68b135679 100644 --- a/middleware/proxy/proxy_test.go +++ b/middleware/proxy/proxy_test.go @@ -3,6 +3,7 @@ package proxy import ( "bufio" "bytes" + "fmt" "io" "io/ioutil" "log" @@ -13,7 +14,9 @@ import ( "os" "strings" "testing" + "runtime" "time" + "path/filepath" "golang.org/x/net/websocket" ) @@ -160,6 +163,69 @@ func TestWebSocketReverseProxyFromWSClient(t *testing.T) { } } +func TestUnixSocketProxy(t *testing.T) { + if runtime.GOOS == "windows" { + return + } + + trialMsg := "Is it working?" + + var proxySuccess bool + + // This is our fake "application" we want to proxy to + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Request was proxied when this is called + proxySuccess = true + + fmt.Fprint(w, trialMsg) + })) + + // Get absolute path for unix: socket + socketPath, err := filepath.Abs("./test_socket") + if err != nil { + t.Fatalf("Unable to get absolute path: %v", err) + } + + // Change httptest.Server listener to listen to unix: socket + ln, err := net.Listen("unix", socketPath) + if err != nil { + t.Fatalf("Unable to listen: %v", err) + } + ts.Listener = ln + + ts.Start() + defer ts.Close() + + url := strings.Replace(ts.URL, "http://", "unix:", 1) + p := newWebSocketTestProxy(url) + + echoProxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p.ServeHTTP(w, r) + })) + defer echoProxy.Close() + + res, err := http.Get(echoProxy.URL) + if err != nil { + t.Fatalf("Unable to GET: %v", err) + } + + greeting, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + t.Fatalf("Unable to GET: %v", err) + } + + actualMsg := fmt.Sprintf("%s", greeting) + + if !proxySuccess { + t.Errorf("Expected request to be proxied, but it wasn't") + } + + if actualMsg != trialMsg { + t.Errorf("Expected '%s' but got '%s' instead", trialMsg, actualMsg) + } +} + func newFakeUpstream(name string, insecure bool) *fakeUpstream { uri, _ := url.Parse(name) u := &fakeUpstream{ diff --git a/middleware/proxy/reverseproxy.go b/middleware/proxy/reverseproxy.go index 32ca2378b..cb4ec8750 100644 --- a/middleware/proxy/reverseproxy.go +++ b/middleware/proxy/reverseproxy.go @@ -59,6 +59,18 @@ func singleJoiningSlash(a, b string) string { return a + b } +// Though the relevant directive prefix is just "unix:", url.Parse +// will - assuming the regular URL scheme - add additional slashes +// as if "unix" was a request protocol. +// What we need is just the path, so if "unix:/var/run/www.socket" +// was the proxy directive, the parsed hostName would be +// "unix:///var/run/www.socket", hence the ambiguous trimming. +func socketDial(hostName string) func(network, addr string) (conn net.Conn, err error) { + return func(network, addr string) (conn net.Conn, err error) { + return net.Dial("unix", hostName[len("unix://"):]) + } +} + // NewSingleHostReverseProxy returns a new ReverseProxy that rewrites // URLs to the scheme, host, and base path provided in target. If the // target's path is "/base" and the incoming request was for "/dir", @@ -68,8 +80,15 @@ func singleJoiningSlash(a, b string) string { func NewSingleHostReverseProxy(target *url.URL, without string) *ReverseProxy { targetQuery := target.RawQuery director := func(req *http.Request) { - req.URL.Scheme = target.Scheme - req.URL.Host = target.Host + if target.Scheme == "unix" { + // to make Dial work with unix URL, + // scheme and host have to be faked + req.URL.Scheme = "http" + req.URL.Host = "socket" + } else { + req.URL.Scheme = target.Scheme + req.URL.Host = target.Host + } req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path) if targetQuery == "" || req.URL.RawQuery == "" { req.URL.RawQuery = targetQuery + req.URL.RawQuery @@ -80,7 +99,13 @@ func NewSingleHostReverseProxy(target *url.URL, without string) *ReverseProxy { req.URL.Path = strings.TrimPrefix(req.URL.Path, without) } } - return &ReverseProxy{Director: director} + rp := &ReverseProxy{Director: director} + if target.Scheme == "unix" { + rp.Transport = &http.Transport{ + Dial: socketDial(target.String()), + } + } + return rp } func copyHeader(dst, src http.Header) { diff --git a/middleware/proxy/upstream.go b/middleware/proxy/upstream.go index 9d87c07ab..faa11cd92 100644 --- a/middleware/proxy/upstream.go +++ b/middleware/proxy/upstream.go @@ -65,7 +65,8 @@ func NewStaticUpstreams(c parse.Dispenser) ([]Upstream, error) { upstream.Hosts = make([]*UpstreamHost, len(to)) for i, host := range to { - if !strings.HasPrefix(host, "http") { + if !strings.HasPrefix(host, "http") && + !strings.HasPrefix(host, "unix:") { host = "http://" + host } uh := &UpstreamHost{ From 11103bd8d68ed9d8dcd2fc0960c5d206175e1b1f Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 11 Feb 2016 00:06:05 -0700 Subject: [PATCH 17/52] Major refactor of all HTTPS/TLS/ACME code Biggest change is no longer using standard library's tls.Config.getCertificate function to get a certificate during TLS handshake. Implemented our own cache which can be changed dynamically at runtime, even during TLS handshakes. As such, restarts are no longer required after certificate renewals or OCSP updates. We also allow loading multiple certificates and keys per host, even by specifying a directory (tls got a new 'load' command for that). Renamed the letsencrypt package to https in a gradual effort to become more generic; and https is more fitting for what the package does now. There are still some known bugs, e.g. reloading where a new certificate is required but port 80 isn't currently listening, will cause the challenge to fail. There's still plenty of cleanup to do and tests to write. It is especially confusing right now how we enable "on-demand" TLS during setup and keep track of that. But this change should basically work so far. --- caddy/caddy.go | 16 +- caddy/caddy_test.go | 13 +- caddy/config.go | 6 +- caddy/directives.go | 3 +- caddy/helpers.go | 6 - caddy/https/certificates.go | 232 +++++++++++++++++ caddy/https/client.go | 215 ++++++++++++++++ caddy/{letsencrypt => https}/crypto.go | 2 +- caddy/{letsencrypt => https}/crypto_test.go | 2 +- caddy/{letsencrypt => https}/handler.go | 2 +- caddy/{letsencrypt => https}/handler_test.go | 2 +- caddy/https/handshake.go | 237 +++++++++++++++++ .../letsencrypt.go => https/https.go} | 239 ++++-------------- .../https_test.go} | 25 +- caddy/https/maintain.go | 168 ++++++++++++ caddy/{setup/tls.go => https/setup.go} | 141 +++++++++-- .../tls_test.go => https/setup_test.go} | 118 ++++++--- caddy/{letsencrypt => https}/storage.go | 2 +- caddy/{letsencrypt => https}/storage_test.go | 2 +- caddy/{letsencrypt => https}/user.go | 20 +- caddy/{letsencrypt => https}/user_test.go | 10 +- caddy/letsencrypt/handshake.go | 99 -------- caddy/letsencrypt/maintain.go | 180 ------------- caddy/restart.go | 12 +- main.go | 12 +- server/config.go | 11 +- server/server.go | 119 ++------- 27 files changed, 1207 insertions(+), 687 deletions(-) create mode 100644 caddy/https/certificates.go create mode 100644 caddy/https/client.go rename caddy/{letsencrypt => https}/crypto.go (97%) rename caddy/{letsencrypt => https}/crypto_test.go (98%) rename caddy/{letsencrypt => https}/handler.go (98%) rename caddy/{letsencrypt => https}/handler_test.go (98%) create mode 100644 caddy/https/handshake.go rename caddy/{letsencrypt/letsencrypt.go => https/https.go} (59%) rename caddy/{letsencrypt/letsencrypt_test.go => https/https_test.go} (92%) create mode 100644 caddy/https/maintain.go rename caddy/{setup/tls.go => https/setup.go} (55%) rename caddy/{setup/tls_test.go => https/setup_test.go} (58%) rename caddy/{letsencrypt => https}/storage.go (99%) rename caddy/{letsencrypt => https}/storage_test.go (99%) rename caddy/{letsencrypt => https}/user.go (91%) rename caddy/{letsencrypt => https}/user_test.go (96%) delete mode 100644 caddy/letsencrypt/handshake.go delete mode 100644 caddy/letsencrypt/maintain.go diff --git a/caddy/caddy.go b/caddy/caddy.go index 600abe668..5d8ceddd8 100644 --- a/caddy/caddy.go +++ b/caddy/caddy.go @@ -28,7 +28,7 @@ import ( "sync" "time" - "github.com/mholt/caddy/caddy/letsencrypt" + "github.com/mholt/caddy/caddy/https" "github.com/mholt/caddy/server" ) @@ -44,7 +44,7 @@ var ( Quiet bool // HTTP2 indicates whether HTTP2 is enabled or not. - HTTP2 bool // TODO: temporary flag until http2 is standard + HTTP2 bool // PidFile is the path to the pidfile to create. PidFile string @@ -191,9 +191,13 @@ func startServers(groupings bindingGroup) error { if err != nil { return err } - s.HTTP2 = HTTP2 // TODO: This setting is temporary - s.ReqCallback = letsencrypt.RequestCallback // ensures we can solve ACME challenges while running - s.SNICallback = letsencrypt.GetCertificateDuringHandshake // TLS on demand -- awesome! + s.HTTP2 = HTTP2 + s.ReqCallback = https.RequestCallback // ensures we can solve ACME challenges while running + if s.OnDemandTLS { + s.TLSConfig.GetCertificate = https.GetOrObtainCertificate // TLS on demand -- awesome! + } else { + s.TLSConfig.GetCertificate = https.GetCertificate + } var ln server.ListenerFile if IsRestart() { @@ -278,7 +282,7 @@ func startServers(groupings bindingGroup) error { // It does NOT execute shutdown callbacks that may have been // configured by middleware (they must be executed separately). func Stop() error { - letsencrypt.Deactivate() + https.Deactivate() serversMu.Lock() for _, s := range servers { diff --git a/caddy/caddy_test.go b/caddy/caddy_test.go index ae84b31df..24a5d3026 100644 --- a/caddy/caddy_test.go +++ b/caddy/caddy_test.go @@ -4,10 +4,21 @@ import ( "net/http" "testing" "time" + + "github.com/mholt/caddy/caddy/https" + "github.com/xenolf/lego/acme" ) func TestCaddyStartStop(t *testing.T) { - caddyfile := "localhost:1984\ntls off" + // Use fake ACME clients for testing + https.NewACMEClient = func(email string, allowPrompts bool) (*https.ACMEClient, error) { + return &https.ACMEClient{ + Client: new(acme.Client), + AllowPrompts: allowPrompts, + }, nil + } + + caddyfile := "localhost:1984" for i := 0; i < 2; i++ { err := Start(CaddyfileInput{Contents: []byte(caddyfile)}) diff --git a/caddy/config.go b/caddy/config.go index 3ff63b481..15420e315 100644 --- a/caddy/config.go +++ b/caddy/config.go @@ -8,7 +8,7 @@ import ( "net" "sync" - "github.com/mholt/caddy/caddy/letsencrypt" + "github.com/mholt/caddy/caddy/https" "github.com/mholt/caddy/caddy/parse" "github.com/mholt/caddy/caddy/setup" "github.com/mholt/caddy/middleware" @@ -128,7 +128,7 @@ func loadConfigs(filename string, input io.Reader) ([]server.Config, error) { if !IsRestart() && !Quiet { fmt.Print("Activating privacy features...") } - configs, err = letsencrypt.Activate(configs) + configs, err = https.Activate(configs) if err != nil { return nil, err } else if !IsRestart() && !Quiet { @@ -318,7 +318,7 @@ func validDirective(d string) bool { // root. func DefaultInput() CaddyfileInput { port := Port - if letsencrypt.HostQualifies(Host) && port == DefaultPort { + if https.HostQualifies(Host) && port == DefaultPort { port = "443" } return CaddyfileInput{ diff --git a/caddy/directives.go b/caddy/directives.go index 39b54b7d6..d98ab5118 100644 --- a/caddy/directives.go +++ b/caddy/directives.go @@ -1,6 +1,7 @@ package caddy import ( + "github.com/mholt/caddy/caddy/https" "github.com/mholt/caddy/caddy/parse" "github.com/mholt/caddy/caddy/setup" "github.com/mholt/caddy/middleware" @@ -43,7 +44,7 @@ var directiveOrder = []directive{ // Essential directives that initialize vital configuration settings {"root", setup.Root}, {"bind", setup.BindHost}, - {"tls", setup.TLS}, // letsencrypt is set up just after tls + {"tls", https.Setup}, // Other directives that don't create HTTP handlers {"startup", setup.Startup}, diff --git a/caddy/helpers.go b/caddy/helpers.go index f864b54b4..0165573ac 100644 --- a/caddy/helpers.go +++ b/caddy/helpers.go @@ -11,14 +11,8 @@ import ( "strconv" "strings" "sync" - - "github.com/mholt/caddy/caddy/letsencrypt" ) -func init() { - letsencrypt.OnChange = func() error { return Restart(nil) } -} - // isLocalhost returns true if host looks explicitly like a localhost address. func isLocalhost(host string) bool { return host == "localhost" || host == "::1" || strings.HasPrefix(host, "127.") diff --git a/caddy/https/certificates.go b/caddy/https/certificates.go new file mode 100644 index 000000000..72a9ff1c7 --- /dev/null +++ b/caddy/https/certificates.go @@ -0,0 +1,232 @@ +package https + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "io/ioutil" + "log" + "strings" + "sync" + "time" + + "github.com/xenolf/lego/acme" + "golang.org/x/crypto/ocsp" +) + +// certCache stores certificates in memory, +// keying certificates by name. +var certCache = make(map[string]Certificate) +var certCacheMu sync.RWMutex + +// Certificate is a tls.Certificate with associated metadata tacked on. +// Even if the metadata can be obtained by parsing the certificate, +// we can be more efficient by extracting the metadata once so it's +// just there, ready to use. +type Certificate struct { + *tls.Certificate + + // Names is the list of names this certificate is written for. + // The first is the CommonName (if any), the rest are SAN. + Names []string + + // NotAfter is when the certificate expires. + NotAfter time.Time + + // Managed certificates are certificates that Caddy is managing, + // as opposed to the user specifying a certificate and key file + // or directory and managing the certificate resources themselves. + Managed bool + + // OnDemand certificates are obtained or loaded on-demand during TLS + // handshakes (as opposed to preloaded certificates, which are loaded + // at startup). If OnDemand is true, Managed must necessarily be true. + // OnDemand certificates are maintained in the background just like + // preloaded ones, however, if an OnDemand certificate fails to renew, + // it is removed from the in-memory cache. + OnDemand bool + + // OCSP contains the certificate's parsed OCSP response. + OCSP *ocsp.Response +} + +// getCertificate gets a certificate from the in-memory cache that +// matches name (a certificate name). Note that if name does not have +// an exact match, it will be checked against names of the form +// '*.example.com' (wildcard certificates) according to RFC 6125. +// +// If cert was found by matching name, matched will be returned true. +// If no match is found, the default certificate will be returned and +// matched will be returned as false. (The default certificate is the +// first one that entered the cache.) If the cache is empty (or there +// is no default certificate for some reason), matched will still be +// false, but cert.Certificate will be nil. +// +// The logic in this function is adapted from the Go standard library, +// which is by the Go Authors. +// +// This function is safe for concurrent use. +func getCertificate(name string) (cert Certificate, matched bool) { + // Not going to trim trailing dots here since RFC 3546 says, + // "The hostname is represented ... without a trailing dot." + // Just normalize to lowercase. + name = strings.ToLower(name) + + certCacheMu.RLock() + defer certCacheMu.RUnlock() + + // exact match? great, let's use it + if cert, ok := certCache[name]; ok { + return cert, true + } + + // try replacing labels in the name with wildcards until we get a match + labels := strings.Split(name, ".") + for i := range labels { + labels[i] = "*" + candidate := strings.Join(labels, ".") + if cert, ok := certCache[candidate]; ok { + return cert, true + } + } + + // if nothing matches, return the default certificate + cert = certCache[""] + return cert, false +} + +// cacheManagedCertificate loads the certificate for domain into the +// cache, flagging it as Managed and, if onDemand is true, as OnDemand +// (meaning that it was obtained or loaded during a TLS handshake). +// +// This function is safe for concurrent use. +func cacheManagedCertificate(domain string, onDemand bool) (Certificate, error) { + cert, err := makeCertificateFromDisk(storage.SiteCertFile(domain), storage.SiteKeyFile(domain)) + if err != nil { + return cert, err + } + cert.Managed = true + cert.OnDemand = onDemand + cacheCertificate(cert) + return cert, nil +} + +// cacheUnmanagedCertificatePEMFile loads a certificate for host using certFile +// and keyFile, which must be in PEM format. It stores the certificate in +// memory. The Managed and OnDemand flags of the certificate will be set to +// false. +// +// This function is safe for concurrent use. +func cacheUnmanagedCertificatePEMFile(certFile, keyFile string) error { + cert, err := makeCertificateFromDisk(certFile, keyFile) + if err != nil { + return err + } + cacheCertificate(cert) + return nil +} + +// cacheUnmanagedCertificatePEMBytes makes a certificate out of the PEM bytes +// of the certificate and key, then caches it in memory. +// +// This function is safe for concurrent use. +func cacheUnmanagedCertificatePEMBytes(certBytes, keyBytes []byte) error { + cert, err := makeCertificate(certBytes, keyBytes) + if err != nil { + return err + } + cacheCertificate(cert) + return nil +} + +// makeCertificateFromDisk makes a Certificate by loading the +// certificate and key files. It fills out all the fields in +// the certificate except for the Managed and OnDemand flags. +// (It is up to the caller to set those.) +func makeCertificateFromDisk(certFile, keyFile string) (Certificate, error) { + certPEMBlock, err := ioutil.ReadFile(certFile) + if err != nil { + return Certificate{}, err + } + keyPEMBlock, err := ioutil.ReadFile(keyFile) + if err != nil { + return Certificate{}, err + } + return makeCertificate(certPEMBlock, keyPEMBlock) +} + +// makeCertificate turns a certificate PEM bundle and a key PEM block into +// a Certificate, with OCSP and other relevant metadata tagged with it, +// except for the OnDemand and Managed flags. It is up to the caller to +// set those properties. +func makeCertificate(certPEMBlock, keyPEMBlock []byte) (Certificate, error) { + var cert Certificate + + // Convert to a tls.Certificate + tlsCert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) + if err != nil { + return cert, err + } + if len(tlsCert.Certificate) == 0 { + return cert, errors.New("certificate is empty") + } + cert.Certificate = &tlsCert + + // Parse leaf certificate and extract relevant metadata + leaf, err := x509.ParseCertificate(tlsCert.Certificate[0]) + if err != nil { + return cert, err + } + if leaf.Subject.CommonName != "" { + cert.Names = []string{strings.ToLower(leaf.Subject.CommonName)} + } + for _, name := range leaf.DNSNames { + if name != leaf.Subject.CommonName { + cert.Names = append(cert.Names, strings.ToLower(name)) + } + } + cert.NotAfter = leaf.NotAfter + + // Staple OCSP + ocspBytes, ocspResp, err := acme.GetOCSPForCert(certPEMBlock) + if err != nil { + // An error here is not a problem because a certificate may simply + // not contain a link to an OCSP server. But we should log it anyway. + log.Printf("[WARNING] No OCSP stapling for %v: %v", cert.Names, err) + } else if ocspResp.Status == ocsp.Good { + tlsCert.OCSPStaple = ocspBytes + cert.OCSP = ocspResp + } + + return cert, nil +} + +// cacheCertificate adds cert to the in-memory cache. If the cache is +// empty, cert will be used as the default certificate. If the cache is +// full, random entries are deleted until there is room to map all the +// names on the certificate. +// +// This certificate will be keyed to the names in cert.Names. Any name +// that is already a key in the cache will be replaced with this cert. +// +// This function is safe for concurrent use. +func cacheCertificate(cert Certificate) { + certCacheMu.Lock() + if _, ok := certCache[""]; !ok { + certCache[""] = cert // use as default + } + for len(certCache)+len(cert.Names) > 10000 { + // for simplicity, just remove random elements + for key := range certCache { + if key == "" { // ... but not the default cert + continue + } + delete(certCache, key) + break + } + } + for _, name := range cert.Names { + certCache[name] = cert + } + certCacheMu.Unlock() +} diff --git a/caddy/https/client.go b/caddy/https/client.go new file mode 100644 index 000000000..b47fd57f3 --- /dev/null +++ b/caddy/https/client.go @@ -0,0 +1,215 @@ +package https + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net" + "sync" + "time" + + "github.com/mholt/caddy/server" + "github.com/xenolf/lego/acme" +) + +// acmeMu ensures that only one ACME challenge occurs at a time. +var acmeMu sync.Mutex + +// ACMEClient is an acme.Client with custom state attached. +type ACMEClient struct { + *acme.Client + AllowPrompts bool // if false, we assume AlternatePort must be used +} + +// NewACMEClient creates a new ACMEClient given an email and whether +// prompting the user is allowed. Clients should not be kept and +// re-used over long periods of time, but immediate re-use is more +// efficient than re-creating on every iteration. +var NewACMEClient = func(email string, allowPrompts bool) (*ACMEClient, error) { + // Look up or create the LE user account + leUser, err := getUser(email) + if err != nil { + return nil, err + } + + // The client facilitates our communication with the CA server. + client, err := acme.NewClient(CAUrl, &leUser, rsaKeySizeToUse) + if err != nil { + return nil, err + } + + // If not registered, the user must register an account with the CA + // and agree to terms + if leUser.Registration == nil { + reg, err := client.Register() + if err != nil { + return nil, errors.New("registration error: " + err.Error()) + } + leUser.Registration = reg + + if allowPrompts { // can't prompt a user who isn't there + if !Agreed && reg.TosURL == "" { + Agreed = promptUserAgreement(saURL, false) // TODO - latest URL + } + if !Agreed && reg.TosURL == "" { + return nil, errors.New("user must agree to terms") + } + } + + err = client.AgreeToTOS() + if err != nil { + saveUser(leUser) // Might as well try, right? + return nil, errors.New("error agreeing to terms: " + err.Error()) + } + + // save user to the file system + err = saveUser(leUser) + if err != nil { + return nil, errors.New("could not save user: " + err.Error()) + } + } + + return &ACMEClient{ + Client: client, + AllowPrompts: allowPrompts, + }, nil +} + +// NewACMEClientGetEmail creates a new ACMEClient and gets an email +// address at the same time (a server config is required, since it +// may contain an email address in it). +func NewACMEClientGetEmail(config server.Config, allowPrompts bool) (*ACMEClient, error) { + return NewACMEClient(getEmail(config, allowPrompts), allowPrompts) +} + +// Configure configures c according to bindHost, which is the host (not +// whole address) to bind the listener to in solving the http and tls-sni +// challenges. +func (c *ACMEClient) Configure(bindHost string) { + // If we allow prompts, operator must be present. In our case, + // that is synonymous with saying the server is not already + // started. So if the user is still there, we don't use + // AlternatePort because we don't need to proxy the challenges. + // Conversely, if the operator is not there, the server has + // already started and we need to proxy the challenge. + if c.AllowPrompts { + // Operator is present; server is not already listening + c.SetHTTPAddress(net.JoinHostPort(bindHost, "")) + c.SetTLSAddress(net.JoinHostPort(bindHost, "")) + //c.ExcludeChallenges([]acme.Challenge{acme.DNS01}) + } else { + // Operator is not present; server is started, so proxy challenges + c.SetHTTPAddress(net.JoinHostPort(bindHost, AlternatePort)) + c.SetTLSAddress(net.JoinHostPort(bindHost, AlternatePort)) + //c.ExcludeChallenges([]acme.Challenge{acme.TLSSNI01, acme.DNS01}) + } + c.ExcludeChallenges([]acme.Challenge{acme.TLSSNI01, acme.DNS01}) // TODO: can we proxy TLS challenges? and we should support DNS... +} + +// Obtain obtains a single certificate for names. It stores the certificate +// on the disk if successful. +func (c *ACMEClient) Obtain(names []string) error { +Attempts: + for attempts := 0; attempts < 2; attempts++ { + acmeMu.Lock() + certificate, failures := c.ObtainCertificate(names, true, nil) + acmeMu.Unlock() + if len(failures) > 0 { + // Error - try to fix it or report it to the user and abort + var errMsg string // we'll combine all the failures into a single error message + var promptedForAgreement bool // only prompt user for agreement at most once + + for errDomain, obtainErr := range failures { + // TODO: Double-check, will obtainErr ever be nil? + if tosErr, ok := obtainErr.(acme.TOSError); ok { + // Terms of Service agreement error; we can probably deal with this + if !Agreed && !promptedForAgreement && c.AllowPrompts { + Agreed = promptUserAgreement(tosErr.Detail, true) // TODO: Use latest URL + promptedForAgreement = true + } + if Agreed || !c.AllowPrompts { + err := c.AgreeToTOS() + if err != nil { + return errors.New("error agreeing to updated terms: " + err.Error()) + } + continue Attempts + } + } + + // If user did not agree or it was any other kind of error, just append to the list of errors + errMsg += "[" + errDomain + "] failed to get certificate: " + obtainErr.Error() + "\n" + } + return errors.New(errMsg) + } + + // Success - immediately save the certificate resource + err := saveCertResource(certificate) + if err != nil { + return fmt.Errorf("error saving assets for %v: %v", names, err) + } + + break + } + + return nil +} + +// Renew renews the managed certificate for name. Right now our storage +// mechanism only supports one name per certificate, so this function only +// accepts one domain as input. It can be easily modified to support SAN +// certificates if, one day, they become desperately needed enough that our +// storage mechanism is upgraded to be more complex to support SAN certs. +// +// Anyway, this function is safe for concurrent use. +func (c *ACMEClient) Renew(name string) error { + // Prepare for renewal (load PEM cert, key, and meta) + certBytes, err := ioutil.ReadFile(storage.SiteCertFile(name)) + if err != nil { + return err + } + keyBytes, err := ioutil.ReadFile(storage.SiteKeyFile(name)) + if err != nil { + return err + } + metaBytes, err := ioutil.ReadFile(storage.SiteMetaFile(name)) + if err != nil { + return err + } + var certMeta acme.CertificateResource + err = json.Unmarshal(metaBytes, &certMeta) + certMeta.Certificate = certBytes + certMeta.PrivateKey = keyBytes + + // Perform renewal and retry if necessary, but not too many times. + var newCertMeta acme.CertificateResource + var success bool + for attempts := 0; attempts < 2; attempts++ { + acmeMu.Lock() + newCertMeta, err = c.RenewCertificate(certMeta, true) + acmeMu.Unlock() + if err == nil { + success = true + break + } + + // If the legal terms changed and need to be agreed to again, + // we can handle that. + if _, ok := err.(acme.TOSError); ok { + err := c.AgreeToTOS() + if err != nil { + return err + } + continue + } + + // For any other kind of error, wait 10s and try again. + time.Sleep(10 * time.Second) + } + + if !success { + return errors.New("too many renewal attempts; last error: " + err.Error()) + } + + return saveCertResource(newCertMeta) +} diff --git a/caddy/letsencrypt/crypto.go b/caddy/https/crypto.go similarity index 97% rename from caddy/letsencrypt/crypto.go rename to caddy/https/crypto.go index 95f2069de..efc40d434 100644 --- a/caddy/letsencrypt/crypto.go +++ b/caddy/https/crypto.go @@ -1,4 +1,4 @@ -package letsencrypt +package https import ( "crypto/rsa" diff --git a/caddy/letsencrypt/crypto_test.go b/caddy/https/crypto_test.go similarity index 98% rename from caddy/letsencrypt/crypto_test.go rename to caddy/https/crypto_test.go index 672095d90..875f2d217 100644 --- a/caddy/letsencrypt/crypto_test.go +++ b/caddy/https/crypto_test.go @@ -1,4 +1,4 @@ -package letsencrypt +package https import ( "bytes" diff --git a/caddy/letsencrypt/handler.go b/caddy/https/handler.go similarity index 98% rename from caddy/letsencrypt/handler.go rename to caddy/https/handler.go index e147e00c8..5b7fa0118 100644 --- a/caddy/letsencrypt/handler.go +++ b/caddy/https/handler.go @@ -1,4 +1,4 @@ -package letsencrypt +package https import ( "crypto/tls" diff --git a/caddy/letsencrypt/handler_test.go b/caddy/https/handler_test.go similarity index 98% rename from caddy/letsencrypt/handler_test.go rename to caddy/https/handler_test.go index ac6f48001..016799ffb 100644 --- a/caddy/letsencrypt/handler_test.go +++ b/caddy/https/handler_test.go @@ -1,4 +1,4 @@ -package letsencrypt +package https import ( "net" diff --git a/caddy/https/handshake.go b/caddy/https/handshake.go new file mode 100644 index 000000000..e06e7d0da --- /dev/null +++ b/caddy/https/handshake.go @@ -0,0 +1,237 @@ +package https + +import ( + "bytes" + "crypto/tls" + "encoding/pem" + "errors" + "fmt" + "log" + "sync" + "time" + + "github.com/mholt/caddy/server" + "github.com/xenolf/lego/acme" +) + +// GetCertificate gets a certificate to satisfy clientHello as long as +// the certificate is already cached in memory. +// +// This function is safe for use as a tls.Config.GetCertificate callback. +func GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := getCertDuringHandshake(clientHello.ServerName, false) + return cert.Certificate, err +} + +// GetOrObtainCertificate will get a certificate to satisfy clientHello, even +// if that means obtaining a new certificate from a CA during the handshake. +// It first checks the in-memory cache, then accesses disk, then accesses the +// network if it must. An obtained certificate will be stored on disk and +// cached in memory. +// +// This function is safe for use as a tls.Config.GetCertificate callback. +func GetOrObtainCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := getCertDuringHandshake(clientHello.ServerName, true) + return cert.Certificate, err +} + +// getCertDuringHandshake will get a certificate for name. It first tries +// the in-memory cache, then, if obtainIfNecessary is true, it goes to disk, +// then asks the CA for a certificate if necessary. +// +// This function is safe for concurrent use. +func getCertDuringHandshake(name string, obtainIfNecessary bool) (Certificate, error) { + // First check our in-memory cache to see if we've already loaded it + cert, ok := getCertificate(name) + if ok { + return cert, nil + } + + if obtainIfNecessary { + // TODO: Mitigate abuse! + var err error + + // Then check to see if we have one on disk + cert, err := cacheManagedCertificate(name, true) + if err != nil { + return cert, err + } else if cert.Certificate != nil { + cert, err := handshakeMaintenance(name, cert) + if err != nil { + log.Printf("[ERROR] Maintaining newly-loaded certificate for %s: %v", name, err) + } + return cert, err + } + + // Only option left is to get one from LE, but the name has to qualify first + if !HostQualifies(name) { + return cert, errors.New("hostname '" + name + "' does not qualify for certificate") + } + + // By this point, we need to obtain one from the CA. + return obtainOnDemandCertificate(name) + } + + return Certificate{}, nil +} + +// obtainOnDemandCertificate obtains a certificate for name for the given +// clientHello. If another goroutine has already started obtaining a cert +// for name, it will wait and use what the other goroutine obtained. +// +// This function is safe for use by multiple concurrent goroutines. +func obtainOnDemandCertificate(name string) (Certificate, error) { + // We must protect this process from happening concurrently, so synchronize. + obtainCertWaitChansMu.Lock() + wait, ok := obtainCertWaitChans[name] + if ok { + // lucky us -- another goroutine is already obtaining the certificate. + // wait for it to finish obtaining the cert and then we'll use it. + obtainCertWaitChansMu.Unlock() + <-wait + return getCertDuringHandshake(name, false) // passing in true might result in infinite loop if obtain failed + } + + // looks like it's up to us to do all the work and obtain the cert + wait = make(chan struct{}) + obtainCertWaitChans[name] = wait + obtainCertWaitChansMu.Unlock() + + // Unblock waiters and delete waitgroup when we return + defer func() { + obtainCertWaitChansMu.Lock() + close(wait) + delete(obtainCertWaitChans, name) + obtainCertWaitChansMu.Unlock() + }() + + log.Printf("[INFO] Obtaining new certificate for %s", name) + + // obtain cert + client, err := NewACMEClientGetEmail(server.Config{}, false) + if err != nil { + return Certificate{}, errors.New("error creating client: " + err.Error()) + } + client.Configure("") // TODO: which BindHost? + err = client.Obtain([]string{name}) + if err != nil { + return Certificate{}, err + } + + // The certificate is on disk; now just start over to load it and serve it + return getCertDuringHandshake(name, false) // pass in false as a fail-safe from infinite-looping +} + +// handshakeMaintenance performs a check on cert for expiration and OCSP +// validity. +// +// This function is safe for use by multiple concurrent goroutines. +func handshakeMaintenance(name string, cert Certificate) (Certificate, error) { + // fmt.Println("ON-DEMAND CERT?", cert.OnDemand) + // if !cert.OnDemand { + // return cert, nil + // } + fmt.Println("Checking expiration of cert; on-demand:", cert.OnDemand) + + // Check cert expiration + timeLeft := cert.NotAfter.Sub(time.Now().UTC()) + if timeLeft < renewDurationBefore { + log.Printf("[INFO] Certificate for %v expires in %v; attempting renewal", cert.Names, timeLeft) + return renewDynamicCertificate(name) + } + + // Check OCSP staple validity + if cert.OCSP != nil { + refreshTime := cert.OCSP.ThisUpdate.Add(cert.OCSP.NextUpdate.Sub(cert.OCSP.ThisUpdate) / 2) + if time.Now().After(refreshTime) { + err := stapleOCSP(&cert, nil) + if err != nil { + // An error with OCSP stapling is not the end of the world, and in fact, is + // quite common considering not all certs have issuer URLs that support it. + log.Printf("[ERROR] Getting OCSP for %s: %v", name, err) + } + certCacheMu.Lock() + certCache[name] = cert + certCacheMu.Unlock() + } + } + + return cert, nil +} + +// renewDynamicCertificate renews currentCert using the clientHello. It returns the +// certificate to use and an error, if any. currentCert may be returned even if an +// error occurs, since we perform renewals before they expire and it may still be +// usable. name should already be lower-cased before calling this function. +// +// This function is safe for use by multiple concurrent goroutines. +func renewDynamicCertificate(name string) (Certificate, error) { + obtainCertWaitChansMu.Lock() + wait, ok := obtainCertWaitChans[name] + if ok { + // lucky us -- another goroutine is already renewing the certificate. + // wait for it to finish, then we'll use the new one. + obtainCertWaitChansMu.Unlock() + <-wait + return getCertDuringHandshake(name, false) + } + + // looks like it's up to us to do all the work and renew the cert + wait = make(chan struct{}) + obtainCertWaitChans[name] = wait + obtainCertWaitChansMu.Unlock() + + // unblock waiters and delete waitgroup when we return + defer func() { + obtainCertWaitChansMu.Lock() + close(wait) + delete(obtainCertWaitChans, name) + obtainCertWaitChansMu.Unlock() + }() + + log.Printf("[INFO] Renewing certificate for %s", name) + + client, err := NewACMEClient("", false) // renewals don't use email + if err != nil { + return Certificate{}, err + } + client.Configure("") // TODO: Bind address of relevant listener, yuck + err = client.Renew(name) + if err != nil { + return Certificate{}, err + } + + return getCertDuringHandshake(name, false) +} + +// stapleOCSP staples OCSP information to cert for hostname name. +// If you have it handy, you should pass in the PEM-encoded certificate +// bundle; otherwise the DER-encoded cert will have to be PEM-encoded. +// If you don't have the PEM blocks handy, just pass in nil. +// +// Errors here are not necessarily fatal, it could just be that the +// certificate doesn't have an issuer URL. +func stapleOCSP(cert *Certificate, pemBundle []byte) error { + if pemBundle == nil { + // The function in the acme package that gets OCSP requires a PEM-encoded cert + bundle := new(bytes.Buffer) + for _, derBytes := range cert.Certificate.Certificate { + pem.Encode(bundle, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + } + pemBundle = bundle.Bytes() + } + + ocspBytes, ocspResp, err := acme.GetOCSPForCert(pemBundle) + if err != nil { + return err + } + + cert.Certificate.OCSPStaple = ocspBytes + cert.OCSP = ocspResp + + return nil +} + +// obtainCertWaitChans is used to coordinate obtaining certs for each hostname. +var obtainCertWaitChans = make(map[string]chan struct{}) +var obtainCertWaitChansMu sync.Mutex diff --git a/caddy/letsencrypt/letsencrypt.go b/caddy/https/https.go similarity index 59% rename from caddy/letsencrypt/letsencrypt.go rename to caddy/https/https.go index d6fb9cc37..2dd1bea39 100644 --- a/caddy/letsencrypt/letsencrypt.go +++ b/caddy/https/https.go @@ -1,12 +1,12 @@ -// Package letsencrypt integrates Let's Encrypt functionality into Caddy -// with first-class support for creating and renewing certificates -// automatically. It is designed to configure sites for HTTPS by default. -package letsencrypt +// Package https facilitates the management of TLS assets and integrates +// Let's Encrypt functionality into Caddy with first-class support for +// creating and renewing certificates automatically. It is designed to +// configure sites for HTTPS by default. +package https import ( "encoding/json" "errors" - "fmt" "io/ioutil" "net" "net/http" @@ -14,9 +14,6 @@ import ( "strings" "time" - "golang.org/x/crypto/ocsp" - - "github.com/mholt/caddy/caddy/setup" "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/middleware/redirect" "github.com/mholt/caddy/server" @@ -38,34 +35,27 @@ import ( // // Also note that calling this function activates asset // management automatically, which keeps certificates -// renewed and OCSP stapling updated. This has the effect -// of causing restarts when assets are updated. +// renewed and OCSP stapling updated. // // Activate returns the updated list of configs, since // some may have been appended, for example, to redirect // plaintext HTTP requests to their HTTPS counterpart. -// This function only appends; it does not prepend or splice. +// This function only appends; it does not splice. func Activate(configs []server.Config) ([]server.Config, error) { // just in case previous caller forgot... Deactivate() - // reset cached ocsp from any previous activations - ocspCache = make(map[*[]byte]*ocsp.Response) - // pre-screen each config and earmark the ones that qualify for managed TLS MarkQualified(configs) // place certificates and keys on disk - err := ObtainCerts(configs, "") + err := ObtainCerts(configs, true) if err != nil { return configs, err } // update TLS configurations - EnableTLS(configs) - - // enable OCSP stapling (this affects all TLS-enabled configs) - err = StapleOCSP(configs) + err = EnableTLS(configs, true) if err != nil { return configs, err } @@ -78,17 +68,18 @@ func Activate(configs []server.Config) ([]server.Config, error) { // the renewal ticker is reset, so if restarts happen more often than // the ticker interval, renewals would never happen. but doing // it right away at start guarantees that renewals aren't missed. - renewCertificates(configs, false) + client, err := NewACMEClient("", true) // renewals don't use email + if err != nil { + return configs, err + } + client.Configure("") + err = renewManagedCertificates(client) + if err != nil { + return configs, err + } // keep certificates renewed and OCSP stapling updated - go maintainAssets(configs, stopChan) - - // TODO - experimental dynamic TLS! - for i := range configs { - if configs[i].Host == "" && configs[i].Port == "443" { - configs[i].TLS.Enabled = true - } - } + go maintainAssets(stopChan) return configs, nil } @@ -121,11 +112,16 @@ func MarkQualified(configs []server.Config) { // ObtainCerts obtains certificates for all these configs as long as a certificate does not // already exist on disk. It does not modify the configs at all; it only obtains and stores // certificates and keys to the disk. -func ObtainCerts(configs []server.Config, altPort string) error { - groupedConfigs := groupConfigsByEmail(configs, altPort != "") // don't prompt user if server already running +func ObtainCerts(configs []server.Config, allowPrompts bool) error { + // We group configs by email so we don't make the same clients over and + // over. This has the potential to prompt the user for an email, but we + // prevent that by assuming that if we already have a listener that can + // proxy ACME challenge requests, then the server is already running and + // the operator is no longer present. + groupedConfigs := groupConfigsByEmail(configs, allowPrompts) for email, group := range groupedConfigs { - client, err := newClientPort(email, altPort) + client, err := NewACMEClient(email, allowPrompts) if err != nil { return errors.New("error creating client: " + err.Error()) } @@ -135,7 +131,9 @@ func ObtainCerts(configs []server.Config, altPort string) error { continue } - err := clientObtain(client, []string{cfg.Host}, altPort == "") + client.Configure(cfg.BindHost) + + err := client.Obtain([]string{cfg.Host}) if err != nil { return err } @@ -147,15 +145,14 @@ func ObtainCerts(configs []server.Config, altPort string) error { // groupConfigsByEmail groups configs by the email address to be used by its // ACME client. It only includes configs that are marked as fully managed. -// This is the function that may prompt for an email address, unless skipPrompt -// is true, in which case it will assume an empty email address. -func groupConfigsByEmail(configs []server.Config, skipPrompt bool) map[string][]server.Config { +// If userPresent is true, the operator MAY be prompted for an email address. +func groupConfigsByEmail(configs []server.Config, userPresent bool) map[string][]server.Config { initMap := make(map[string][]server.Config) for _, cfg := range configs { if !cfg.TLS.Managed { continue } - leEmail := getEmail(cfg, skipPrompt) + leEmail := getEmail(cfg, userPresent) initMap[leEmail] = append(initMap[leEmail], cfg) } return initMap @@ -163,50 +160,24 @@ func groupConfigsByEmail(configs []server.Config, skipPrompt bool) map[string][] // EnableTLS configures each config to use TLS according to default settings. // It will only change configs that are marked as managed, and assumes that -// certificates and keys are already on disk. -func EnableTLS(configs []server.Config) { +// certificates and keys are already on disk. If loadCertificates is true, +// the certificates will be loaded from disk into the cache for this process +// to use. If false, TLS will still be enabled and configured with default +// settings, but no certificates will be parsed loaded into the cache, and +// the returned error value will always be nil. +func EnableTLS(configs []server.Config, loadCertificates bool) error { for i := 0; i < len(configs); i++ { if !configs[i].TLS.Managed { continue } configs[i].TLS.Enabled = true - if configs[i].Host != "" { - configs[i].TLS.Certificate = storage.SiteCertFile(configs[i].Host) - configs[i].TLS.Key = storage.SiteKeyFile(configs[i].Host) - } - setup.SetDefaultTLSParams(&configs[i]) - } -} - -// StapleOCSP staples OCSP responses to each config according to their certificate. -// This should work for any TLS-enabled config, not just Let's Encrypt ones. -func StapleOCSP(configs []server.Config) error { - for i := 0; i < len(configs); i++ { - if configs[i].TLS.Certificate == "" { - continue - } - - bundleBytes, err := ioutil.ReadFile(configs[i].TLS.Certificate) - if err != nil { - return errors.New("load certificate to staple ocsp: " + err.Error()) - } - - ocspBytes, ocspResp, err := acme.GetOCSPForCert(bundleBytes) - if err == nil { - // TODO: We ignore the error if it exists because some certificates - // may not have an issuer URL which we should ignore anyway, and - // sometimes we get syntax errors in the responses. To reproduce this - // behavior, start Caddy with an empty Caddyfile and -log stderr. Then - // add a host to the Caddyfile which requires a new LE certificate. - // Reload Caddy's config with SIGUSR1, and see the log report that it - // obtains the certificate, but then an error: - // getting ocsp: asn1: syntax error: sequence truncated - // But retrying the reload again sometimes solves the problem. It's flaky... - ocspCache[&bundleBytes] = ocspResp - if ocspResp.Status == ocsp.Good { - configs[i].TLS.OCSPStaple = ocspBytes + if loadCertificates && configs[i].Host != "" { + _, err := cacheManagedCertificate(configs[i].Host, false) + if err != nil { + return err } } + setDefaultTLSParams(&configs[i]) } return nil } @@ -251,8 +222,7 @@ func MakePlaintextRedirects(allConfigs []server.Config) []server.Config { // setting up the config may make it look like it // doesn't qualify even though it originally did. func ConfigQualifies(cfg server.Config) bool { - return cfg.TLS.Certificate == "" && // user could provide their own cert and key - cfg.TLS.Key == "" && + return !cfg.TLS.Manual && // user can provide own cert and key // user can force-disable automatic HTTPS for this host cfg.Scheme != "http" && @@ -297,71 +267,6 @@ func existingCertAndKey(host string) bool { return true } -// newClient creates a new ACME client to facilitate communication -// with the Let's Encrypt CA server on behalf of the user specified -// by leEmail. As part of this process, a user will be loaded from -// disk (if already exists) or created new and registered via ACME -// and saved to the file system for next time. -func newClient(leEmail string) (*acme.Client, error) { - return newClientPort(leEmail, "") -} - -// newClientPort does the same thing as newClient, except it creates a -// new client with a custom port used for ACME transactions instead of -// the default port. This is important if the default port is already in -// use or is not exposed to the public, etc. -func newClientPort(leEmail, port string) (*acme.Client, error) { - // Look up or create the LE user account - leUser, err := getUser(leEmail) - if err != nil { - return nil, err - } - - // The client facilitates our communication with the CA server. - client, err := acme.NewClient(CAUrl, &leUser, rsaKeySizeToUse) - if err != nil { - return nil, err - } - if port != "" { - client.SetHTTPAddress(":" + port) - client.SetTLSAddress(":" + port) - } - client.ExcludeChallenges([]acme.Challenge{acme.TLSSNI01, acme.DNS01}) // We can only guarantee http-01 at this time, but tls-01 should work if port is not custom! - - // If not registered, the user must register an account with the CA - // and agree to terms - if leUser.Registration == nil { - reg, err := client.Register() - if err != nil { - return nil, errors.New("registration error: " + err.Error()) - } - leUser.Registration = reg - - if port == "" { // can't prompt a user who isn't there - if !Agreed && reg.TosURL == "" { - Agreed = promptUserAgreement(saURL, false) // TODO - latest URL - } - if !Agreed && reg.TosURL == "" { - return nil, errors.New("user must agree to terms") - } - } - - err = client.AgreeToTOS() - if err != nil { - saveUser(leUser) // TODO: Might as well try, right? Error check? - return nil, errors.New("error agreeing to terms: " + err.Error()) - } - - // save user to the file system - err = saveUser(leUser) - if err != nil { - return nil, errors.New("could not save user: " + err.Error()) - } - } - - return client, nil -} - // saveCertResource saves the certificate resource to disk. This // includes the certificate file itself, the private key, and the // metadata file. @@ -427,61 +332,18 @@ func redirPlaintextHost(cfg server.Config) server.Config { } } -// clientObtain uses client to obtain a single certificate for domains in names. If -// the user is present to provide an email address, pass in true for allowPrompt, -// otherwise pass in false. If err == nil, the certificate (and key) will be saved -// to disk in the storage folder. -func clientObtain(client *acme.Client, names []string, allowPrompt bool) error { - certificate, failures := client.ObtainCertificate(names, true, nil) - if len(failures) > 0 { - // Error - either try to fix it or report them it to the user and abort - var errMsg string // we'll combine all the failures into a single error message - var promptedForAgreement bool // only prompt user for agreement at most once - - for errDomain, obtainErr := range failures { - // TODO: Double-check, will obtainErr ever be nil? - if tosErr, ok := obtainErr.(acme.TOSError); ok { - // Terms of Service agreement error; we can probably deal with this - if !Agreed && !promptedForAgreement && allowPrompt { // don't prompt if server is already running - Agreed = promptUserAgreement(tosErr.Detail, true) // TODO: Use latest URL - promptedForAgreement = true - } - if Agreed || !allowPrompt { - err := client.AgreeToTOS() - if err != nil { - return errors.New("error agreeing to updated terms: " + err.Error()) - } - return clientObtain(client, names, allowPrompt) - } - } - - // If user did not agree or it was any other kind of error, just append to the list of errors - errMsg += "[" + errDomain + "] failed to get certificate: " + obtainErr.Error() + "\n" - } - return errors.New(errMsg) - } - - // Success - immediately save the certificate resource - err := saveCertResource(certificate) - if err != nil { - return fmt.Errorf("error saving assets for %v: %v", names, err) - } - - return nil -} - // Revoke revokes the certificate for host via ACME protocol. func Revoke(host string) error { if !existingCertAndKey(host) { return errors.New("no certificate and key for " + host) } - email := getEmail(server.Config{Host: host}, false) + email := getEmail(server.Config{Host: host}, true) if email == "" { return errors.New("email is required to revoke") } - client, err := newClient(email) + client, err := NewACMEClient(email, true) if err != nil { return err } @@ -525,7 +387,7 @@ const ( AlternatePort = "5033" // RenewInterval is how often to check certificates for renewal. - RenewInterval = 24 * time.Hour + RenewInterval = 6 * time.Hour // OCSPInterval is how often to check if OCSP stapling needs updating. OCSPInterval = 1 * time.Hour @@ -550,8 +412,3 @@ var rsaKeySizeToUse = Rsa2048 // stopChan is used to signal the maintenance goroutine // to terminate. var stopChan chan struct{} - -// ocspCache maps certificate bundle to OCSP response. -// It is used during regular OCSP checks to see if the OCSP -// response needs to be updated. -var ocspCache = make(map[*[]byte]*ocsp.Response) diff --git a/caddy/letsencrypt/letsencrypt_test.go b/caddy/https/https_test.go similarity index 92% rename from caddy/letsencrypt/letsencrypt_test.go rename to caddy/https/https_test.go index e3ac2212e..e4efd2373 100644 --- a/caddy/letsencrypt/letsencrypt_test.go +++ b/caddy/https/https_test.go @@ -1,4 +1,4 @@ -package letsencrypt +package https import ( "io/ioutil" @@ -48,9 +48,9 @@ func TestConfigQualifies(t *testing.T) { }{ {server.Config{Host: ""}, true}, {server.Config{Host: "localhost"}, false}, + {server.Config{Host: "123.44.3.21"}, false}, {server.Config{Host: "example.com"}, true}, - {server.Config{Host: "example.com", TLS: server.TLSConfig{Certificate: "cert.pem"}}, false}, - {server.Config{Host: "example.com", TLS: server.TLSConfig{Key: "key.pem"}}, false}, + {server.Config{Host: "example.com", TLS: server.TLSConfig{Manual: true}}, false}, {server.Config{Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "off"}}, false}, {server.Config{Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "foo@bar.com"}}, true}, {server.Config{Host: "example.com", Scheme: "http"}, false}, @@ -257,27 +257,14 @@ func TestEnableTLS(t *testing.T) { server.Config{}, // not managed - no changes! } - EnableTLS(configs) + EnableTLS(configs, false) if !configs[0].TLS.Enabled { t.Errorf("Expected config 0 to have TLS.Enabled == true, but it was false") } - if configs[0].TLS.Certificate == "" { - t.Errorf("Expected config 0 to have TLS.Certificate set, but it was empty") - } - if configs[0].TLS.Key == "" { - t.Errorf("Expected config 0 to have TLS.Key set, but it was empty") - } - if configs[1].TLS.Enabled { t.Errorf("Expected config 1 to have TLS.Enabled == false, but it was true") } - if configs[1].TLS.Certificate != "" { - t.Errorf("Expected config 1 to have TLS.Certificate empty, but it was: %s", configs[1].TLS.Certificate) - } - if configs[1].TLS.Key != "" { - t.Errorf("Expected config 1 to have TLS.Key empty, but it was: %s", configs[1].TLS.Key) - } } func TestGroupConfigsByEmail(t *testing.T) { @@ -316,9 +303,9 @@ func TestMarkQualified(t *testing.T) { // TODO: TestConfigQualifies and this test share the same config list... configs := []server.Config{ {Host: "localhost"}, + {Host: "123.44.3.21"}, {Host: "example.com"}, - {Host: "example.com", TLS: server.TLSConfig{Certificate: "cert.pem"}}, - {Host: "example.com", TLS: server.TLSConfig{Key: "key.pem"}}, + {Host: "example.com", TLS: server.TLSConfig{Manual: true}}, {Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "off"}}, {Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "foo@bar.com"}}, {Host: "example.com", Scheme: "http"}, diff --git a/caddy/https/maintain.go b/caddy/https/maintain.go new file mode 100644 index 000000000..03d841c72 --- /dev/null +++ b/caddy/https/maintain.go @@ -0,0 +1,168 @@ +package https + +import ( + "log" + "time" + + "golang.org/x/crypto/ocsp" +) + +// maintainAssets is a permanently-blocking function +// that loops indefinitely and, on a regular schedule, checks +// certificates for expiration and initiates a renewal of certs +// that are expiring soon. It also updates OCSP stapling and +// performs other maintenance of assets. +// +// You must pass in the channel which you'll close when +// maintenance should stop, to allow this goroutine to clean up +// after itself and unblock. +func maintainAssets(stopChan chan struct{}) { + renewalTicker := time.NewTicker(RenewInterval) + ocspTicker := time.NewTicker(OCSPInterval) + + for { + select { + case <-renewalTicker.C: + log.Println("[INFO] Scanning for expiring certificates") + client, err := NewACMEClient("", false) // renewals don't use email + if err != nil { + log.Printf("[ERROR] Creating client for renewals: %v", err) + continue + } + client.Configure("") // TODO: Bind address of relevant listener, yuck + renewManagedCertificates(client) + log.Println("[INFO] Done checking certificates") + case <-ocspTicker.C: + log.Println("[INFO] Scanning for stale OCSP staples") + updatePreloadedOCSPStaples() + log.Println("[INFO] Done checking OCSP staples") + case <-stopChan: + renewalTicker.Stop() + ocspTicker.Stop() + log.Println("[INFO] Stopped background maintenance routine") + return + } + } +} + +func renewManagedCertificates(client *ACMEClient) error { + var renewed, deleted []Certificate + visitedNames := make(map[string]struct{}) + + certCacheMu.RLock() + for name, cert := range certCache { + if !cert.Managed { + continue + } + + // the list of names on this cert should never be empty... + if cert.Names == nil || len(cert.Names) == 0 { + log.Printf("[WARNING] Certificate keyed by '%s' has no names: %v", name, cert.Names) + deleted = append(deleted, cert) + continue + } + + // skip names whose certificate we've already renewed + if _, ok := visitedNames[name]; ok { + continue + } + for _, name := range cert.Names { + visitedNames[name] = struct{}{} + } + + timeLeft := cert.NotAfter.Sub(time.Now().UTC()) + if timeLeft < renewDurationBefore { + log.Printf("[INFO] Certificate for %v expires in %v; attempting renewal", cert.Names, timeLeft) + err := client.Renew(cert.Names[0]) // managed certs better have only one name + if err != nil { + if client.AllowPrompts { + // User is present, so stop immediately and report the error + certCacheMu.RUnlock() + return err + } + log.Printf("[ERROR] %v", err) + if cert.OnDemand { + deleted = append(deleted, cert) + } + } else { + renewed = append(renewed, cert) + } + } + } + certCacheMu.RUnlock() + + // Apply changes to the cache + for _, cert := range renewed { + _, err := cacheManagedCertificate(cert.Names[0], cert.OnDemand) + if err != nil { + if client.AllowPrompts { + return err // operator is present, so report error immediately + } + log.Printf("[ERROR] %v", err) + } + } + for _, cert := range deleted { + certCacheMu.Lock() + for _, name := range cert.Names { + delete(certCache, name) + } + certCacheMu.Unlock() + } + + return nil +} + +func updatePreloadedOCSPStaples() { + // Create a temporary place to store updates + // until we release the potentially slow read + // lock so we can use a quick write lock. + type ocspUpdate struct { + rawBytes []byte + parsedResponse *ocsp.Response + } + updated := make(map[string]ocspUpdate) + + certCacheMu.RLock() + for name, cert := range certCache { + // we update OCSP for managed and un-managed certs here, but only + // if it has OCSP stapled and only for pre-loaded certificates + if cert.OnDemand || cert.OCSP == nil { + continue + } + + // start checking OCSP staple about halfway through validity period for good measure + oldNextUpdate := cert.OCSP.NextUpdate + refreshTime := cert.OCSP.ThisUpdate.Add(oldNextUpdate.Sub(cert.OCSP.ThisUpdate) / 2) + + // only check for updated OCSP validity window if the refresh time is + // in the past and the certificate is not expired + if time.Now().After(refreshTime) && time.Now().Before(cert.NotAfter) { + err := stapleOCSP(&cert, nil) + if err != nil { + log.Printf("[ERROR] Checking OCSP for %s: %v", name, err) + continue + } + + // if the OCSP response has been updated, we use it + if oldNextUpdate != cert.OCSP.NextUpdate { + log.Printf("[INFO] Moving validity period of OCSP staple for %s from %v to %v", + name, oldNextUpdate, cert.OCSP.NextUpdate) + updated[name] = ocspUpdate{rawBytes: cert.Certificate.OCSPStaple, parsedResponse: cert.OCSP} + } + } + } + certCacheMu.RUnlock() + + // This write lock should be brief since we have all the info we need now. + certCacheMu.Lock() + for name, update := range updated { + cert := certCache[name] + cert.OCSP = update.parsedResponse + cert.Certificate.OCSPStaple = update.rawBytes + certCache[name] = cert + } + certCacheMu.Unlock() +} + +// renewDurationBefore is how long before expiration to renew certificates. +const renewDurationBefore = (24 * time.Hour) * 30 diff --git a/caddy/setup/tls.go b/caddy/https/setup.go similarity index 55% rename from caddy/setup/tls.go rename to caddy/https/setup.go index cf45278ca..592dfee59 100644 --- a/caddy/setup/tls.go +++ b/caddy/https/setup.go @@ -1,16 +1,24 @@ -package setup +package https import ( + "bytes" "crypto/tls" + "encoding/pem" + "io/ioutil" "log" + "os" + "path/filepath" "strings" + "github.com/mholt/caddy/caddy/setup" "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/server" ) -// TLS sets up the TLS configuration (but does not activate Let's Encrypt; that is handled elsewhere). -func TLS(c *Controller) (middleware.Middleware, error) { +// Setup sets up the TLS configuration and installs certificates that +// are specified by the user in the config file. All the automatic HTTPS +// stuff comes later outside of this function. +func Setup(c *setup.Controller) (middleware.Middleware, error) { if c.Scheme == "http" { c.TLS.Enabled = false log.Printf("[WARNING] TLS disabled for %s://%s.", c.Scheme, c.Address()) @@ -19,18 +27,21 @@ func TLS(c *Controller) (middleware.Middleware, error) { } for c.Next() { + var certificateFile, keyFile, loadDir string + args := c.RemainingArgs() switch len(args) { case 1: c.TLS.LetsEncryptEmail = args[0] - // user can force-disable LE activation this way + // user can force-disable managed TLS this way if c.TLS.LetsEncryptEmail == "off" { c.TLS.Enabled = false } case 2: - c.TLS.Certificate = args[0] - c.TLS.Key = args[1] + certificateFile = args[0] + keyFile = args[1] + c.TLS.Manual = true } // Optional block with extra parameters @@ -66,9 +77,9 @@ func TLS(c *Controller) (middleware.Middleware, error) { if len(c.TLS.ClientCerts) == 0 { return nil, c.ArgErr() } - // TODO: Allow this? It's a bad idea to allow HTTP. If we do this, make sure invoking tls at all (even manually) also sets up a redirect if possible? - // case "allow_http": - // c.TLS.DisableHTTPRedir = true + case "load": + c.Args(&loadDir) + c.TLS.Manual = true default: return nil, c.Errf("Unknown keyword '%s'", c.Val()) } @@ -78,18 +89,112 @@ func TLS(c *Controller) (middleware.Middleware, error) { if len(args) == 0 && !hadBlock { return nil, c.ArgErr() } + + // don't load certificates unless we're supposed to + if !c.TLS.Enabled || !c.TLS.Manual { + continue + } + + // load a single certificate and key, if specified + if certificateFile != "" && keyFile != "" { + err := cacheUnmanagedCertificatePEMFile(certificateFile, keyFile) + if err != nil { + return nil, c.Errf("Unable to load certificate and key files for %s: %v", c.Host, err) + } + log.Printf("[INFO] Successfully loaded TLS assets from %s and %s", certificateFile, keyFile) + } + + // load a directory of certificates, if specified + // modeled after haproxy: https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#5.1-crt + if loadDir != "" { + err := filepath.Walk(loadDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + log.Printf("[WARNING] Unable to traverse into %s; skipping", path) + return nil + } + if info.IsDir() { + return nil + } + if strings.HasSuffix(strings.ToLower(info.Name()), ".pem") { + certBuilder, keyBuilder := new(bytes.Buffer), new(bytes.Buffer) + var foundKey bool + + bundle, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + for { + // Decode next block so we can see what type it is + var derBlock *pem.Block + derBlock, bundle = pem.Decode(bundle) + if derBlock == nil { + break + } + + if derBlock.Type == "CERTIFICATE" { + // Re-encode certificate as PEM, appending to certificate chain + pem.Encode(certBuilder, derBlock) + } else if derBlock.Type == "EC PARAMETERS" { + // EC keys are composed of two blocks: parameters and key + // (parameter block should come first) + if !foundKey { + // Encode parameters + pem.Encode(keyBuilder, derBlock) + + // Key must immediately follow + derBlock, bundle = pem.Decode(bundle) + if derBlock == nil || derBlock.Type != "EC PRIVATE KEY" { + return c.Errf("%s: expected elliptic private key to immediately follow EC parameters", path) + } + pem.Encode(keyBuilder, derBlock) + foundKey = true + } + } else if derBlock.Type == "PRIVATE KEY" || strings.HasSuffix(derBlock.Type, " PRIVATE KEY") { + // RSA key + if !foundKey { + pem.Encode(keyBuilder, derBlock) + foundKey = true + } + } else { + return c.Errf("%s: unrecognized PEM block type: %s", path, derBlock.Type) + } + } + + certPEMBytes, keyPEMBytes := certBuilder.Bytes(), keyBuilder.Bytes() + if len(certPEMBytes) == 0 { + return c.Errf("%s: failed to parse PEM data", path) + } + if len(keyPEMBytes) == 0 { + return c.Errf("%s: no private key block found", path) + } + + err = cacheUnmanagedCertificatePEMBytes(certPEMBytes, keyPEMBytes) + if err != nil { + return c.Errf("%s: failed to load cert and key for %s: %v", path, c.Host, err) + } + log.Printf("[INFO] Successfully loaded TLS assets from %s", path) + } + return nil + }) + if err != nil { + return nil, err + } + } } - SetDefaultTLSParams(c.Config) + setDefaultTLSParams(c.Config) return nil, nil } -// SetDefaultTLSParams sets the default TLS cipher suites, protocol versions, +// setDefaultTLSParams sets the default TLS cipher suites, protocol versions, // and server preferences of a server.Config if they were not previously set -// (it does not overwrite; only fills in missing values). -func SetDefaultTLSParams(c *server.Config) { - // If no ciphers provided, use all that Caddy supports for the protocol +// (it does not overwrite; only fills in missing values). It will also set the +// port to 443 if not already set, TLS is enabled, TLS is manual, and the host +// does not equal localhost. +func setDefaultTLSParams(c *server.Config) { + // If no ciphers provided, use default list if len(c.TLS.Ciphers) == 0 { c.TLS.Ciphers = defaultCiphers } @@ -111,14 +216,14 @@ func SetDefaultTLSParams(c *server.Config) { // Default TLS port is 443; only use if port is not manually specified, // TLS is enabled, and the host is not localhost - if c.Port == "" && c.TLS.Enabled && c.Host != "localhost" { + if c.Port == "" && c.TLS.Enabled && !c.TLS.Manual && c.Host != "localhost" { c.Port = "443" } } -// Map of supported protocols -// SSLv3 will be not supported in future release -// HTTP/2 only supports TLS 1.2 and higher +// Map of supported protocols. +// SSLv3 will be not supported in future release. +// HTTP/2 only supports TLS 1.2 and higher. var supportedProtocols = map[string]uint16{ "ssl3.0": tls.VersionSSL30, "tls1.0": tls.VersionTLS10, diff --git a/caddy/setup/tls_test.go b/caddy/https/setup_test.go similarity index 58% rename from caddy/setup/tls_test.go rename to caddy/https/setup_test.go index 727a7996e..4ca57b823 100644 --- a/caddy/setup/tls_test.go +++ b/caddy/https/setup_test.go @@ -1,24 +1,46 @@ -package setup +package https import ( "crypto/tls" + "io/ioutil" + "log" + "os" "testing" + + "github.com/mholt/caddy/caddy/setup" ) -func TestTLSParseBasic(t *testing.T) { - c := NewTestController(`tls cert.pem key.pem`) +func TestMain(m *testing.M) { + // Write test certificates to disk before tests, and clean up + // when we're done. + err := ioutil.WriteFile(certFile, testCert, 0644) + if err != nil { + log.Fatal(err) + } + err = ioutil.WriteFile(keyFile, testKey, 0644) + if err != nil { + os.Remove(certFile) + log.Fatal(err) + } - _, err := TLS(c) + result := m.Run() + + os.Remove(certFile) + os.Remove(keyFile) + os.Exit(result) +} + +func TestSetupParseBasic(t *testing.T) { + c := setup.NewTestController(`tls ` + certFile + ` ` + keyFile + ``) + + _, err := Setup(c) if err != nil { t.Errorf("Expected no errors, got: %v", err) } // Basic checks - if c.TLS.Certificate != "cert.pem" { - t.Errorf("Expected certificate arg to be 'cert.pem', was '%s'", c.TLS.Certificate) - } - if c.TLS.Key != "key.pem" { - t.Errorf("Expected key arg to be 'key.pem', was '%s'", c.TLS.Key) + if !c.TLS.Manual { + t.Error("Expected TLS Manual=true, but was false") } if !c.TLS.Enabled { t.Error("Expected TLS Enabled=true, but was false") @@ -63,23 +85,23 @@ func TestTLSParseBasic(t *testing.T) { } } -func TestTLSParseIncompleteParams(t *testing.T) { +func TestSetupParseIncompleteParams(t *testing.T) { // Using tls without args is an error because it's unnecessary. - c := NewTestController(`tls`) - _, err := TLS(c) + c := setup.NewTestController(`tls`) + _, err := Setup(c) if err == nil { t.Error("Expected an error, but didn't get one") } } -func TestTLSParseWithOptionalParams(t *testing.T) { - params := `tls cert.crt cert.key { +func TestSetupParseWithOptionalParams(t *testing.T) { + params := `tls ` + certFile + ` ` + keyFile + ` { protocols ssl3.0 tls1.2 ciphers RSA-3DES-EDE-CBC-SHA RSA-AES256-CBC-SHA ECDHE-RSA-AES128-GCM-SHA256 }` - c := NewTestController(params) + c := setup.NewTestController(params) - _, err := TLS(c) + _, err := Setup(c) if err != nil { t.Errorf("Expected no errors, got: %v", err) } @@ -97,13 +119,13 @@ func TestTLSParseWithOptionalParams(t *testing.T) { } } -func TestTLSDefaultWithOptionalParams(t *testing.T) { +func TestSetupDefaultWithOptionalParams(t *testing.T) { params := `tls { ciphers RSA-3DES-EDE-CBC-SHA }` - c := NewTestController(params) + c := setup.NewTestController(params) - _, err := TLS(c) + _, err := Setup(c) if err != nil { t.Errorf("Expected no errors, got: %v", err) } @@ -113,7 +135,7 @@ func TestTLSDefaultWithOptionalParams(t *testing.T) { } // TODO: If we allow this... but probably not a good idea. -// func TestTLSDisableHTTPRedirect(t *testing.T) { +// func TestSetupDisableHTTPRedirect(t *testing.T) { // c := NewTestController(`tls { // allow_http // }`) @@ -126,34 +148,34 @@ func TestTLSDefaultWithOptionalParams(t *testing.T) { // } // } -func TestTLSParseWithWrongOptionalParams(t *testing.T) { +func TestSetupParseWithWrongOptionalParams(t *testing.T) { // Test protocols wrong params - params := `tls cert.crt cert.key { + params := `tls ` + certFile + ` ` + keyFile + ` { protocols ssl tls }` - c := NewTestController(params) - _, err := TLS(c) + c := setup.NewTestController(params) + _, err := Setup(c) if err == nil { t.Errorf("Expected errors, but no error returned") } // Test ciphers wrong params - params = `tls cert.crt cert.key { + params = `tls ` + certFile + ` ` + keyFile + ` { ciphers not-valid-cipher }` - c = NewTestController(params) - _, err = TLS(c) + c = setup.NewTestController(params) + _, err = Setup(c) if err == nil { t.Errorf("Expected errors, but no error returned") } } -func TestTLSParseWithClientAuth(t *testing.T) { - params := `tls cert.crt cert.key { +func TestSetupParseWithClientAuth(t *testing.T) { + params := `tls ` + certFile + ` ` + keyFile + ` { clients client_ca.crt client2_ca.crt }` - c := NewTestController(params) - _, err := TLS(c) + c := setup.NewTestController(params) + _, err := Setup(c) if err != nil { t.Errorf("Expected no errors, got: %v", err) } @@ -169,12 +191,40 @@ func TestTLSParseWithClientAuth(t *testing.T) { } // Test missing client cert file - params = `tls cert.crt cert.key { + params = `tls ` + certFile + ` ` + keyFile + ` { clients }` - c = NewTestController(params) - _, err = TLS(c) + c = setup.NewTestController(params) + _, err = Setup(c) if err == nil { t.Errorf("Expected an error, but no error returned") } } + +const ( + certFile = "test_cert.pem" + keyFile = "test_key.pem" +) + +var testCert = []byte(`-----BEGIN CERTIFICATE----- +MIIBkjCCATmgAwIBAgIJANfFCBcABL6LMAkGByqGSM49BAEwFDESMBAGA1UEAxMJ +bG9jYWxob3N0MB4XDTE2MDIxMDIyMjAyNFoXDTE4MDIwOTIyMjAyNFowFDESMBAG +A1UEAxMJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEs22MtnG7 +9K1mvIyjEO9GLx7BFD0tBbGnwQ0VPsuCxC6IeVuXbQDLSiVQvFZ6lUszTlczNxVk +pEfqrM6xAupB7qN1MHMwHQYDVR0OBBYEFHxYDvAxUwL4XrjPev6qZ/BiLDs5MEQG +A1UdIwQ9MDuAFHxYDvAxUwL4XrjPev6qZ/BiLDs5oRikFjAUMRIwEAYDVQQDEwls +b2NhbGhvc3SCCQDXxQgXAAS+izAMBgNVHRMEBTADAQH/MAkGByqGSM49BAEDSAAw +RQIgRvBqbyJM2JCJqhA1FmcoZjeMocmhxQHTt1c+1N2wFUgCIQDtvrivbBPA688N +Qh3sMeAKNKPsx5NxYdoWuu9KWcKz9A== +-----END CERTIFICATE----- +`) + +var testKey = []byte(`-----BEGIN EC PARAMETERS----- +BggqhkjOPQMBBw== +-----END EC PARAMETERS----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIGLtRmwzYVcrH3J0BnzYbGPdWVF10i9p6mxkA4+b2fURoAoGCCqGSM49 +AwEHoUQDQgAEs22MtnG79K1mvIyjEO9GLx7BFD0tBbGnwQ0VPsuCxC6IeVuXbQDL +SiVQvFZ6lUszTlczNxVkpEfqrM6xAupB7g== +-----END EC PRIVATE KEY----- +`) diff --git a/caddy/letsencrypt/storage.go b/caddy/https/storage.go similarity index 99% rename from caddy/letsencrypt/storage.go rename to caddy/https/storage.go index 7a00aa18a..5d487837f 100644 --- a/caddy/letsencrypt/storage.go +++ b/caddy/https/storage.go @@ -1,4 +1,4 @@ -package letsencrypt +package https import ( "path/filepath" diff --git a/caddy/letsencrypt/storage_test.go b/caddy/https/storage_test.go similarity index 99% rename from caddy/letsencrypt/storage_test.go rename to caddy/https/storage_test.go index 545c46b64..85c2220eb 100644 --- a/caddy/letsencrypt/storage_test.go +++ b/caddy/https/storage_test.go @@ -1,4 +1,4 @@ -package letsencrypt +package https import ( "path/filepath" diff --git a/caddy/letsencrypt/user.go b/caddy/https/user.go similarity index 91% rename from caddy/letsencrypt/user.go rename to caddy/https/user.go index 1fac1d71d..c5a742526 100644 --- a/caddy/letsencrypt/user.go +++ b/caddy/https/user.go @@ -1,4 +1,4 @@ -package letsencrypt +package https import ( "bufio" @@ -41,7 +41,7 @@ func (u User) GetPrivateKey() *rsa.PrivateKey { // getUser loads the user with the given email from disk. // If the user does not exist, it will create a new one, // but it does NOT save new users to the disk or register -// them via ACME. +// them via ACME. It does NOT prompt the user. func getUser(email string) (User, error) { var user User @@ -72,7 +72,8 @@ func getUser(email string) (User, error) { } // saveUser persists a user's key and account registration -// to the file system. It does NOT register the user via ACME. +// to the file system. It does NOT register the user via ACME +// or prompt the user. func saveUser(user User) error { // make user account folder err := os.MkdirAll(storage.User(user.Email), 0700) @@ -99,7 +100,7 @@ func saveUser(user User) error { // with a new private key. This function does NOT save the // user to disk or register it via ACME. If you want to use // a user account that might already exist, call getUser -// instead. +// instead. It does NOT prompt the user. func newUser(email string) (User, error) { user := User{Email: email} privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeySizeToUse) @@ -114,10 +115,10 @@ func newUser(email string) (User, error) { // address from the user to use for TLS for cfg. If it // cannot get an email address, it returns empty string. // (It will warn the user of the consequences of an -// empty email.) If skipPrompt is true, the user will -// NOT be prompted and an empty email will be returned -// instead. -func getEmail(cfg server.Config, skipPrompt bool) string { +// empty email.) This function MAY prompt the user for +// input. If userPresent is false, the operator will +// NOT be prompted and an empty email may be returned. +func getEmail(cfg server.Config, userPresent bool) string { // First try the tls directive from the Caddyfile leEmail := cfg.TLS.LetsEncryptEmail if leEmail == "" { @@ -135,11 +136,12 @@ func getEmail(cfg server.Config, skipPrompt bool) string { } if mostRecent == nil || dir.ModTime().After(mostRecent.ModTime()) { leEmail = dir.Name() + DefaultEmail = leEmail // save for next time } } } } - if leEmail == "" && !skipPrompt { + if leEmail == "" && userPresent { // Alas, we must bother the user and ask for an email address; // if they proceed they also agree to the SA. reader := bufio.NewReader(stdin) diff --git a/caddy/letsencrypt/user_test.go b/caddy/https/user_test.go similarity index 96% rename from caddy/letsencrypt/user_test.go rename to caddy/https/user_test.go index 765bd3d4d..5bc28b04c 100644 --- a/caddy/letsencrypt/user_test.go +++ b/caddy/https/user_test.go @@ -1,4 +1,4 @@ -package letsencrypt +package https import ( "bytes" @@ -140,13 +140,13 @@ func TestGetEmail(t *testing.T) { LetsEncryptEmail: "test1@foo.com", }, } - actual := getEmail(config, false) + actual := getEmail(config, true) if actual != "test1@foo.com" { t.Errorf("Did not get correct email from config; expected '%s' but got '%s'", "test1@foo.com", actual) } // Test2: Use default email from flag (or user previously typing it) - actual = getEmail(server.Config{}, false) + actual = getEmail(server.Config{}, true) if actual != DefaultEmail { t.Errorf("Did not get correct email from config; expected '%s' but got '%s'", DefaultEmail, actual) } @@ -158,7 +158,7 @@ func TestGetEmail(t *testing.T) { if err != nil { t.Fatalf("Could not simulate user input, error: %v", err) } - actual = getEmail(server.Config{}, false) + actual = getEmail(server.Config{}, true) if actual != "test3@foo.com" { t.Errorf("Did not get correct email from user input prompt; expected '%s' but got '%s'", "test3@foo.com", actual) } @@ -189,7 +189,7 @@ func TestGetEmail(t *testing.T) { t.Fatalf("Could not change user folder mod time for '%s': %v", eml, err) } } - actual = getEmail(server.Config{}, false) + actual = getEmail(server.Config{}, true) if actual != "test4-3@foo.com" { t.Errorf("Did not get correct email from storage; expected '%s' but got '%s'", "test4-3@foo.com", actual) } diff --git a/caddy/letsencrypt/handshake.go b/caddy/letsencrypt/handshake.go deleted file mode 100644 index 690eb0767..000000000 --- a/caddy/letsencrypt/handshake.go +++ /dev/null @@ -1,99 +0,0 @@ -package letsencrypt - -import ( - "crypto/tls" - "errors" - "strings" - "sync" - - "github.com/mholt/caddy/server" -) - -// GetCertificateDuringHandshake is a function that gets a certificate during a TLS handshake. -// It first checks an in-memory cache in case the cert was requested before, then tries to load -// a certificate in the storage folder from disk. If it can't find an existing certificate, it -// will try to obtain one using ACME, which will then be stored on disk and cached in memory. -// -// This function is safe for use by multiple concurrent goroutines. -func GetCertificateDuringHandshake(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { - // Utility function to help us load a cert from disk and put it in the cache if successful - loadCertFromDisk := func(domain string) *tls.Certificate { - cert, err := tls.LoadX509KeyPair(storage.SiteCertFile(domain), storage.SiteKeyFile(domain)) - if err == nil { - certCacheMu.Lock() - if len(certCache) < 10000 { // limit size of cache to prevent a ridiculous, unusual kind of attack - certCache[domain] = &cert - } - certCacheMu.Unlock() - return &cert - } - return nil - } - - // First check our in-memory cache to see if we've already loaded it - certCacheMu.RLock() - cert := server.GetCertificateFromCache(clientHello, certCache) - certCacheMu.RUnlock() - if cert != nil { - return cert, nil - } - - // Then check to see if we already have one on disk; if we do, add it to cache and use it - name := strings.ToLower(clientHello.ServerName) - cert = loadCertFromDisk(name) - if cert != nil { - return cert, nil - } - - // Only option left is to get one from LE, but the name has to qualify first - if !HostQualifies(name) { - return nil, nil - } - - // By this point, we need to obtain one from the CA. We must protect this process - // from happening concurrently, so synchronize. - obtainCertWaitGroupsMutex.Lock() - wg, ok := obtainCertWaitGroups[name] - if ok { - // lucky us -- another goroutine is already obtaining the certificate. - // wait for it to finish obtaining the cert and then we'll use it. - obtainCertWaitGroupsMutex.Unlock() - wg.Wait() - return GetCertificateDuringHandshake(clientHello) - } - - // looks like it's up to us to do all the work and obtain the cert - wg = new(sync.WaitGroup) - wg.Add(1) - obtainCertWaitGroups[name] = wg - obtainCertWaitGroupsMutex.Unlock() - - // Unblock waiters and delete waitgroup when we return - defer func() { - obtainCertWaitGroupsMutex.Lock() - wg.Done() - delete(obtainCertWaitGroups, name) - obtainCertWaitGroupsMutex.Unlock() - }() - - // obtain cert - client, err := newClientPort(DefaultEmail, AlternatePort) - if err != nil { - return nil, errors.New("error creating client: " + err.Error()) - } - err = clientObtain(client, []string{name}, false) - if err != nil { - return nil, err - } - - // load certificate into memory and return it - return loadCertFromDisk(name), nil -} - -// obtainCertWaitGroups is used to coordinate obtaining certs for each hostname. -var obtainCertWaitGroups = make(map[string]*sync.WaitGroup) -var obtainCertWaitGroupsMutex sync.Mutex - -// certCache stores certificates that have been obtained in memory. -var certCache = make(map[string]*tls.Certificate) -var certCacheMu sync.RWMutex diff --git a/caddy/letsencrypt/maintain.go b/caddy/letsencrypt/maintain.go deleted file mode 100644 index 5a59dc23a..000000000 --- a/caddy/letsencrypt/maintain.go +++ /dev/null @@ -1,180 +0,0 @@ -package letsencrypt - -import ( - "encoding/json" - "io/ioutil" - "log" - "time" - - "github.com/mholt/caddy/server" - "github.com/xenolf/lego/acme" -) - -// OnChange is a callback function that will be used to restart -// the application or the part of the application that uses -// the certificates maintained by this package. When at least -// one certificate is renewed or an OCSP status changes, this -// function will be called. -var OnChange func() error - -// maintainAssets is a permanently-blocking function -// that loops indefinitely and, on a regular schedule, checks -// certificates for expiration and initiates a renewal of certs -// that are expiring soon. It also updates OCSP stapling and -// performs other maintenance of assets. -// -// You must pass in the server configs to maintain and the channel -// which you'll close when maintenance should stop, to allow this -// goroutine to clean up after itself and unblock. -func maintainAssets(configs []server.Config, stopChan chan struct{}) { - renewalTicker := time.NewTicker(RenewInterval) - ocspTicker := time.NewTicker(OCSPInterval) - - for { - select { - case <-renewalTicker.C: - n, errs := renewCertificates(configs, true) - if len(errs) > 0 { - for _, err := range errs { - log.Printf("[ERROR] Certificate renewal: %v", err) - } - } - // even if there was an error, some renewals may have succeeded - if n > 0 && OnChange != nil { - err := OnChange() - if err != nil { - log.Printf("[ERROR] OnChange after cert renewal: %v", err) - } - } - case <-ocspTicker.C: - for bundle, oldResp := range ocspCache { - // start checking OCSP staple about halfway through validity period for good measure - refreshTime := oldResp.ThisUpdate.Add(oldResp.NextUpdate.Sub(oldResp.ThisUpdate) / 2) - - // only check for updated OCSP validity window if refreshTime is in the past - if time.Now().After(refreshTime) { - _, newResp, err := acme.GetOCSPForCert(*bundle) - if err != nil { - log.Printf("[ERROR] Checking OCSP for bundle: %v", err) - continue - } - - // we're not looking for different status, just a more future expiration - if newResp.NextUpdate != oldResp.NextUpdate { - if OnChange != nil { - log.Printf("[INFO] Updating OCSP stapling to extend validity period to %v", newResp.NextUpdate) - err := OnChange() - if err != nil { - log.Printf("[ERROR] OnChange after OCSP trigger: %v", err) - } - break - } - } - } - } - case <-stopChan: - renewalTicker.Stop() - ocspTicker.Stop() - return - } - } -} - -// renewCertificates loops through all configured site and -// looks for certificates to renew. Nothing is mutated -// through this function; all changes happen directly on disk. -// It returns the number of certificates renewed and any errors -// that occurred. It only performs a renewal if necessary. -// If useCustomPort is true, a custom port will be used, and -// whatever is listening at 443 better proxy ACME requests to it. -// Otherwise, the acme package will create its own listener on 443. -func renewCertificates(configs []server.Config, useCustomPort bool) (int, []error) { - log.Printf("[INFO] Checking certificates for %d hosts", len(configs)) - var errs []error - var n int - - for _, cfg := range configs { - // Host must be TLS-enabled and have existing assets managed by LE - if !cfg.TLS.Enabled || !existingCertAndKey(cfg.Host) { - continue - } - - // Read the certificate and get the NotAfter time. - certBytes, err := ioutil.ReadFile(storage.SiteCertFile(cfg.Host)) - if err != nil { - errs = append(errs, err) - continue // still have to check other certificates - } - expTime, err := acme.GetPEMCertExpiration(certBytes) - if err != nil { - errs = append(errs, err) - continue - } - - // The time returned from the certificate is always in UTC. - // So calculate the time left with local time as UTC. - // Directly convert it to days for the following checks. - daysLeft := int(expTime.Sub(time.Now().UTC()).Hours() / 24) - - // Renew if getting close to expiration. - if daysLeft <= renewDaysBefore { - log.Printf("[INFO] Certificate for %s has %d days remaining; attempting renewal", cfg.Host, daysLeft) - var client *acme.Client - if useCustomPort { - client, err = newClientPort("", AlternatePort) // email not used for renewal - } else { - client, err = newClient("") - } - if err != nil { - errs = append(errs, err) - continue - } - - // Read and set up cert meta, required for renewal - metaBytes, err := ioutil.ReadFile(storage.SiteMetaFile(cfg.Host)) - if err != nil { - errs = append(errs, err) - continue - } - privBytes, err := ioutil.ReadFile(storage.SiteKeyFile(cfg.Host)) - if err != nil { - errs = append(errs, err) - continue - } - var certMeta acme.CertificateResource - err = json.Unmarshal(metaBytes, &certMeta) - certMeta.Certificate = certBytes - certMeta.PrivateKey = privBytes - - // Renew certificate - Renew: - newCertMeta, err := client.RenewCertificate(certMeta, true) - if err != nil { - if _, ok := err.(acme.TOSError); ok { - err := client.AgreeToTOS() - if err != nil { - errs = append(errs, err) - } - goto Renew - } - - time.Sleep(10 * time.Second) - newCertMeta, err = client.RenewCertificate(certMeta, true) - if err != nil { - errs = append(errs, err) - continue - } - } - - saveCertResource(newCertMeta) - n++ - } else if daysLeft <= renewDaysBefore+7 && daysLeft >= renewDaysBefore+6 { - log.Printf("[WARNING] Certificate for %s has %d days remaining; will automatically renew when %d days remain\n", cfg.Host, daysLeft, renewDaysBefore) - } - } - - return n, errs -} - -// renewDaysBefore is how many days before expiration to renew certificates. -const renewDaysBefore = 14 diff --git a/caddy/restart.go b/caddy/restart.go index cc16568f7..c8dc8c7e2 100644 --- a/caddy/restart.go +++ b/caddy/restart.go @@ -12,7 +12,7 @@ import ( "os/exec" "path" - "github.com/mholt/caddy/caddy/letsencrypt" + "github.com/mholt/caddy/caddy/https" ) func init() { @@ -133,13 +133,15 @@ func getCertsForNewCaddyfile(newCaddyfile Input) error { } // first mark the configs that are qualified for managed TLS - letsencrypt.MarkQualified(configs) + https.MarkQualified(configs) - // we must make sure port is set before we group by bind address - letsencrypt.EnableTLS(configs) + // since we group by bind address to obtain certs, we must call + // EnableTLS to make sure the port is set properly first + // (can ignore error since we aren't actually using the certs) + https.EnableTLS(configs, false) // place certs on the disk - err = letsencrypt.ObtainCerts(configs, letsencrypt.AlternatePort) + err = https.ObtainCerts(configs, false) if err != nil { return errors.New("obtaining certs: " + err.Error()) } diff --git a/main.go b/main.go index 813423018..d83ef09ce 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,7 @@ import ( "time" "github.com/mholt/caddy/caddy" - "github.com/mholt/caddy/caddy/letsencrypt" + "github.com/mholt/caddy/caddy/https" "github.com/xenolf/lego/acme" ) @@ -32,14 +32,14 @@ const ( func init() { caddy.TrapSignals() - flag.BoolVar(&letsencrypt.Agreed, "agree", false, "Agree to Let's Encrypt Subscriber Agreement") - flag.StringVar(&letsencrypt.CAUrl, "ca", "https://acme-v01.api.letsencrypt.org/directory", "Certificate authority ACME server") + flag.BoolVar(&https.Agreed, "agree", false, "Agree to Let's Encrypt Subscriber Agreement") + flag.StringVar(&https.CAUrl, "ca", "https://acme-v01.api.letsencrypt.org/directory", "Certificate authority ACME server") flag.StringVar(&conf, "conf", "", "Configuration file to use (default="+caddy.DefaultConfigFile+")") flag.StringVar(&cpu, "cpu", "100%", "CPU cap") - flag.StringVar(&letsencrypt.DefaultEmail, "email", "", "Default Let's Encrypt account email address") + flag.StringVar(&https.DefaultEmail, "email", "", "Default Let's Encrypt account email address") flag.DurationVar(&caddy.GracefulTimeout, "grace", 5*time.Second, "Maximum duration of graceful shutdown") flag.StringVar(&caddy.Host, "host", caddy.DefaultHost, "Default host") - flag.BoolVar(&caddy.HTTP2, "http2", true, "HTTP/2 support") // TODO: temporary flag until http2 merged into std lib + flag.BoolVar(&caddy.HTTP2, "http2", true, "HTTP/2 support") flag.StringVar(&logfile, "log", "", "Process log file") flag.StringVar(&caddy.PidFile, "pidfile", "", "Path to write pid file") flag.StringVar(&caddy.Port, "port", caddy.DefaultPort, "Default port") @@ -73,7 +73,7 @@ func main() { } if revoke != "" { - err := letsencrypt.Revoke(revoke) + err := https.Revoke(revoke) if err != nil { log.Fatal(err) } diff --git a/server/config.go b/server/config.go index 11d69e142..cae1edf56 100644 --- a/server/config.go +++ b/server/config.go @@ -65,13 +65,10 @@ func (c Config) Address() string { // TLSConfig describes how TLS should be configured and used. type TLSConfig struct { - Enabled bool - Certificate string - Key string - LetsEncryptEmail string - Managed bool // will be set to true if config qualifies for automatic, managed TLS - //DisableHTTPRedir bool // TODO: not a good idea - should we really allow it? - OCSPStaple []byte + Enabled bool + LetsEncryptEmail string + Managed bool // will be set to true if config qualifies for automatic, managed TLS + Manual bool // will be set to true if user provides the cert and key files Ciphers []uint16 ProtocolMinVersion uint16 ProtocolMaxVersion uint16 diff --git a/server/server.go b/server/server.go index 293092c6e..a0235979f 100644 --- a/server/server.go +++ b/server/server.go @@ -13,7 +13,6 @@ import ( "net/http" "os" "runtime" - "strings" "sync" "time" ) @@ -25,8 +24,9 @@ import ( // graceful termination (POSIX only). type Server struct { *http.Server - HTTP2 bool // temporary while http2 is not in std lib (TODO: remove flag when part of std lib) + HTTP2 bool // whether to enable HTTP/2 tls bool // whether this server is serving all HTTPS hosts or not + OnDemandTLS bool // whether this server supports on-demand TLS (load certs at handshake-time) vhosts map[string]virtualHost // virtual hosts keyed by their address listener ListenerFile // the listener which is bound to the socket listenerMu sync.Mutex // protects listener @@ -60,20 +60,29 @@ type OptionalCallback func(http.ResponseWriter, *http.Request) bool // as it stands, you should dispose of a server after stopping it. // The behavior of serving with a spent server is undefined. func New(addr string, configs []Config, gracefulTimeout time.Duration) (*Server, error) { - var tls bool + var useTLS, useOnDemandTLS bool if len(configs) > 0 { - tls = configs[0].TLS.Enabled + useTLS = configs[0].TLS.Enabled + host, _, err := net.SplitHostPort(addr) + if err != nil { + host = addr + } + if useTLS && host == "" && !configs[0].TLS.Manual { + useOnDemandTLS = true + } } s := &Server{ Server: &http.Server{ - Addr: addr, + Addr: addr, + TLSConfig: new(tls.Config), // TODO: Make these values configurable? // ReadTimeout: 2 * time.Minute, // WriteTimeout: 2 * time.Minute, // MaxHeaderBytes: 1 << 16, }, - tls: tls, + tls: useTLS, + OnDemandTLS: useOnDemandTLS, vhosts: make(map[string]virtualHost), startChan: make(chan struct{}), connTimeout: gracefulTimeout, @@ -168,7 +177,7 @@ func (s *Server) serve(ln ListenerFile) error { for _, vh := range s.vhosts { tlsConfigs = append(tlsConfigs, vh.config.TLS) } - return serveTLSWithSNI(s, s.listener, tlsConfigs) + return serveTLS(s, s.listener, tlsConfigs) } close(s.startChan) // unblock anyone waiting for this to start listening @@ -196,106 +205,32 @@ func (s *Server) setup() error { return nil } -// serveTLSWithSNI serves TLS with Server Name Indication (SNI) support, which allows -// multiple sites (different hostnames) to be served from the same address. It also -// supports client authentication if srv has it enabled. It blocks until s quits. -// -// This method is adapted from the std lib's net/http ServeTLS function, which was written -// by the Go Authors. It has been modified to support multiple certificate/key pairs, -// client authentication, and our custom Server type. -func serveTLSWithSNI(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error { - config := cloneTLSConfig(s.TLSConfig) - - // Here we diverge from the stdlib a bit by loading multiple certs/key pairs - // then we map the server names to their certs - for _, tlsConfig := range tlsConfigs { - if tlsConfig.Certificate == "" || tlsConfig.Key == "" { - continue - } - cert, err := tls.LoadX509KeyPair(tlsConfig.Certificate, tlsConfig.Key) - if err != nil { - defer close(s.startChan) - return fmt.Errorf("loading certificate and key pair: %v", err) - } - cert.OCSPStaple = tlsConfig.OCSPStaple - config.Certificates = append(config.Certificates, cert) - } - if len(config.Certificates) > 0 { - config.BuildNameToCertificate() - } - - config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { - // TODO: When Caddy starts, if it is to issue certs dynamically, we need - // terms agreement and an email address. make sure this is enforced at server - // start if the Caddyfile enables dynamic certificate issuance! - - // Check NameToCertificate like the std lib does in "getCertificate" (unexported, bah) - cert := GetCertificateFromCache(clientHello, config.NameToCertificate) - if cert != nil { - return cert, nil - } - - if s.SNICallback != nil { - return s.SNICallback(clientHello) - } - - return nil, nil - } - +// serveTLS serves TLS with SNI and client auth support if s has them enabled. It +// blocks until s quits. +func serveTLS(s *Server, ln net.Listener, tlsConfigs []TLSConfig) error { // Customize our TLS configuration - config.MinVersion = tlsConfigs[0].ProtocolMinVersion - config.MaxVersion = tlsConfigs[0].ProtocolMaxVersion - config.CipherSuites = tlsConfigs[0].Ciphers - config.PreferServerCipherSuites = tlsConfigs[0].PreferServerCipherSuites + s.TLSConfig.MinVersion = tlsConfigs[0].ProtocolMinVersion + s.TLSConfig.MaxVersion = tlsConfigs[0].ProtocolMaxVersion + s.TLSConfig.CipherSuites = tlsConfigs[0].Ciphers + s.TLSConfig.PreferServerCipherSuites = tlsConfigs[0].PreferServerCipherSuites // TLS client authentication, if user enabled it - err := setupClientAuth(tlsConfigs, config) + err := setupClientAuth(tlsConfigs, s.TLSConfig) if err != nil { defer close(s.startChan) return err } - s.TLSConfig = config // Create TLS listener - note that we do not replace s.listener // with this TLS listener; tls.listener is unexported and does // not implement the File() method we need for graceful restarts // on POSIX systems. - ln = tls.NewListener(ln, config) + ln = tls.NewListener(ln, s.TLSConfig) close(s.startChan) // unblock anyone waiting for this to start listening return s.Server.Serve(ln) } -// Borrowed from the Go standard library, crypto/tls pacakge, common.go. -// It has been modified to fit this program. -// Original license: -// -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. -func GetCertificateFromCache(clientHello *tls.ClientHelloInfo, cache map[string]*tls.Certificate) *tls.Certificate { - name := strings.ToLower(clientHello.ServerName) - for len(name) > 0 && name[len(name)-1] == '.' { - name = name[:len(name)-1] - } - - // exact match? great! use it - if cert, ok := cache[name]; ok { - return cert - } - - // try replacing labels in the name with wildcards until we get a match. - labels := strings.Split(name, ".") - for i := range labels { - labels[i] = "*" - candidate := strings.Join(labels, ".") - if cert, ok := cache[candidate]; ok { - return cert - } - } - return nil -} - // Stop stops the server. It blocks until the server is // totally stopped. On POSIX systems, it will wait for // connections to close (up to a max timeout of a few @@ -482,6 +417,8 @@ func (ln tcpKeepAliveListener) File() (*os.File, error) { } // copied from net/http/transport.go +/* + TODO - remove - not necessary? func cloneTLSConfig(cfg *tls.Config) *tls.Config { if cfg == nil { return &tls.Config{} @@ -507,7 +444,7 @@ func cloneTLSConfig(cfg *tls.Config) *tls.Config { MaxVersion: cfg.MaxVersion, CurvePreferences: cfg.CurvePreferences, } -} +}*/ // ShutdownCallbacks executes all the shutdown callbacks // for all the virtualhosts in servers, and returns all the From 216a6172492c4ee5ebdd871558bcb220e83deb9b Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 11 Feb 2016 13:48:52 -0700 Subject: [PATCH 18/52] tls: Some bug fixes, basic rate limiting, max_certs setting --- caddy/caddy.go | 2 + caddy/helpers.go | 8 ++- caddy/https/handler.go | 12 +--- caddy/https/handshake.go | 102 ++++++++++++++++++++++++---------- caddy/https/setup.go | 19 ++++++- caddy/restart.go | 6 +- caddy/setup/basicauth_test.go | 2 +- 7 files changed, 105 insertions(+), 46 deletions(-) diff --git a/caddy/caddy.go b/caddy/caddy.go index 5d8ceddd8..da6975496 100644 --- a/caddy/caddy.go +++ b/caddy/caddy.go @@ -26,6 +26,7 @@ import ( "path" "strings" "sync" + "sync/atomic" "time" "github.com/mholt/caddy/caddy/https" @@ -317,6 +318,7 @@ func LoadCaddyfile(loader func() (Input, error)) (cdyfile Input, err error) { return nil, err } cdyfile = loadedGob.Caddyfile + atomic.StoreInt32(https.OnDemandIssuedCount, loadedGob.OnDemandTLSCertsIssued) } // Try user's loader diff --git a/caddy/helpers.go b/caddy/helpers.go index 0165573ac..0a2299dfc 100644 --- a/caddy/helpers.go +++ b/caddy/helpers.go @@ -63,10 +63,12 @@ var signalParentOnce sync.Once // caddyfileGob maps bind address to index of the file descriptor // in the Files array passed to the child process. It also contains -// the caddyfile contents. Used only during graceful restarts. +// the caddyfile contents and other state needed by the new process. +// Used only during graceful restarts where a new process is spawned. type caddyfileGob struct { - ListenerFds map[string]uintptr - Caddyfile Input + ListenerFds map[string]uintptr + Caddyfile Input + OnDemandTLSCertsIssued int32 } // IsRestart returns whether this process is, according diff --git a/caddy/https/handler.go b/caddy/https/handler.go index 446539296..f3139f54e 100644 --- a/caddy/https/handler.go +++ b/caddy/https/handler.go @@ -3,7 +3,6 @@ package https import ( "crypto/tls" "log" - "net" "net/http" "net/http/httputil" "net/url" @@ -23,21 +22,16 @@ func RequestCallback(w http.ResponseWriter, r *http.Request) bool { scheme = "https" } - hostname, _, err := net.SplitHostPort(r.Host) - if err != nil { - hostname = r.Host - } - - upstream, err := url.Parse(scheme + "://" + hostname + ":" + AlternatePort) + upstream, err := url.Parse(scheme + "://localhost:" + AlternatePort) if err != nil { w.WriteHeader(http.StatusInternalServerError) - log.Printf("[ERROR] letsencrypt handler: %v", err) + log.Printf("[ERROR] ACME proxy handler: %v", err) return true } proxy := httputil.NewSingleHostReverseProxy(upstream) proxy.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // client would use self-signed cert + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // solver uses self-signed certs } proxy.ServeHTTP(w, r) diff --git a/caddy/https/handshake.go b/caddy/https/handshake.go index e06e7d0da..e535cf6f4 100644 --- a/caddy/https/handshake.go +++ b/caddy/https/handshake.go @@ -7,7 +7,9 @@ import ( "errors" "fmt" "log" + "strings" "sync" + "sync/atomic" "time" "github.com/mholt/caddy/server" @@ -15,11 +17,12 @@ import ( ) // GetCertificate gets a certificate to satisfy clientHello as long as -// the certificate is already cached in memory. +// the certificate is already cached in memory. It will not be loaded +// from disk or obtained from the CA during the handshake. // // This function is safe for use as a tls.Config.GetCertificate callback. func GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { - cert, err := getCertDuringHandshake(clientHello.ServerName, false) + cert, err := getCertDuringHandshake(clientHello.ServerName, false, false) return cert.Certificate, err } @@ -31,45 +34,60 @@ func GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) // // This function is safe for use as a tls.Config.GetCertificate callback. func GetOrObtainCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { - cert, err := getCertDuringHandshake(clientHello.ServerName, true) + cert, err := getCertDuringHandshake(clientHello.ServerName, true, true) return cert.Certificate, err } // getCertDuringHandshake will get a certificate for name. It first tries -// the in-memory cache, then, if obtainIfNecessary is true, it goes to disk, -// then asks the CA for a certificate if necessary. +// the in-memory cache. If no certificate for name is in the cach and if +// loadIfNecessary == true, it goes to disk to load it into the cache and +// serve it. If it's not on disk and if obtainIfNecessary == true, the +// certificate will be obtained from the CA, cached, and served. If +// obtainIfNecessary is true, then loadIfNecessary must also be set to true. // // This function is safe for concurrent use. -func getCertDuringHandshake(name string, obtainIfNecessary bool) (Certificate, error) { +func getCertDuringHandshake(name string, loadIfNecessary, obtainIfNecessary bool) (Certificate, error) { // First check our in-memory cache to see if we've already loaded it cert, ok := getCertificate(name) if ok { return cert, nil } - if obtainIfNecessary { - // TODO: Mitigate abuse! + if loadIfNecessary { var err error // Then check to see if we have one on disk - cert, err := cacheManagedCertificate(name, true) - if err != nil { - return cert, err - } else if cert.Certificate != nil { - cert, err := handshakeMaintenance(name, cert) + cert, err = cacheManagedCertificate(name, true) + if err == nil { + cert, err = handshakeMaintenance(name, cert) if err != nil { log.Printf("[ERROR] Maintaining newly-loaded certificate for %s: %v", name, err) } - return cert, err + return cert, nil } - // Only option left is to get one from LE, but the name has to qualify first - if !HostQualifies(name) { - return cert, errors.New("hostname '" + name + "' does not qualify for certificate") - } + if obtainIfNecessary { + name = strings.ToLower(name) - // By this point, we need to obtain one from the CA. - return obtainOnDemandCertificate(name) + // Make sure aren't over any applicable limits + if onDemandMaxIssue > 0 && atomic.LoadInt32(OnDemandIssuedCount) >= onDemandMaxIssue { + return Certificate{}, fmt.Errorf("%s: maximum certificates issued (%d)", name, onDemandMaxIssue) + } + failedIssuanceMu.RLock() + when, ok := failedIssuance[name] + failedIssuanceMu.RUnlock() + if ok { + return Certificate{}, fmt.Errorf("%s: throttled; refusing to issue cert since last attempt on %s failed", name, when.String()) + } + + // Only option left is to get one from LE, but the name has to qualify first + if !HostQualifies(name) { + return cert, errors.New("hostname '" + name + "' does not qualify for certificate") + } + + // By this point, we need to obtain one from the CA. + return obtainOnDemandCertificate(name) + } } return Certificate{}, nil @@ -89,7 +107,7 @@ func obtainOnDemandCertificate(name string) (Certificate, error) { // wait for it to finish obtaining the cert and then we'll use it. obtainCertWaitChansMu.Unlock() <-wait - return getCertDuringHandshake(name, false) // passing in true might result in infinite loop if obtain failed + return getCertDuringHandshake(name, true, false) } // looks like it's up to us to do all the work and obtain the cert @@ -115,11 +133,24 @@ func obtainOnDemandCertificate(name string) (Certificate, error) { client.Configure("") // TODO: which BindHost? err = client.Obtain([]string{name}) if err != nil { + // Failed to solve challenge, so don't allow another on-demand + // issue for this name to be attempted for a little while. + failedIssuanceMu.Lock() + failedIssuance[name] = time.Now() + go func(name string) { + time.Sleep(5 * time.Minute) + failedIssuanceMu.Lock() + delete(failedIssuance, name) + failedIssuanceMu.Unlock() + }(name) + failedIssuanceMu.Unlock() return Certificate{}, err } + atomic.AddInt32(OnDemandIssuedCount, 1) + // The certificate is on disk; now just start over to load it and serve it - return getCertDuringHandshake(name, false) // pass in false as a fail-safe from infinite-looping + return getCertDuringHandshake(name, true, false) } // handshakeMaintenance performs a check on cert for expiration and OCSP @@ -127,12 +158,6 @@ func obtainOnDemandCertificate(name string) (Certificate, error) { // // This function is safe for use by multiple concurrent goroutines. func handshakeMaintenance(name string, cert Certificate) (Certificate, error) { - // fmt.Println("ON-DEMAND CERT?", cert.OnDemand) - // if !cert.OnDemand { - // return cert, nil - // } - fmt.Println("Checking expiration of cert; on-demand:", cert.OnDemand) - // Check cert expiration timeLeft := cert.NotAfter.Sub(time.Now().UTC()) if timeLeft < renewDurationBefore { @@ -173,7 +198,7 @@ func renewDynamicCertificate(name string) (Certificate, error) { // wait for it to finish, then we'll use the new one. obtainCertWaitChansMu.Unlock() <-wait - return getCertDuringHandshake(name, false) + return getCertDuringHandshake(name, true, false) } // looks like it's up to us to do all the work and renew the cert @@ -201,7 +226,7 @@ func renewDynamicCertificate(name string) (Certificate, error) { return Certificate{}, err } - return getCertDuringHandshake(name, false) + return getCertDuringHandshake(name, true, false) } // stapleOCSP staples OCSP information to cert for hostname name. @@ -235,3 +260,20 @@ func stapleOCSP(cert *Certificate, pemBundle []byte) error { // obtainCertWaitChans is used to coordinate obtaining certs for each hostname. var obtainCertWaitChans = make(map[string]chan struct{}) var obtainCertWaitChansMu sync.Mutex + +// OnDemandIssuedCount is the number of certificates that have been issued +// on-demand by this process. It is only safe to modify this count atomically. +// If it reaches max_certs, on-demand issuances will fail. +var OnDemandIssuedCount = new(int32) + +// onDemandMaxIssue is set based on max_certs in tls config. It specifies the +// maximum number of certificates that can be issued. +// TODO: This applies globally, but we should probably make a server-specific +// way to keep track of these limits and counts... +var onDemandMaxIssue int32 + +// failedIssuance is a set of names that we recently failed to get a +// certificate for from the ACME CA. They are removed after some time. +// When a name is in this map, do not issue a certificate for it. +var failedIssuance = make(map[string]time.Time) +var failedIssuanceMu sync.RWMutex diff --git a/caddy/https/setup.go b/caddy/https/setup.go index 592dfee59..2500bce07 100644 --- a/caddy/https/setup.go +++ b/caddy/https/setup.go @@ -8,6 +8,7 @@ import ( "log" "os" "path/filepath" + "strconv" "strings" "github.com/mholt/caddy/caddy/setup" @@ -27,7 +28,7 @@ func Setup(c *setup.Controller) (middleware.Middleware, error) { } for c.Next() { - var certificateFile, keyFile, loadDir string + var certificateFile, keyFile, loadDir, maxCerts string args := c.RemainingArgs() switch len(args) { @@ -80,6 +81,8 @@ func Setup(c *setup.Controller) (middleware.Middleware, error) { case "load": c.Args(&loadDir) c.TLS.Manual = true + case "max_certs": + c.Args(&maxCerts) default: return nil, c.Errf("Unknown keyword '%s'", c.Val()) } @@ -90,6 +93,20 @@ func Setup(c *setup.Controller) (middleware.Middleware, error) { return nil, c.ArgErr() } + if c.TLS.Manual && maxCerts != "" { + return nil, c.Err("Cannot limit certificate count (max_certs) for manual TLS configurations") + } + + if maxCerts != "" { + maxCertsNum, err := strconv.Atoi(maxCerts) + if err != nil || maxCertsNum < 0 { + return nil, c.Err("max_certs must be a positive integer") + } + if onDemandMaxIssue == 0 || int32(maxCertsNum) < onDemandMaxIssue { // keep the minimum; TODO: This is global; should be per-server or per-vhost... + onDemandMaxIssue = int32(maxCertsNum) + } + } + // don't load certificates unless we're supposed to if !c.TLS.Enabled || !c.TLS.Manual { continue diff --git a/caddy/restart.go b/caddy/restart.go index c8dc8c7e2..255f9cd7c 100644 --- a/caddy/restart.go +++ b/caddy/restart.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path" + "sync/atomic" "github.com/mholt/caddy/caddy/https" ) @@ -55,8 +56,9 @@ func Restart(newCaddyfile Input) error { // Prepare our payload to the child process cdyfileGob := caddyfileGob{ - ListenerFds: make(map[string]uintptr), - Caddyfile: newCaddyfile, + ListenerFds: make(map[string]uintptr), + Caddyfile: newCaddyfile, + OnDemandTLSCertsIssued: atomic.LoadInt32(https.OnDemandIssuedCount), } // Prepare a pipe to the fork's stdin so it can get the Caddyfile diff --git a/caddy/setup/basicauth_test.go b/caddy/setup/basicauth_test.go index a94d6e695..186a3e97e 100644 --- a/caddy/setup/basicauth_test.go +++ b/caddy/setup/basicauth_test.go @@ -118,7 +118,7 @@ md5:$apr1$l42y8rex$pOA2VJ0x/0TwaFeAF9nX61` } if !actualRule.Password(pwd) || actualRule.Password(test.password+"!") { t.Errorf("Test %d, rule %d: Expected password '%v', got '%v'", - i, j, test.password, actualRule.Password) + i, j, test.password, actualRule.Password("")) } expectedRes := fmt.Sprintf("%v", expectedRule.Resources) From 1fe39e4633105b512d3d7e90fc853cd8eeadf412 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 11 Feb 2016 14:27:57 -0700 Subject: [PATCH 19/52] Additional mitigation for on-demand TLS After 10 certificates are issued, no new certificate requests are allowed for 10 minutes after a successful issuance. --- caddy/https/handshake.go | 64 ++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/caddy/https/handshake.go b/caddy/https/handshake.go index e535cf6f4..3d759b31b 100644 --- a/caddy/https/handshake.go +++ b/caddy/https/handshake.go @@ -67,25 +67,22 @@ func getCertDuringHandshake(name string, loadIfNecessary, obtainIfNecessary bool } if obtainIfNecessary { + // By this point, we need to ask the CA for a certificate + name = strings.ToLower(name) // Make sure aren't over any applicable limits - if onDemandMaxIssue > 0 && atomic.LoadInt32(OnDemandIssuedCount) >= onDemandMaxIssue { - return Certificate{}, fmt.Errorf("%s: maximum certificates issued (%d)", name, onDemandMaxIssue) - } - failedIssuanceMu.RLock() - when, ok := failedIssuance[name] - failedIssuanceMu.RUnlock() - if ok { - return Certificate{}, fmt.Errorf("%s: throttled; refusing to issue cert since last attempt on %s failed", name, when.String()) + err := checkLimitsForObtainingNewCerts(name) + if err != nil { + return Certificate{}, err } - // Only option left is to get one from LE, but the name has to qualify first + // Name has to qualify for a certificate if !HostQualifies(name) { return cert, errors.New("hostname '" + name + "' does not qualify for certificate") } - // By this point, we need to obtain one from the CA. + // Obtain certificate from the CA return obtainOnDemandCertificate(name) } } @@ -93,6 +90,37 @@ func getCertDuringHandshake(name string, loadIfNecessary, obtainIfNecessary bool return Certificate{}, nil } +// checkLimitsForObtainingNewCerts checks to see if name can be issued right +// now according to mitigating factors we keep track of and preferences the +// user has set. If a non-nil error is returned, do not issue a new certificate +// for name. +func checkLimitsForObtainingNewCerts(name string) error { + // User can set hard limit for number of certs for the process to issue + if onDemandMaxIssue > 0 && atomic.LoadInt32(OnDemandIssuedCount) >= onDemandMaxIssue { + return fmt.Errorf("%s: maximum certificates issued (%d)", name, onDemandMaxIssue) + } + + // Make sure name hasn't failed a challenge recently + failedIssuanceMu.RLock() + when, ok := failedIssuance[name] + failedIssuanceMu.RUnlock() + if ok { + return fmt.Errorf("%s: throttled; refusing to issue cert since last attempt on %s failed", name, when.String()) + } + + // Make sure, if we've issued a few certificates already, that we haven't + // issued any recently + lastIssueTimeMu.Lock() + since := time.Since(lastIssueTime) + lastIssueTimeMu.Unlock() + if atomic.LoadInt32(OnDemandIssuedCount) >= 10 && since < 10*time.Minute { + return fmt.Errorf("%s: throttled; last certificate was obtained %v ago", name, since) + } + + // 👍Good to go + return nil +} + // obtainOnDemandCertificate obtains a certificate for name for the given // clientHello. If another goroutine has already started obtaining a cert // for name, it will wait and use what the other goroutine obtained. @@ -147,9 +175,13 @@ func obtainOnDemandCertificate(name string) (Certificate, error) { return Certificate{}, err } + // Success - update counters and stuff atomic.AddInt32(OnDemandIssuedCount, 1) + lastIssueTimeMu.Lock() + lastIssueTime = time.Now() + lastIssueTimeMu.Unlock() - // The certificate is on disk; now just start over to load it and serve it + // The certificate is already on disk; now just start over to load it and serve it return getCertDuringHandshake(name, true, false) } @@ -269,11 +301,17 @@ var OnDemandIssuedCount = new(int32) // onDemandMaxIssue is set based on max_certs in tls config. It specifies the // maximum number of certificates that can be issued. // TODO: This applies globally, but we should probably make a server-specific -// way to keep track of these limits and counts... +// way to keep track of these limits and counts, since it's specified in the +// Caddyfile... var onDemandMaxIssue int32 // failedIssuance is a set of names that we recently failed to get a // certificate for from the ACME CA. They are removed after some time. -// When a name is in this map, do not issue a certificate for it. +// When a name is in this map, do not issue a certificate for it on-demand. var failedIssuance = make(map[string]time.Time) var failedIssuanceMu sync.RWMutex + +// lastIssueTime records when we last obtained a certificate successfully. +// If this value is recent, do not make any on-demand certificate requests. +var lastIssueTime time.Time +var lastIssueTimeMu sync.Mutex From 7bd2adf0dc3a6f4c0eefe3072043de42d1cd8273 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 11 Feb 2016 15:37:51 -0700 Subject: [PATCH 20/52] Fix edge case related to reloaded configs and ACME challenge If Caddy is running but not listening on port 80, reloading Caddy with a new Caddyfile that needs to obtain a TLS cert from the CA would fail, because it was just assumed that, if reloading, port 80 as already in use. That is not always the case, so we scan the servers to see if one of them is listening on port 80, and we configure the ACME client accordingly. Kind of a hack... but it works. --- caddy/https/https.go | 26 ++++++++++++++++++++------ caddy/restart.go | 16 +++++++++++++++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/caddy/https/https.go b/caddy/https/https.go index 2dd1bea39..4f8e989c3 100644 --- a/caddy/https/https.go +++ b/caddy/https/https.go @@ -49,7 +49,7 @@ func Activate(configs []server.Config) ([]server.Config, error) { MarkQualified(configs) // place certificates and keys on disk - err := ObtainCerts(configs, true) + err := ObtainCerts(configs, true, false) if err != nil { return configs, err } @@ -109,10 +109,12 @@ func MarkQualified(configs []server.Config) { } } -// ObtainCerts obtains certificates for all these configs as long as a certificate does not -// already exist on disk. It does not modify the configs at all; it only obtains and stores -// certificates and keys to the disk. -func ObtainCerts(configs []server.Config, allowPrompts bool) error { +// ObtainCerts obtains certificates for all these configs as long as a +// certificate does not already exist on disk. It does not modify the +// configs at all; it only obtains and stores certificates and keys to +// the disk. If allowPrompts is true, the user may be shown a prompt. +// If proxyACME is true, the ACME challenges will be proxied to our alt port. +func ObtainCerts(configs []server.Config, allowPrompts, proxyACME bool) error { // We group configs by email so we don't make the same clients over and // over. This has the potential to prompt the user for an email, but we // prevent that by assuming that if we already have a listener that can @@ -131,7 +133,19 @@ func ObtainCerts(configs []server.Config, allowPrompts bool) error { continue } - client.Configure(cfg.BindHost) + // c.Configure assumes that allowPrompts == !proxyACME, + // but that's not always true. For example, a restart where + // the user isn't present and we're not listening on port 80. + // TODO: This could probably be refactored better. + if proxyACME { + client.SetHTTPAddress(net.JoinHostPort(cfg.BindHost, AlternatePort)) + client.SetTLSAddress(net.JoinHostPort(cfg.BindHost, AlternatePort)) + client.ExcludeChallenges([]acme.Challenge{acme.TLSSNI01, acme.DNS01}) + } else { + client.SetHTTPAddress(net.JoinHostPort(cfg.BindHost, "")) + client.SetTLSAddress(net.JoinHostPort(cfg.BindHost, "")) + client.ExcludeChallenges([]acme.Challenge{acme.DNS01}) + } err := client.Obtain([]string{cfg.Host}) if err != nil { diff --git a/caddy/restart.go b/caddy/restart.go index 255f9cd7c..cc1ac516f 100644 --- a/caddy/restart.go +++ b/caddy/restart.go @@ -8,6 +8,7 @@ import ( "errors" "io/ioutil" "log" + "net" "os" "os/exec" "path" @@ -142,8 +143,21 @@ func getCertsForNewCaddyfile(newCaddyfile Input) error { // (can ignore error since we aren't actually using the certs) https.EnableTLS(configs, false) + // find out if we can let the acme package start its own challenge listener + // on port 80 + var proxyACME bool + serversMu.Lock() + for _, s := range servers { + _, port, _ := net.SplitHostPort(s.Addr) + if port == "80" { + proxyACME = true + break + } + } + serversMu.Unlock() + // place certs on the disk - err = https.ObtainCerts(configs, false) + err = https.ObtainCerts(configs, false, proxyACME) if err != nil { return errors.New("obtaining certs: " + err.Error()) } From 04c7c442c5de5d7c5fe023f23d75a999e75bd469 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 11 Feb 2016 16:20:59 -0700 Subject: [PATCH 21/52] https: Only create ACMEClient if it's actually going to be used Otherwise it tries to create an account and stuff at first start, even without a Caddyfile or when serving localhost. --- caddy/caddy_test.go | 11 ----------- caddy/https/https.go | 7 +------ caddy/https/maintain.go | 20 ++++++++++++-------- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/caddy/caddy_test.go b/caddy/caddy_test.go index 24a5d3026..be40075dc 100644 --- a/caddy/caddy_test.go +++ b/caddy/caddy_test.go @@ -4,20 +4,9 @@ import ( "net/http" "testing" "time" - - "github.com/mholt/caddy/caddy/https" - "github.com/xenolf/lego/acme" ) func TestCaddyStartStop(t *testing.T) { - // Use fake ACME clients for testing - https.NewACMEClient = func(email string, allowPrompts bool) (*https.ACMEClient, error) { - return &https.ACMEClient{ - Client: new(acme.Client), - AllowPrompts: allowPrompts, - }, nil - } - caddyfile := "localhost:1984" for i := 0; i < 2; i++ { diff --git a/caddy/https/https.go b/caddy/https/https.go index 4f8e989c3..776425bf9 100644 --- a/caddy/https/https.go +++ b/caddy/https/https.go @@ -68,12 +68,7 @@ func Activate(configs []server.Config) ([]server.Config, error) { // the renewal ticker is reset, so if restarts happen more often than // the ticker interval, renewals would never happen. but doing // it right away at start guarantees that renewals aren't missed. - client, err := NewACMEClient("", true) // renewals don't use email - if err != nil { - return configs, err - } - client.Configure("") - err = renewManagedCertificates(client) + err = renewManagedCertificates(true) if err != nil { return configs, err } diff --git a/caddy/https/maintain.go b/caddy/https/maintain.go index 03d841c72..9aa293d06 100644 --- a/caddy/https/maintain.go +++ b/caddy/https/maintain.go @@ -24,13 +24,7 @@ func maintainAssets(stopChan chan struct{}) { select { case <-renewalTicker.C: log.Println("[INFO] Scanning for expiring certificates") - client, err := NewACMEClient("", false) // renewals don't use email - if err != nil { - log.Printf("[ERROR] Creating client for renewals: %v", err) - continue - } - client.Configure("") // TODO: Bind address of relevant listener, yuck - renewManagedCertificates(client) + renewManagedCertificates(false) log.Println("[INFO] Done checking certificates") case <-ocspTicker.C: log.Println("[INFO] Scanning for stale OCSP staples") @@ -45,8 +39,9 @@ func maintainAssets(stopChan chan struct{}) { } } -func renewManagedCertificates(client *ACMEClient) error { +func renewManagedCertificates(allowPrompts bool) (err error) { var renewed, deleted []Certificate + var client *ACMEClient visitedNames := make(map[string]struct{}) certCacheMu.RLock() @@ -73,6 +68,15 @@ func renewManagedCertificates(client *ACMEClient) error { timeLeft := cert.NotAfter.Sub(time.Now().UTC()) if timeLeft < renewDurationBefore { log.Printf("[INFO] Certificate for %v expires in %v; attempting renewal", cert.Names, timeLeft) + + if client == nil { + client, err = NewACMEClient("", allowPrompts) // renewals don't use email + if err != nil { + return err + } + client.Configure("") // TODO: Bind address of relevant listener, yuck + } + err := client.Renew(cert.Names[0]) // managed certs better have only one name if err != nil { if client.AllowPrompts { From dc63e501723890c44c3fcf8bdad5414eea5d5fda Mon Sep 17 00:00:00 2001 From: Jacob Hands Date: Fri, 12 Feb 2016 08:30:47 -0600 Subject: [PATCH 22/52] Use rotating log files --- main.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index d83ef09ce..ed4449fdd 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( "github.com/mholt/caddy/caddy" "github.com/mholt/caddy/caddy/https" "github.com/xenolf/lego/acme" + "gopkg.in/natefinch/lumberjack.v2" ) var ( @@ -65,11 +66,12 @@ func main() { case "": log.SetOutput(ioutil.Discard) default: - file, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - log.Fatalf("Error opening process log file: %v", err) - } - log.SetOutput(file) + log.SetOutput(&lumberjack.Logger{ + Filename: logfile, + MaxSize: 100, + MaxAge: 14, + MaxBackups: 10, + }) } if revoke != "" { From a11e14aca8190cd0779cbcc6991742d4a376d45b Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Fri, 12 Feb 2016 13:04:24 -0700 Subject: [PATCH 23/52] Fix HTTPS config for empty/no Caddyfile This fixes a regression introduced in recent commits that enabled TLS on the default ":2015" config. This fix is possible because On-Demand TLS is no longer implicit; it must be explicitly enabled by the user by setting a maximum number of certificates to issue. --- caddy/https/https.go | 22 +++++++++++----------- caddy/https/https_test.go | 6 +++--- caddy/https/setup.go | 12 +++++------- server/config.go | 7 ++++--- server/server.go | 14 ++++++++------ 5 files changed, 31 insertions(+), 30 deletions(-) diff --git a/caddy/https/https.go b/caddy/https/https.go index 776425bf9..cfff28687 100644 --- a/caddy/https/https.go +++ b/caddy/https/https.go @@ -95,7 +95,7 @@ func Deactivate() (err error) { } // MarkQualified scans each config and, if it qualifies for managed -// TLS, it sets the Marked field of the TLSConfig to true. +// TLS, it sets the Managed field of the TLSConfig to true. func MarkQualified(configs []server.Config) { for i := 0; i < len(configs); i++ { if ConfigQualifies(configs[i]) { @@ -152,9 +152,10 @@ func ObtainCerts(configs []server.Config, allowPrompts, proxyACME bool) error { return nil } -// groupConfigsByEmail groups configs by the email address to be used by its -// ACME client. It only includes configs that are marked as fully managed. -// If userPresent is true, the operator MAY be prompted for an email address. +// groupConfigsByEmail groups configs by the email address to be used by an +// ACME client. It only groups configs that have TLS enabled and that are +// marked as Managed. If userPresent is true, the operator MAY be prompted +// for an email address. func groupConfigsByEmail(configs []server.Config, userPresent bool) map[string][]server.Config { initMap := make(map[string][]server.Config) for _, cfg := range configs { @@ -214,7 +215,7 @@ func hostHasOtherPort(allConfigs []server.Config, thisConfigIdx int, otherPort s // all configs. func MakePlaintextRedirects(allConfigs []server.Config) []server.Config { for i, cfg := range allConfigs { - if cfg.TLS.Managed && + if (cfg.TLS.Managed || cfg.TLS.OnDemand) && !hostHasOtherPort(allConfigs, i, "80") && (cfg.Port == "443" || !hostHasOtherPort(allConfigs, i, "443")) { allConfigs = append(allConfigs, redirPlaintextHost(cfg)) @@ -224,10 +225,11 @@ func MakePlaintextRedirects(allConfigs []server.Config) []server.Config { } // ConfigQualifies returns true if cfg qualifies for -// fully managed TLS. It does NOT check to see if a +// fully managed TLS (but not on-demand TLS, which is +// not considered here). It does NOT check to see if a // cert and key already exist for the config. If the // config does qualify, you should set cfg.TLS.Managed -// to true and use that instead, because the process of +// to true and check that instead, because the process of // setting up the config may make it look like it // doesn't qualify even though it originally did. func ConfigQualifies(cfg server.Config) bool { @@ -238,10 +240,8 @@ func ConfigQualifies(cfg server.Config) bool { cfg.Port != "80" && cfg.TLS.LetsEncryptEmail != "off" && - // we get can't certs for some kinds of hostnames, - // but we CAN get certs at request-time even if - // the hostname in the config is empty right now. - (cfg.Host == "" || HostQualifies(cfg.Host)) + // we get can't certs for some kinds of hostnames + HostQualifies(cfg.Host) } // HostQualifies returns true if the hostname alone diff --git a/caddy/https/https_test.go b/caddy/https/https_test.go index e4efd2373..d724635b7 100644 --- a/caddy/https/https_test.go +++ b/caddy/https/https_test.go @@ -46,7 +46,7 @@ func TestConfigQualifies(t *testing.T) { cfg server.Config expect bool }{ - {server.Config{Host: ""}, true}, + {server.Config{Host: ""}, false}, {server.Config{Host: "localhost"}, false}, {server.Config{Host: "123.44.3.21"}, false}, {server.Config{Host: "example.com"}, true}, @@ -302,6 +302,7 @@ func TestGroupConfigsByEmail(t *testing.T) { func TestMarkQualified(t *testing.T) { // TODO: TestConfigQualifies and this test share the same config list... configs := []server.Config{ + {Host: ""}, {Host: "localhost"}, {Host: "123.44.3.21"}, {Host: "example.com"}, @@ -313,9 +314,8 @@ func TestMarkQualified(t *testing.T) { {Host: "example.com", Port: "1234"}, {Host: "example.com", Scheme: "https"}, {Host: "example.com", Port: "80", Scheme: "https"}, - {Host: ""}, } - expectedManagedCount := 5 + expectedManagedCount := 4 MarkQualified(configs) diff --git a/caddy/https/setup.go b/caddy/https/setup.go index 2500bce07..ebf46d244 100644 --- a/caddy/https/setup.go +++ b/caddy/https/setup.go @@ -83,6 +83,7 @@ func Setup(c *setup.Controller) (middleware.Middleware, error) { c.TLS.Manual = true case "max_certs": c.Args(&maxCerts) + c.TLS.OnDemand = true default: return nil, c.Errf("Unknown keyword '%s'", c.Val()) } @@ -93,21 +94,18 @@ func Setup(c *setup.Controller) (middleware.Middleware, error) { return nil, c.ArgErr() } - if c.TLS.Manual && maxCerts != "" { - return nil, c.Err("Cannot limit certificate count (max_certs) for manual TLS configurations") - } - + // set certificate limit if on-demand TLS is enabled if maxCerts != "" { maxCertsNum, err := strconv.Atoi(maxCerts) - if err != nil || maxCertsNum < 0 { + if err != nil || maxCertsNum < 1 { return nil, c.Err("max_certs must be a positive integer") } - if onDemandMaxIssue == 0 || int32(maxCertsNum) < onDemandMaxIssue { // keep the minimum; TODO: This is global; should be per-server or per-vhost... + if onDemandMaxIssue == 0 || int32(maxCertsNum) < onDemandMaxIssue { // keep the minimum; TODO: We have to do this because it is global; should be per-server or per-vhost... onDemandMaxIssue = int32(maxCertsNum) } } - // don't load certificates unless we're supposed to + // don't try to load certificates unless we're supposed to if !c.TLS.Enabled || !c.TLS.Manual { continue } diff --git a/server/config.go b/server/config.go index cae1edf56..9acdac7f5 100644 --- a/server/config.go +++ b/server/config.go @@ -65,10 +65,11 @@ func (c Config) Address() string { // TLSConfig describes how TLS should be configured and used. type TLSConfig struct { - Enabled bool + Enabled bool // will be set to true if TLS is enabled LetsEncryptEmail string - Managed bool // will be set to true if config qualifies for automatic, managed TLS - Manual bool // will be set to true if user provides the cert and key files + Manual bool // will be set to true if user provides own certs and keys + Managed bool // will be set to true if config qualifies for automatic/managed HTTPS + OnDemand bool // will be set to true if user enables on-demand TLS (obtain certs during handshakes) Ciphers []uint16 ProtocolMinVersion uint16 ProtocolMaxVersion uint16 diff --git a/server/server.go b/server/server.go index 43b1d8571..3b38e4833 100644 --- a/server/server.go +++ b/server/server.go @@ -63,12 +63,14 @@ func New(addr string, configs []Config, gracefulTimeout time.Duration) (*Server, var useTLS, useOnDemandTLS bool if len(configs) > 0 { useTLS = configs[0].TLS.Enabled - host, _, err := net.SplitHostPort(addr) - if err != nil { - host = addr - } - if useTLS && host == "" && !configs[0].TLS.Manual { - useOnDemandTLS = true + if useTLS { + host, _, err := net.SplitHostPort(addr) + if err != nil { + host = addr + } + if host == "" && configs[0].TLS.OnDemand { + useOnDemandTLS = true + } } } From cae9f7de9c7915bfd02cc8606df9029986c97071 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Sun, 14 Feb 2016 00:10:57 -0700 Subject: [PATCH 24/52] gofmt -s; fix misspellings and lint; Go 1.5.3 in Travis CI --- .travis.yml | 2 +- caddy/https/https.go | 2 +- caddy/https/https_test.go | 34 +++++++++++++-------------- caddy/parse/parsing_test.go | 20 ++++++++-------- caddy/setup/browse_test.go | 2 +- caddy/setup/redir_test.go | 20 ++++++++-------- caddy/setup/rewrite_test.go | 2 +- caddy/setup/startupshutdown_test.go | 2 +- middleware/context_test.go | 8 +++---- middleware/fastcgi/fastcgi.go | 2 +- middleware/fastcgi/fastcgi_test.go | 8 +++---- middleware/fastcgi/fcgiclient_test.go | 8 +++---- middleware/fileserver_test.go | 4 ++-- middleware/middleware.go | 3 ++- middleware/proxy/proxy_test.go | 4 ++-- server/server.go | 2 +- 16 files changed, 61 insertions(+), 62 deletions(-) diff --git a/.travis.yml b/.travis.yml index 19ba6dbab..92bbffe59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.4.3 - - 1.5.2 + - 1.5.3 - tip install: diff --git a/caddy/https/https.go b/caddy/https/https.go index cfff28687..f6cdcd467 100644 --- a/caddy/https/https.go +++ b/caddy/https/https.go @@ -336,7 +336,7 @@ func redirPlaintextHost(cfg server.Config) server.Config { BindHost: cfg.BindHost, Port: "80", Middleware: map[string][]middleware.Middleware{ - "/": []middleware.Middleware{redirMidware}, + "/": {redirMidware}, }, } } diff --git a/caddy/https/https_test.go b/caddy/https/https_test.go index d724635b7..199c6266b 100644 --- a/caddy/https/https_test.go +++ b/caddy/https/https_test.go @@ -209,9 +209,9 @@ func TestExistingCertAndKey(t *testing.T) { func TestHostHasOtherPort(t *testing.T) { configs := []server.Config{ - server.Config{Host: "example.com", Port: "80"}, - server.Config{Host: "sub1.example.com", Port: "80"}, - server.Config{Host: "sub1.example.com", Port: "443"}, + {Host: "example.com", Port: "80"}, + {Host: "sub1.example.com", Port: "80"}, + {Host: "sub1.example.com", Port: "443"}, } if hostHasOtherPort(configs, 0, "80") { @@ -228,18 +228,18 @@ func TestHostHasOtherPort(t *testing.T) { func TestMakePlaintextRedirects(t *testing.T) { configs := []server.Config{ // Happy path = standard redirect from 80 to 443 - server.Config{Host: "example.com", TLS: server.TLSConfig{Managed: true}}, + {Host: "example.com", TLS: server.TLSConfig{Managed: true}}, // Host on port 80 already defined; don't change it (no redirect) - server.Config{Host: "sub1.example.com", Port: "80", Scheme: "http"}, - server.Config{Host: "sub1.example.com", TLS: server.TLSConfig{Managed: true}}, + {Host: "sub1.example.com", Port: "80", Scheme: "http"}, + {Host: "sub1.example.com", TLS: server.TLSConfig{Managed: true}}, // Redirect from port 80 to port 5000 in this case - server.Config{Host: "sub2.example.com", Port: "5000", TLS: server.TLSConfig{Managed: true}}, + {Host: "sub2.example.com", Port: "5000", TLS: server.TLSConfig{Managed: true}}, // Can redirect from 80 to either 443 or 5001, but choose 443 - server.Config{Host: "sub3.example.com", Port: "443", TLS: server.TLSConfig{Managed: true}}, - server.Config{Host: "sub3.example.com", Port: "5001", Scheme: "https", TLS: server.TLSConfig{Managed: true}}, + {Host: "sub3.example.com", Port: "443", TLS: server.TLSConfig{Managed: true}}, + {Host: "sub3.example.com", Port: "5001", Scheme: "https", TLS: server.TLSConfig{Managed: true}}, } result := MakePlaintextRedirects(configs) @@ -253,8 +253,8 @@ func TestMakePlaintextRedirects(t *testing.T) { func TestEnableTLS(t *testing.T) { configs := []server.Config{ - server.Config{Host: "example.com", TLS: server.TLSConfig{Managed: true}}, - server.Config{}, // not managed - no changes! + {Host: "example.com", TLS: server.TLSConfig{Managed: true}}, + {}, // not managed - no changes! } EnableTLS(configs, false) @@ -273,12 +273,12 @@ func TestGroupConfigsByEmail(t *testing.T) { } configs := []server.Config{ - server.Config{Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "", Managed: true}}, - server.Config{Host: "sub1.example.com", TLS: server.TLSConfig{LetsEncryptEmail: "foo@bar", Managed: true}}, - server.Config{Host: "sub2.example.com", TLS: server.TLSConfig{LetsEncryptEmail: "", Managed: true}}, - server.Config{Host: "sub3.example.com", TLS: server.TLSConfig{LetsEncryptEmail: "foo@bar", Managed: true}}, - server.Config{Host: "sub4.example.com", TLS: server.TLSConfig{LetsEncryptEmail: "", Managed: true}}, - server.Config{Host: "sub5.example.com", TLS: server.TLSConfig{LetsEncryptEmail: ""}}, // not managed + {Host: "example.com", TLS: server.TLSConfig{LetsEncryptEmail: "", Managed: true}}, + {Host: "sub1.example.com", TLS: server.TLSConfig{LetsEncryptEmail: "foo@bar", Managed: true}}, + {Host: "sub2.example.com", TLS: server.TLSConfig{LetsEncryptEmail: "", Managed: true}}, + {Host: "sub3.example.com", TLS: server.TLSConfig{LetsEncryptEmail: "foo@bar", Managed: true}}, + {Host: "sub4.example.com", TLS: server.TLSConfig{LetsEncryptEmail: "", Managed: true}}, + {Host: "sub5.example.com", TLS: server.TLSConfig{LetsEncryptEmail: ""}}, // not managed } DefaultEmail = "test@example.com" diff --git a/caddy/parse/parsing_test.go b/caddy/parse/parsing_test.go index 462cd40fe..493c0fff9 100644 --- a/caddy/parse/parsing_test.go +++ b/caddy/parse/parsing_test.go @@ -311,19 +311,19 @@ func TestParseAll(t *testing.T) { }}, {`localhost:1234`, false, [][]address{ - []address{{"localhost:1234", "", "localhost", "1234"}}, + {{"localhost:1234", "", "localhost", "1234"}}, }}, {`localhost:1234 { } localhost:2015 { }`, false, [][]address{ - []address{{"localhost:1234", "", "localhost", "1234"}}, - []address{{"localhost:2015", "", "localhost", "2015"}}, + {{"localhost:1234", "", "localhost", "1234"}}, + {{"localhost:2015", "", "localhost", "2015"}}, }}, {`localhost:1234, http://host2`, false, [][]address{ - []address{{"localhost:1234", "", "localhost", "1234"}, {"http://host2", "http", "host2", "80"}}, + {{"localhost:1234", "", "localhost", "1234"}, {"http://host2", "http", "host2", "80"}}, }}, {`localhost:1234, http://host2,`, true, [][]address{}}, @@ -332,15 +332,15 @@ func TestParseAll(t *testing.T) { } https://host3.com, https://host4.com { }`, false, [][]address{ - []address{{"http://host1.com", "http", "host1.com", "80"}, {"http://host2.com", "http", "host2.com", "80"}}, - []address{{"https://host3.com", "https", "host3.com", "443"}, {"https://host4.com", "https", "host4.com", "443"}}, + {{"http://host1.com", "http", "host1.com", "80"}, {"http://host2.com", "http", "host2.com", "80"}}, + {{"https://host3.com", "https", "host3.com", "443"}, {"https://host4.com", "https", "host4.com", "443"}}, }}, {`import import_glob*.txt`, false, [][]address{ - []address{{"glob0.host0", "", "glob0.host0", ""}}, - []address{{"glob0.host1", "", "glob0.host1", ""}}, - []address{{"glob1.host0", "", "glob1.host0", ""}}, - []address{{"glob2.host0", "", "glob2.host0", ""}}, + {{"glob0.host0", "", "glob0.host0", ""}}, + {{"glob0.host1", "", "glob0.host1", ""}}, + {{"glob1.host0", "", "glob1.host0", ""}}, + {{"glob2.host0", "", "glob2.host0", ""}}, }}, } { p := testParser(test.input) diff --git a/caddy/setup/browse_test.go b/caddy/setup/browse_test.go index 3714a51dd..443e008bb 100644 --- a/caddy/setup/browse_test.go +++ b/caddy/setup/browse_test.go @@ -41,7 +41,7 @@ func TestBrowse(t *testing.T) { // test case #2 tests detectaction of custom template {"browse . " + tempTemplatePath, []string{"."}, false}, - // test case #3 tests detection of non-existant template + // test case #3 tests detection of non-existent template {"browse . " + nonExistantDirPath, nil, true}, // test case #4 tests detection of duplicate pathscopes diff --git a/caddy/setup/redir_test.go b/caddy/setup/redir_test.go index 773666f8d..0285784fa 100644 --- a/caddy/setup/redir_test.go +++ b/caddy/setup/redir_test.go @@ -14,34 +14,34 @@ func TestRedir(t *testing.T) { expectedRules []redirect.Rule }{ // test case #0 tests the recognition of a valid HTTP status code defined outside of block statement - {"redir 300 {\n/ /foo\n}", false, []redirect.Rule{redirect.Rule{FromPath: "/", To: "/foo", Code: 300}}}, + {"redir 300 {\n/ /foo\n}", false, []redirect.Rule{{FromPath: "/", To: "/foo", Code: 300}}}, // test case #1 tests the recognition of an invalid HTTP status code defined outside of block statement - {"redir 9000 {\n/ /foo\n}", true, []redirect.Rule{redirect.Rule{}}}, + {"redir 9000 {\n/ /foo\n}", true, []redirect.Rule{{}}}, // test case #2 tests the detection of a valid HTTP status code outside of a block statement being overriden by an invalid HTTP status code inside statement of a block statement - {"redir 300 {\n/ /foo 9000\n}", true, []redirect.Rule{redirect.Rule{}}}, + {"redir 300 {\n/ /foo 9000\n}", true, []redirect.Rule{{}}}, // test case #3 tests the detection of an invalid HTTP status code outside of a block statement being overriden by a valid HTTP status code inside statement of a block statement - {"redir 9000 {\n/ /foo 300\n}", true, []redirect.Rule{redirect.Rule{}}}, + {"redir 9000 {\n/ /foo 300\n}", true, []redirect.Rule{{}}}, // test case #4 tests the recognition of a TO redirection in a block statement.The HTTP status code is set to the default of 301 - MovedPermanently - {"redir 302 {\n/foo\n}", false, []redirect.Rule{redirect.Rule{FromPath: "/", To: "/foo", Code: 302}}}, + {"redir 302 {\n/foo\n}", false, []redirect.Rule{{FromPath: "/", To: "/foo", Code: 302}}}, // test case #5 tests the recognition of a TO and From redirection in a block statement - {"redir {\n/bar /foo 303\n}", false, []redirect.Rule{redirect.Rule{FromPath: "/bar", To: "/foo", Code: 303}}}, + {"redir {\n/bar /foo 303\n}", false, []redirect.Rule{{FromPath: "/bar", To: "/foo", Code: 303}}}, // test case #6 tests the recognition of a TO redirection in a non-block statement. The HTTP status code is set to the default of 301 - MovedPermanently - {"redir /foo", false, []redirect.Rule{redirect.Rule{FromPath: "/", To: "/foo", Code: 301}}}, + {"redir /foo", false, []redirect.Rule{{FromPath: "/", To: "/foo", Code: 301}}}, // test case #7 tests the recognition of a TO and From redirection in a non-block statement - {"redir /bar /foo 303", false, []redirect.Rule{redirect.Rule{FromPath: "/bar", To: "/foo", Code: 303}}}, + {"redir /bar /foo 303", false, []redirect.Rule{{FromPath: "/bar", To: "/foo", Code: 303}}}, // test case #8 tests the recognition of multiple redirections - {"redir {\n / /foo 304 \n} \n redir {\n /bar /foobar 305 \n}", false, []redirect.Rule{redirect.Rule{FromPath: "/", To: "/foo", Code: 304}, redirect.Rule{FromPath: "/bar", To: "/foobar", Code: 305}}}, + {"redir {\n / /foo 304 \n} \n redir {\n /bar /foobar 305 \n}", false, []redirect.Rule{{FromPath: "/", To: "/foo", Code: 304}, {FromPath: "/bar", To: "/foobar", Code: 305}}}, // test case #9 tests the detection of duplicate redirections - {"redir {\n /bar /foo 304 \n} redir {\n /bar /foo 304 \n}", true, []redirect.Rule{redirect.Rule{}}}, + {"redir {\n /bar /foo 304 \n} redir {\n /bar /foo 304 \n}", true, []redirect.Rule{{}}}, } { recievedFunc, err := Redir(NewTestController(test.input)) if err != nil && !test.shouldErr { diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index 224ab643f..29bfe9975 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -135,7 +135,7 @@ func TestRewriteParse(t *testing.T) { to /to if {path} is a }`, false, []rewrite.Rule{ - &rewrite.ComplexRule{Base: "/", To: "/to", Ifs: []rewrite.If{rewrite.If{A: "{path}", Operator: "is", B: "a"}}}, + &rewrite.ComplexRule{Base: "/", To: "/to", Ifs: []rewrite.If{{A: "{path}", Operator: "is", B: "a"}}}, }}, {`rewrite { status 400 diff --git a/caddy/setup/startupshutdown_test.go b/caddy/setup/startupshutdown_test.go index 16fa973c3..871a64214 100644 --- a/caddy/setup/startupshutdown_test.go +++ b/caddy/setup/startupshutdown_test.go @@ -37,7 +37,7 @@ func TestStartup(t *testing.T) { // test case #1 tests proper functionality of non-blocking commands {"startup mkdir " + osSenitiveTestDir + " &", false, true}, - // test case #2 tests handling of non-existant commands + // test case #2 tests handling of non-existent commands {"startup " + strconv.Itoa(int(time.Now().UnixNano())), true, true}, } diff --git a/middleware/context_test.go b/middleware/context_test.go index 5fb883c6f..5c6473e9e 100644 --- a/middleware/context_test.go +++ b/middleware/context_test.go @@ -105,13 +105,13 @@ func TestMarkdown(t *testing.T) { }() tests := []struct { - fileContent string - expectedContent string + fileContent string + expectedContent string }{ // Test 0 - test parsing of markdown { - fileContent: "* str1\n* str2\n", - expectedContent: "
    \n
  • str1
    • \n
  • str2
    • \n
\n", + fileContent: "* str1\n* str2\n", + expectedContent: "
    \n
  • str1
    • \n
  • str2
    • \n
\n", }, } diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go index 153cae7f6..bddb04705 100755 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -138,7 +138,7 @@ func (r Rule) parseAddress() (string, string) { if strings.HasPrefix(r.Address, "tcp://") { return "tcp", r.Address[len("tcp://"):] } - // check if address has fastcgi scheme explicity set + // check if address has fastcgi scheme explicitly set if strings.HasPrefix(r.Address, "fastcgi://") { return "tcp", r.Address[len("fastcgi://"):] } diff --git a/middleware/fastcgi/fastcgi_test.go b/middleware/fastcgi/fastcgi_test.go index 1fc7446d0..c33f47af9 100644 --- a/middleware/fastcgi/fastcgi_test.go +++ b/middleware/fastcgi/fastcgi_test.go @@ -35,20 +35,20 @@ func TestRuleParseAddress(t *testing.T) { func TestBuildEnv(t *testing.T) { buildEnvSingle := func(r *http.Request, rule Rule, fpath string, envExpected map[string]string, t *testing.T) { - + h := Handler{} - + env, err := h.buildEnv(r, rule, fpath) if err != nil { t.Error("Unexpected error:", err.Error()) } - + for k, v := range envExpected { if env[k] != v { t.Errorf("Unexpected %v. Got %v, expected %v", k, env[k], v) } } - + } rule := Rule{} diff --git a/middleware/fastcgi/fcgiclient_test.go b/middleware/fastcgi/fcgiclient_test.go index 6ed37bb4a..0eeebb4b1 100644 --- a/middleware/fastcgi/fcgiclient_test.go +++ b/middleware/fastcgi/fcgiclient_test.go @@ -39,9 +39,7 @@ const ( ipPort = "127.0.0.1:59000" ) -var ( - t_ *testing.T -) +var globalt *testing.T type FastCGIServer struct{} @@ -158,7 +156,7 @@ func sendFcgi(reqType int, fcgiParams map[string]string, data []byte, posts map[ time.Sleep(1 * time.Second) if bytes.Index(content, []byte("FAILED")) >= 0 { - t_.Error("Server return failed message") + globalt.Error("Server return failed message") } return @@ -193,7 +191,7 @@ func generateRandFile(size int) (p string, m string) { func DisabledTest(t *testing.T) { // TODO: test chunked reader - t_ = t + globalt = t rand.Seed(time.Now().UTC().UnixNano()) diff --git a/middleware/fileserver_test.go b/middleware/fileserver_test.go index 0ce454bb1..0f5b1faca 100644 --- a/middleware/fileserver_test.go +++ b/middleware/fileserver_test.go @@ -45,7 +45,7 @@ func TestServeHTTP(t *testing.T) { expectedStatus int expectedBodyContent string }{ - // Test 0 - access withoutt any path + // Test 0 - access without any path { url: "https://foo", expectedStatus: http.StatusNotFound, @@ -78,7 +78,7 @@ func TestServeHTTP(t *testing.T) { url: "https://foo/dir/", expectedStatus: http.StatusNotFound, }, - // Test 6 - access folder withtout trailing slash + // Test 6 - access folder without trailing slash { url: "https://foo/dir", expectedStatus: http.StatusMovedPermanently, diff --git a/middleware/middleware.go b/middleware/middleware.go index b88b24474..c7036f3c9 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -102,7 +102,8 @@ func SetLastModifiedHeader(w http.ResponseWriter, modTime time.Time) { w.Header().Set("Last-Modified", modTime.UTC().Format(http.TimeFormat)) } -// currentTime returns time.Now() everytime it's called. It's used for mocking in tests. +// currentTime, as it is defined here, returns time.Now(). +// It's defined as a variable for mocking time in tests. var currentTime = func() time.Time { return time.Now() } diff --git a/middleware/proxy/proxy_test.go b/middleware/proxy/proxy_test.go index 68b135679..8066874d2 100644 --- a/middleware/proxy/proxy_test.go +++ b/middleware/proxy/proxy_test.go @@ -12,11 +12,11 @@ import ( "net/http/httptest" "net/url" "os" + "path/filepath" + "runtime" "strings" "testing" - "runtime" "time" - "path/filepath" "golang.org/x/net/websocket" ) diff --git a/server/server.go b/server/server.go index 3b38e4833..c921566c5 100644 --- a/server/server.go +++ b/server/server.go @@ -336,7 +336,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err != nil { remoteHost = r.RemoteAddr } - + w.WriteHeader(http.StatusNotFound) fmt.Fprintf(w, "No such host at %s", s.Server.Addr) log.Printf("[INFO] %s - No such host at %s (requested by %s)", host, s.Server.Addr, remoteHost) From 1cfd960f3ccc3fcfaef240c93069fcc0b11c0890 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 15 Feb 2016 23:39:04 -0700 Subject: [PATCH 25/52] Bug fixes and other improvements to TLS functions Now attempt to staple OCSP even for certs that don't have an existing staple (issue #605). "tls off" short-circuits tls setup function. Now we call getEmail() when setting up an acme.Client that does renewals, rather than making a new account with empty email address. Check certificate expiry every 12 hours, and OCSP every hour. --- caddy/https/certificates.go | 8 +- caddy/https/handshake.go | 12 +-- caddy/https/https.go | 29 +++---- caddy/https/maintain.go | 84 +++++++++++++------ caddy/https/setup.go | 161 +++++++++++++++++++----------------- server/config.go | 2 +- server/server.go | 10 +-- 7 files changed, 168 insertions(+), 138 deletions(-) diff --git a/caddy/https/certificates.go b/caddy/https/certificates.go index 72a9ff1c7..b123d4c32 100644 --- a/caddy/https/certificates.go +++ b/caddy/https/certificates.go @@ -24,7 +24,7 @@ var certCacheMu sync.RWMutex // we can be more efficient by extracting the metadata once so it's // just there, ready to use. type Certificate struct { - *tls.Certificate + tls.Certificate // Names is the list of names this certificate is written for. // The first is the CommonName (if any), the rest are SAN. @@ -170,7 +170,6 @@ func makeCertificate(certPEMBlock, keyPEMBlock []byte) (Certificate, error) { if len(tlsCert.Certificate) == 0 { return cert, errors.New("certificate is empty") } - cert.Certificate = &tlsCert // Parse leaf certificate and extract relevant metadata leaf, err := x509.ParseCertificate(tlsCert.Certificate[0]) @@ -198,6 +197,7 @@ func makeCertificate(certPEMBlock, keyPEMBlock []byte) (Certificate, error) { cert.OCSP = ocspResp } + cert.Certificate = tlsCert return cert, nil } @@ -213,7 +213,9 @@ func makeCertificate(certPEMBlock, keyPEMBlock []byte) (Certificate, error) { func cacheCertificate(cert Certificate) { certCacheMu.Lock() if _, ok := certCache[""]; !ok { - certCache[""] = cert // use as default + // use as default + certCache[""] = cert + cert.Names = append(cert.Names, "") } for len(certCache)+len(cert.Names) > 10000 { // for simplicity, just remove random elements diff --git a/caddy/https/handshake.go b/caddy/https/handshake.go index 3d759b31b..38f9afb55 100644 --- a/caddy/https/handshake.go +++ b/caddy/https/handshake.go @@ -23,7 +23,7 @@ import ( // This function is safe for use as a tls.Config.GetCertificate callback. func GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { cert, err := getCertDuringHandshake(clientHello.ServerName, false, false) - return cert.Certificate, err + return &cert.Certificate, err } // GetOrObtainCertificate will get a certificate to satisfy clientHello, even @@ -35,7 +35,7 @@ func GetCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) // This function is safe for use as a tls.Config.GetCertificate callback. func GetOrObtainCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { cert, err := getCertDuringHandshake(clientHello.ServerName, true, true) - return cert.Certificate, err + return &cert.Certificate, err } // getCertDuringHandshake will get a certificate for name. It first tries @@ -122,8 +122,8 @@ func checkLimitsForObtainingNewCerts(name string) error { } // obtainOnDemandCertificate obtains a certificate for name for the given -// clientHello. If another goroutine has already started obtaining a cert -// for name, it will wait and use what the other goroutine obtained. +// name. If another goroutine has already started obtaining a cert for +// name, it will wait and use what the other goroutine obtained. // // This function is safe for use by multiple concurrent goroutines. func obtainOnDemandCertificate(name string) (Certificate, error) { @@ -248,7 +248,7 @@ func renewDynamicCertificate(name string) (Certificate, error) { log.Printf("[INFO] Renewing certificate for %s", name) - client, err := NewACMEClient("", false) // renewals don't use email + client, err := NewACMEClientGetEmail(server.Config{}, false) if err != nil { return Certificate{}, err } @@ -295,7 +295,7 @@ var obtainCertWaitChansMu sync.Mutex // OnDemandIssuedCount is the number of certificates that have been issued // on-demand by this process. It is only safe to modify this count atomically. -// If it reaches max_certs, on-demand issuances will fail. +// If it reaches onDemandMaxIssue, on-demand issuances will fail. var OnDemandIssuedCount = new(int32) // onDemandMaxIssue is set based on max_certs in tls config. It specifies the diff --git a/caddy/https/https.go b/caddy/https/https.go index f6cdcd467..4526a31cd 100644 --- a/caddy/https/https.go +++ b/caddy/https/https.go @@ -12,7 +12,6 @@ import ( "net/http" "os" "strings" - "time" "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/middleware/redirect" @@ -215,7 +214,7 @@ func hostHasOtherPort(allConfigs []server.Config, thisConfigIdx int, otherPort s // all configs. func MakePlaintextRedirects(allConfigs []server.Config) []server.Config { for i, cfg := range allConfigs { - if (cfg.TLS.Managed || cfg.TLS.OnDemand) && + if cfg.TLS.Managed && !hostHasOtherPort(allConfigs, i, "80") && (cfg.Port == "443" || !hostHasOtherPort(allConfigs, i, "443")) { allConfigs = append(allConfigs, redirPlaintextHost(cfg)) @@ -233,15 +232,16 @@ func MakePlaintextRedirects(allConfigs []server.Config) []server.Config { // setting up the config may make it look like it // doesn't qualify even though it originally did. func ConfigQualifies(cfg server.Config) bool { - return !cfg.TLS.Manual && // user can provide own cert and key + return (!cfg.TLS.Manual || cfg.TLS.OnDemand) && // user might provide own cert and key // user can force-disable automatic HTTPS for this host cfg.Scheme != "http" && cfg.Port != "80" && cfg.TLS.LetsEncryptEmail != "off" && - // we get can't certs for some kinds of hostnames - HostQualifies(cfg.Host) + // we get can't certs for some kinds of hostnames, but + // on-demand TLS allows empty hostnames at startup + (HostQualifies(cfg.Host) || cfg.TLS.OnDemand) } // HostQualifies returns true if the hostname alone @@ -387,20 +387,11 @@ var ( CAUrl string ) -// Some essential values related to the Let's Encrypt process -const ( - // AlternatePort is the port on which the acme client will open a - // listener and solve the CA's challenges. If this alternate port - // is used instead of the default port (80 or 443), then the - // default port for the challenge must be forwarded to this one. - AlternatePort = "5033" - - // RenewInterval is how often to check certificates for renewal. - RenewInterval = 6 * time.Hour - - // OCSPInterval is how often to check if OCSP stapling needs updating. - OCSPInterval = 1 * time.Hour -) +// AlternatePort is the port on which the acme client will open a +// listener and solve the CA's challenges. If this alternate port +// is used instead of the default port (80 or 443), then the +// default port for the challenge must be forwarded to this one. +const AlternatePort = "5033" // KeySize represents the length of a key in bits. type KeySize int diff --git a/caddy/https/maintain.go b/caddy/https/maintain.go index 9aa293d06..49fc1c169 100644 --- a/caddy/https/maintain.go +++ b/caddy/https/maintain.go @@ -4,9 +4,19 @@ import ( "log" "time" + "github.com/mholt/caddy/server" + "golang.org/x/crypto/ocsp" ) +const ( + // RenewInterval is how often to check certificates for renewal. + RenewInterval = 12 * time.Hour + + // OCSPInterval is how often to check if OCSP stapling needs updating. + OCSPInterval = 1 * time.Hour +) + // maintainAssets is a permanently-blocking function // that loops indefinitely and, on a regular schedule, checks // certificates for expiration and initiates a renewal of certs @@ -28,7 +38,7 @@ func maintainAssets(stopChan chan struct{}) { log.Println("[INFO] Done checking certificates") case <-ocspTicker.C: log.Println("[INFO] Scanning for stale OCSP staples") - updatePreloadedOCSPStaples() + updateOCSPStaples() log.Println("[INFO] Done checking OCSP staples") case <-stopChan: renewalTicker.Stop() @@ -70,7 +80,7 @@ func renewManagedCertificates(allowPrompts bool) (err error) { log.Printf("[INFO] Certificate for %v expires in %v; attempting renewal", cert.Names, timeLeft) if client == nil { - client, err = NewACMEClient("", allowPrompts) // renewals don't use email + client, err = NewACMEClientGetEmail(server.Config{}, allowPrompts) if err != nil { return err } @@ -116,42 +126,66 @@ func renewManagedCertificates(allowPrompts bool) (err error) { return nil } -func updatePreloadedOCSPStaples() { +func updateOCSPStaples() { // Create a temporary place to store updates - // until we release the potentially slow read - // lock so we can use a quick write lock. + // until we release the potentially long-lived + // read lock and use a short-lived write lock. type ocspUpdate struct { - rawBytes []byte - parsedResponse *ocsp.Response + rawBytes []byte + parsed *ocsp.Response } updated := make(map[string]ocspUpdate) + // A single SAN certificate maps to multiple names, so we use this + // set to make sure we don't waste cycles checking OCSP for the same + // certificate multiple times. + visited := make(map[string]struct{}) + certCacheMu.RLock() for name, cert := range certCache { - // we update OCSP for managed and un-managed certs here, but only - // if it has OCSP stapled and only for pre-loaded certificates - if cert.OnDemand || cert.OCSP == nil { + // skip this certificate if we've already visited it, + // and if not, mark all the names as visited + if _, ok := visited[name]; ok { + continue + } + for _, n := range cert.Names { + visited[n] = struct{}{} + } + + // no point in updating OCSP for expired certificates + if time.Now().After(cert.NotAfter) { continue } - // start checking OCSP staple about halfway through validity period for good measure - oldNextUpdate := cert.OCSP.NextUpdate - refreshTime := cert.OCSP.ThisUpdate.Add(oldNextUpdate.Sub(cert.OCSP.ThisUpdate) / 2) + var lastNextUpdate time.Time + if cert.OCSP != nil { + // start checking OCSP staple about halfway through validity period for good measure + lastNextUpdate = cert.OCSP.NextUpdate + refreshTime := cert.OCSP.ThisUpdate.Add(lastNextUpdate.Sub(cert.OCSP.ThisUpdate) / 2) - // only check for updated OCSP validity window if the refresh time is - // in the past and the certificate is not expired - if time.Now().After(refreshTime) && time.Now().Before(cert.NotAfter) { - err := stapleOCSP(&cert, nil) - if err != nil { - log.Printf("[ERROR] Checking OCSP for %s: %v", name, err) + // since OCSP is already stapled, we need only check if we're in that "refresh window" + if time.Now().Before(refreshTime) { continue } + } - // if the OCSP response has been updated, we use it - if oldNextUpdate != cert.OCSP.NextUpdate { - log.Printf("[INFO] Moving validity period of OCSP staple for %s from %v to %v", - name, oldNextUpdate, cert.OCSP.NextUpdate) - updated[name] = ocspUpdate{rawBytes: cert.Certificate.OCSPStaple, parsedResponse: cert.OCSP} + err := stapleOCSP(&cert, nil) + if err != nil { + if cert.OCSP != nil { + // if it was no staple before, that's fine, otherwise we should log the error + log.Printf("[ERROR] Checking OCSP for %s: %v", name, err) + } + continue + } + + // By this point, we've obtained the latest OCSP response. + // If there was no staple before, or if the response is updated, make + // sure we apply the update to all names on the certificate. + if lastNextUpdate.IsZero() || lastNextUpdate != cert.OCSP.NextUpdate { + log.Printf("[INFO] Advancing OCSP staple for %v from %s to %s", + cert.Names, lastNextUpdate, cert.OCSP.NextUpdate) + for _, n := range cert.Names { + updated[n] = ocspUpdate{rawBytes: cert.Certificate.OCSPStaple, parsed: cert.OCSP} } } } @@ -161,7 +195,7 @@ func updatePreloadedOCSPStaples() { certCacheMu.Lock() for name, update := range updated { cert := certCache[name] - cert.OCSP = update.parsedResponse + cert.OCSP = update.parsed cert.Certificate.OCSPStaple = update.rawBytes certCache[name] = cert } diff --git a/caddy/https/setup.go b/caddy/https/setup.go index ebf46d244..b5b53454f 100644 --- a/caddy/https/setup.go +++ b/caddy/https/setup.go @@ -20,12 +20,12 @@ import ( // are specified by the user in the config file. All the automatic HTTPS // stuff comes later outside of this function. func Setup(c *setup.Controller) (middleware.Middleware, error) { - if c.Scheme == "http" { + if c.Port == "80" || c.Scheme == "http" { c.TLS.Enabled = false log.Printf("[WARNING] TLS disabled for %s://%s.", c.Scheme, c.Address()) - } else { - c.TLS.Enabled = true + return nil, nil } + c.TLS.Enabled = true for c.Next() { var certificateFile, keyFile, loadDir, maxCerts string @@ -38,6 +38,7 @@ func Setup(c *setup.Controller) (middleware.Middleware, error) { // user can force-disable managed TLS this way if c.TLS.LetsEncryptEmail == "off" { c.TLS.Enabled = false + return nil, nil } case 2: certificateFile = args[0] @@ -120,78 +121,8 @@ func Setup(c *setup.Controller) (middleware.Middleware, error) { } // load a directory of certificates, if specified - // modeled after haproxy: https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#5.1-crt if loadDir != "" { - err := filepath.Walk(loadDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - log.Printf("[WARNING] Unable to traverse into %s; skipping", path) - return nil - } - if info.IsDir() { - return nil - } - if strings.HasSuffix(strings.ToLower(info.Name()), ".pem") { - certBuilder, keyBuilder := new(bytes.Buffer), new(bytes.Buffer) - var foundKey bool - - bundle, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - for { - // Decode next block so we can see what type it is - var derBlock *pem.Block - derBlock, bundle = pem.Decode(bundle) - if derBlock == nil { - break - } - - if derBlock.Type == "CERTIFICATE" { - // Re-encode certificate as PEM, appending to certificate chain - pem.Encode(certBuilder, derBlock) - } else if derBlock.Type == "EC PARAMETERS" { - // EC keys are composed of two blocks: parameters and key - // (parameter block should come first) - if !foundKey { - // Encode parameters - pem.Encode(keyBuilder, derBlock) - - // Key must immediately follow - derBlock, bundle = pem.Decode(bundle) - if derBlock == nil || derBlock.Type != "EC PRIVATE KEY" { - return c.Errf("%s: expected elliptic private key to immediately follow EC parameters", path) - } - pem.Encode(keyBuilder, derBlock) - foundKey = true - } - } else if derBlock.Type == "PRIVATE KEY" || strings.HasSuffix(derBlock.Type, " PRIVATE KEY") { - // RSA key - if !foundKey { - pem.Encode(keyBuilder, derBlock) - foundKey = true - } - } else { - return c.Errf("%s: unrecognized PEM block type: %s", path, derBlock.Type) - } - } - - certPEMBytes, keyPEMBytes := certBuilder.Bytes(), keyBuilder.Bytes() - if len(certPEMBytes) == 0 { - return c.Errf("%s: failed to parse PEM data", path) - } - if len(keyPEMBytes) == 0 { - return c.Errf("%s: no private key block found", path) - } - - err = cacheUnmanagedCertificatePEMBytes(certPEMBytes, keyPEMBytes) - if err != nil { - return c.Errf("%s: failed to load cert and key for %s: %v", path, c.Host, err) - } - log.Printf("[INFO] Successfully loaded TLS assets from %s", path) - } - return nil - }) + err := loadCertsInDir(c, loadDir) if err != nil { return nil, err } @@ -203,6 +134,86 @@ func Setup(c *setup.Controller) (middleware.Middleware, error) { return nil, nil } +// loadCertsInDir loads all the certificates/keys in dir, as long as +// the file ends with .pem. This method of loading certificates is +// modeled after haproxy, which expects the certificate and key to +// be bundled into the same file: +// https://cbonte.github.io/haproxy-dconv/configuration-1.5.html#5.1-crt +// +// This function may write to the log as it walks the directory tree. +func loadCertsInDir(c *setup.Controller, dir string) error { + return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + log.Printf("[WARNING] Unable to traverse into %s; skipping", path) + return nil + } + if info.IsDir() { + return nil + } + if strings.HasSuffix(strings.ToLower(info.Name()), ".pem") { + certBuilder, keyBuilder := new(bytes.Buffer), new(bytes.Buffer) + var foundKey bool // use only the first key in the file + + bundle, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + for { + // Decode next block so we can see what type it is + var derBlock *pem.Block + derBlock, bundle = pem.Decode(bundle) + if derBlock == nil { + break + } + + if derBlock.Type == "CERTIFICATE" { + // Re-encode certificate as PEM, appending to certificate chain + pem.Encode(certBuilder, derBlock) + } else if derBlock.Type == "EC PARAMETERS" { + // EC keys generated from openssl can be composed of two blocks: + // parameters and key (parameter block should come first) + if !foundKey { + // Encode parameters + pem.Encode(keyBuilder, derBlock) + + // Key must immediately follow + derBlock, bundle = pem.Decode(bundle) + if derBlock == nil || derBlock.Type != "EC PRIVATE KEY" { + return c.Errf("%s: expected elliptic private key to immediately follow EC parameters", path) + } + pem.Encode(keyBuilder, derBlock) + foundKey = true + } + } else if derBlock.Type == "PRIVATE KEY" || strings.HasSuffix(derBlock.Type, " PRIVATE KEY") { + // RSA key + if !foundKey { + pem.Encode(keyBuilder, derBlock) + foundKey = true + } + } else { + return c.Errf("%s: unrecognized PEM block type: %s", path, derBlock.Type) + } + } + + certPEMBytes, keyPEMBytes := certBuilder.Bytes(), keyBuilder.Bytes() + if len(certPEMBytes) == 0 { + return c.Errf("%s: failed to parse PEM data", path) + } + if len(keyPEMBytes) == 0 { + return c.Errf("%s: no private key block found", path) + } + + err = cacheUnmanagedCertificatePEMBytes(certPEMBytes, keyPEMBytes) + if err != nil { + return c.Errf("%s: failed to load cert and key for %s: %v", path, c.Host, err) + } + log.Printf("[INFO] Successfully loaded TLS assets from %s", path) + } + return nil + }) +} + // setDefaultTLSParams sets the default TLS cipher suites, protocol versions, // and server preferences of a server.Config if they were not previously set // (it does not overwrite; only fills in missing values). It will also set the @@ -231,7 +242,7 @@ func setDefaultTLSParams(c *server.Config) { // Default TLS port is 443; only use if port is not manually specified, // TLS is enabled, and the host is not localhost - if c.Port == "" && c.TLS.Enabled && !c.TLS.Manual && c.Host != "localhost" { + if c.Port == "" && c.TLS.Enabled && (!c.TLS.Manual || c.TLS.OnDemand) && c.Host != "localhost" { c.Port = "443" } } diff --git a/server/config.go b/server/config.go index 9acdac7f5..332f45750 100644 --- a/server/config.go +++ b/server/config.go @@ -68,7 +68,7 @@ type TLSConfig struct { Enabled bool // will be set to true if TLS is enabled LetsEncryptEmail string Manual bool // will be set to true if user provides own certs and keys - Managed bool // will be set to true if config qualifies for automatic/managed HTTPS + Managed bool // will be set to true if config qualifies for implicit automatic/managed HTTPS OnDemand bool // will be set to true if user enables on-demand TLS (obtain certs during handshakes) Ciphers []uint16 ProtocolMinVersion uint16 diff --git a/server/server.go b/server/server.go index c921566c5..963692fe7 100644 --- a/server/server.go +++ b/server/server.go @@ -63,15 +63,7 @@ func New(addr string, configs []Config, gracefulTimeout time.Duration) (*Server, var useTLS, useOnDemandTLS bool if len(configs) > 0 { useTLS = configs[0].TLS.Enabled - if useTLS { - host, _, err := net.SplitHostPort(addr) - if err != nil { - host = addr - } - if host == "" && configs[0].TLS.OnDemand { - useOnDemandTLS = true - } - } + useOnDemandTLS = configs[0].TLS.OnDemand } s := &Server{ From f25ae8230f2d52f058b0019b07b162867b2c26ca Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 17 Feb 2016 16:08:25 -0700 Subject: [PATCH 26/52] Move to Go 1.6 and set CGO_ENABLED=0 in tests --- .travis.yml | 8 +++++--- README.md | 2 +- appveyor.yml | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 92bbffe59..6a2da63db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,14 @@ language: go go: - - 1.4.3 - - 1.5.3 + - 1.6 - tip +env: +- CGO_ENABLED=0 + install: - - go get -d ./... + - go get -t ./... - go get golang.org/x/tools/cmd/vet script: diff --git a/README.md b/README.md index 6aa9510a1..05723869c 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ You may also be interested in the [developer guide] ## Running from Source -Note: You will need **[Go 1.4](https://golang.org/dl/)** or a later version. +Note: You will need **[Go 1.6](https://golang.org/dl/)** or newer. 1. `$ go get github.com/mholt/caddy` 2. `cd` into your website's directory diff --git a/appveyor.yml b/appveyor.yml index eddfcaa7f..a486bc24d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,13 +6,14 @@ clone_folder: c:\gopath\src\github.com\mholt\caddy environment: GOPATH: c:\gopath + CGO_ENABLED: 0 install: - go get golang.org/x/tools/cmd/vet - echo %GOPATH% - go version - go env - - go get -d ./... + - go get -t ./... build_script: - go vet ./... From 1ef7f3c4b1c4f11d04888efb84e98a880b7d5c6a Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 17 Feb 2016 18:11:03 -0700 Subject: [PATCH 27/52] Remove path scoping for middleware slice It was implemented for almost a year but we'll probably never use it, especially since we'll match more than the path in the future. --- caddy/config.go | 8 ++------ caddy/https/https.go | 10 ++++------ caddy/https/https_test.go | 6 +++--- server/config.go | 4 ++-- server/virtualhost.go | 8 +------- 5 files changed, 12 insertions(+), 24 deletions(-) diff --git a/caddy/config.go b/caddy/config.go index 15420e315..c8ea6b4da 100644 --- a/caddy/config.go +++ b/caddy/config.go @@ -11,7 +11,6 @@ import ( "github.com/mholt/caddy/caddy/https" "github.com/mholt/caddy/caddy/parse" "github.com/mholt/caddy/caddy/setup" - "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/server" ) @@ -55,7 +54,6 @@ func loadConfigsUpToIncludingTLS(filename string, input io.Reader) ([]server.Con Port: addr.Port, Scheme: addr.Scheme, Root: Root, - Middleware: make(map[string][]middleware.Middleware), ConfigFile: filename, AppName: AppName, AppVersion: AppVersion, @@ -89,8 +87,7 @@ func loadConfigsUpToIncludingTLS(filename string, input io.Reader) ([]server.Con return nil, nil, lastDirectiveIndex, err } if midware != nil { - // TODO: For now, we only support the default path scope / - config.Middleware["/"] = append(config.Middleware["/"], midware) + config.Middleware = append(config.Middleware, midware) } storages[dir.name] = controller.ServerBlockStorage // persist for this server block } @@ -171,8 +168,7 @@ func loadConfigs(filename string, input io.Reader) ([]server.Config, error) { return nil, err } if midware != nil { - // TODO: For now, we only support the default path scope / - configs[configIndex].Middleware["/"] = append(configs[configIndex].Middleware["/"], midware) + configs[configIndex].Middleware = append(configs[configIndex].Middleware, midware) } storages[dir.name] = controller.ServerBlockStorage // persist for this server block } diff --git a/caddy/https/https.go b/caddy/https/https.go index 4526a31cd..50ed53d62 100644 --- a/caddy/https/https.go +++ b/caddy/https/https.go @@ -332,12 +332,10 @@ func redirPlaintextHost(cfg server.Config) server.Config { } return server.Config{ - Host: cfg.Host, - BindHost: cfg.BindHost, - Port: "80", - Middleware: map[string][]middleware.Middleware{ - "/": {redirMidware}, - }, + Host: cfg.Host, + BindHost: cfg.BindHost, + Port: "80", + Middleware: []middleware.Middleware{redirMidware}, } } diff --git a/caddy/https/https_test.go b/caddy/https/https_test.go index 199c6266b..e06af138b 100644 --- a/caddy/https/https_test.go +++ b/caddy/https/https_test.go @@ -87,11 +87,11 @@ func TestRedirPlaintextHost(t *testing.T) { } // Make sure redirect handler is set up properly - if cfg.Middleware == nil || len(cfg.Middleware["/"]) != 1 { + if cfg.Middleware == nil || len(cfg.Middleware) != 1 { t.Fatalf("Redir config middleware not set up properly; got: %#v", cfg.Middleware) } - handler, ok := cfg.Middleware["/"][0](nil).(redirect.Redirect) + handler, ok := cfg.Middleware[0](nil).(redirect.Redirect) if !ok { t.Fatalf("Expected a redirect.Redirect middleware, but got: %#v", handler) } @@ -116,7 +116,7 @@ func TestRedirPlaintextHost(t *testing.T) { // browsers can infer a default port from scheme, so make sure the port // doesn't get added in explicitly for default ports like 443 for https. cfg = redirPlaintextHost(server.Config{Host: "example.com", Port: "443"}) - handler, ok = cfg.Middleware["/"][0](nil).(redirect.Redirect) + handler, ok = cfg.Middleware[0](nil).(redirect.Redirect) if actual, expected := handler.Rules[0].To, "https://{host}{uri}"; actual != expected { t.Errorf("(Default Port) Expected redirect rule to be to URL '%s' but is actually to '%s'", expected, actual) } diff --git a/server/config.go b/server/config.go index 332f45750..1f4acdb6c 100644 --- a/server/config.go +++ b/server/config.go @@ -26,8 +26,8 @@ type Config struct { // HTTPS configuration TLS TLSConfig - // Middleware stack; map of path scope to middleware -- TODO: Support path scope? - Middleware map[string][]middleware.Middleware + // Middleware stack + Middleware []middleware.Middleware // Startup is a list of functions (or methods) to execute at // server startup and restart; these are executed before any diff --git a/server/virtualhost.go b/server/virtualhost.go index b0d157971..0f44cc68c 100644 --- a/server/virtualhost.go +++ b/server/virtualhost.go @@ -21,13 +21,7 @@ type virtualHost struct { // ListenAndServe begins. func (vh *virtualHost) buildStack() error { vh.fileServer = middleware.FileServer(http.Dir(vh.config.Root), []string{vh.config.ConfigFile}) - - // TODO: We only compile middleware for the "/" scope. - // Partial support for multiple location contexts already - // exists at the parser and config levels, but until full - // support is implemented, this is all we do right here. - vh.compile(vh.config.Middleware["/"]) - + vh.compile(vh.config.Middleware) return nil } From d05f89294ee28417a6d7734ce79e1ee02d603561 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 18 Feb 2016 20:33:15 -0700 Subject: [PATCH 28/52] https: Minor refactoring and some new tests --- caddy/https/certificates.go | 40 +++++++++++----------- caddy/https/certificates_test.go | 59 ++++++++++++++++++++++++++++++++ caddy/https/handshake.go | 21 +++++++----- caddy/https/handshake_test.go | 54 +++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 29 deletions(-) create mode 100644 caddy/https/certificates_test.go create mode 100644 caddy/https/handshake_test.go diff --git a/caddy/https/certificates.go b/caddy/https/certificates.go index b123d4c32..0dc3db523 100644 --- a/caddy/https/certificates.go +++ b/caddy/https/certificates.go @@ -50,23 +50,21 @@ type Certificate struct { OCSP *ocsp.Response } -// getCertificate gets a certificate from the in-memory cache that -// matches name (a certificate name). Note that if name does not have -// an exact match, it will be checked against names of the form -// '*.example.com' (wildcard certificates) according to RFC 6125. -// -// If cert was found by matching name, matched will be returned true. -// If no match is found, the default certificate will be returned and -// matched will be returned as false. (The default certificate is the -// first one that entered the cache.) If the cache is empty (or there -// is no default certificate for some reason), matched will still be -// false, but cert.Certificate will be nil. +// getCertificate gets a certificate that matches name (a server name) +// from the in-memory cache. If there is no exact match for name, it +// will be checked against names of the form '*.example.com' (wildcard +// certificates) according to RFC 6125. If a match is found, matched will +// be true. If no matches are found, matched will be false and a default +// certificate will be returned with defaulted set to true. If no default +// certificate is set, defaulted will be set to false. // // The logic in this function is adapted from the Go standard library, // which is by the Go Authors. // // This function is safe for concurrent use. -func getCertificate(name string) (cert Certificate, matched bool) { +func getCertificate(name string) (cert Certificate, matched, defaulted bool) { + var ok bool + // Not going to trim trailing dots here since RFC 3546 says, // "The hostname is represented ... without a trailing dot." // Just normalize to lowercase. @@ -76,8 +74,9 @@ func getCertificate(name string) (cert Certificate, matched bool) { defer certCacheMu.RUnlock() // exact match? great, let's use it - if cert, ok := certCache[name]; ok { - return cert, true + if cert, ok = certCache[name]; ok { + matched = true + return } // try replacing labels in the name with wildcards until we get a match @@ -85,14 +84,15 @@ func getCertificate(name string) (cert Certificate, matched bool) { for i := range labels { labels[i] = "*" candidate := strings.Join(labels, ".") - if cert, ok := certCache[candidate]; ok { - return cert, true + if cert, ok = certCache[candidate]; ok { + matched = true + return } } - // if nothing matches, return the default certificate - cert = certCache[""] - return cert, false + // if nothing matches, use the default certificate or bust + cert, defaulted = certCache[""] + return } // cacheManagedCertificate loads the certificate for domain into the @@ -214,8 +214,8 @@ func cacheCertificate(cert Certificate) { certCacheMu.Lock() if _, ok := certCache[""]; !ok { // use as default - certCache[""] = cert cert.Names = append(cert.Names, "") + certCache[""] = cert } for len(certCache)+len(cert.Names) > 10000 { // for simplicity, just remove random elements diff --git a/caddy/https/certificates_test.go b/caddy/https/certificates_test.go new file mode 100644 index 000000000..dbfb4efc1 --- /dev/null +++ b/caddy/https/certificates_test.go @@ -0,0 +1,59 @@ +package https + +import "testing" + +func TestUnexportedGetCertificate(t *testing.T) { + defer func() { certCache = make(map[string]Certificate) }() + + // When cache is empty + if _, matched, defaulted := getCertificate("example.com"); matched || defaulted { + t.Errorf("Got a certificate when cache was empty; matched=%v, defaulted=%v", matched, defaulted) + } + + // When cache has one certificate in it (also is default) + defaultCert := Certificate{Names: []string{"example.com", ""}} + certCache[""] = defaultCert + certCache["example.com"] = defaultCert + if cert, matched, defaulted := getCertificate("Example.com"); !matched || defaulted || cert.Names[0] != "example.com" { + t.Errorf("Didn't get a cert for 'Example.com' or got the wrong one: %v, matched=%v, defaulted=%v", cert, matched, defaulted) + } + if cert, matched, defaulted := getCertificate(""); !matched || defaulted || cert.Names[0] != "example.com" { + t.Errorf("Didn't get a cert for '' or got the wrong one: %v, matched=%v, defaulted=%v", cert, matched, defaulted) + } + + // When retrieving wildcard certificate + certCache["*.example.com"] = Certificate{Names: []string{"*.example.com"}} + if cert, matched, defaulted := getCertificate("sub.example.com"); !matched || defaulted || cert.Names[0] != "*.example.com" { + t.Errorf("Didn't get wildcard cert for 'sub.example.com' or got the wrong one: %v, matched=%v, defaulted=%v", cert, matched, defaulted) + } + + // When no certificate matches, the default is returned + if cert, matched, defaulted := getCertificate("nomatch"); matched || !defaulted { + t.Errorf("Expected matched=false, defaulted=true; but got matched=%v, defaulted=%v (cert: %v)", matched, defaulted, cert) + } else if cert.Names[0] != "example.com" { + t.Errorf("Expected default cert, got: %v", cert) + } +} + +func TestCacheCertificate(t *testing.T) { + defer func() { certCache = make(map[string]Certificate) }() + + cacheCertificate(Certificate{Names: []string{"example.com", "sub.example.com"}}) + if _, ok := certCache["example.com"]; !ok { + t.Error("Expected first cert to be cached by key 'example.com', but it wasn't") + } + if _, ok := certCache["sub.example.com"]; !ok { + t.Error("Expected first cert to be cached by key 'sub.exmaple.com', but it wasn't") + } + if cert, ok := certCache[""]; !ok || cert.Names[2] != "" { + t.Error("Expected first cert to be cached additionally as the default certificate with empty name added, but it wasn't") + } + + cacheCertificate(Certificate{Names: []string{"example2.com"}}) + if _, ok := certCache["example2.com"]; !ok { + t.Error("Expected second cert to be cached by key 'exmaple2.com', but it wasn't") + } + if cert, ok := certCache[""]; ok && cert.Names[0] == "example2.com" { + t.Error("Expected second cert to NOT be cached as default, but it was") + } +} diff --git a/caddy/https/handshake.go b/caddy/https/handshake.go index 38f9afb55..fc6ef809e 100644 --- a/caddy/https/handshake.go +++ b/caddy/https/handshake.go @@ -39,31 +39,30 @@ func GetOrObtainCertificate(clientHello *tls.ClientHelloInfo) (*tls.Certificate, } // getCertDuringHandshake will get a certificate for name. It first tries -// the in-memory cache. If no certificate for name is in the cach and if +// the in-memory cache. If no certificate for name is in the cache and if // loadIfNecessary == true, it goes to disk to load it into the cache and // serve it. If it's not on disk and if obtainIfNecessary == true, the // certificate will be obtained from the CA, cached, and served. If // obtainIfNecessary is true, then loadIfNecessary must also be set to true. +// An error will be returned if and only if no certificate is available. // // This function is safe for concurrent use. func getCertDuringHandshake(name string, loadIfNecessary, obtainIfNecessary bool) (Certificate, error) { // First check our in-memory cache to see if we've already loaded it - cert, ok := getCertificate(name) - if ok { + cert, matched, defaulted := getCertificate(name) + if matched { return cert, nil } if loadIfNecessary { - var err error - // Then check to see if we have one on disk - cert, err = cacheManagedCertificate(name, true) + loadedCert, err := cacheManagedCertificate(name, true) if err == nil { - cert, err = handshakeMaintenance(name, cert) + loadedCert, err = handshakeMaintenance(name, loadedCert) if err != nil { log.Printf("[ERROR] Maintaining newly-loaded certificate for %s: %v", name, err) } - return cert, nil + return loadedCert, nil } if obtainIfNecessary { @@ -87,7 +86,11 @@ func getCertDuringHandshake(name string, loadIfNecessary, obtainIfNecessary bool } } - return Certificate{}, nil + if defaulted { + return cert, nil + } + + return Certificate{}, errors.New("no certificate for " + name) } // checkLimitsForObtainingNewCerts checks to see if name can be issued right diff --git a/caddy/https/handshake_test.go b/caddy/https/handshake_test.go new file mode 100644 index 000000000..cf70eb17d --- /dev/null +++ b/caddy/https/handshake_test.go @@ -0,0 +1,54 @@ +package https + +import ( + "crypto/tls" + "crypto/x509" + "testing" +) + +func TestGetCertificate(t *testing.T) { + defer func() { certCache = make(map[string]Certificate) }() + + hello := &tls.ClientHelloInfo{ServerName: "example.com"} + helloSub := &tls.ClientHelloInfo{ServerName: "sub.example.com"} + helloNoSNI := &tls.ClientHelloInfo{} + helloNoMatch := &tls.ClientHelloInfo{ServerName: "nomatch"} + + // When cache is empty + if cert, err := GetCertificate(hello); err == nil { + t.Errorf("GetCertificate should return error when cache is empty, got: %v", cert) + } + if cert, err := GetCertificate(helloNoSNI); err == nil { + t.Errorf("GetCertificate should return error when cache is empty even if server name is blank, got: %v", cert) + } + + // When cache has one certificate in it (also is default) + defaultCert := Certificate{Names: []string{"example.com", ""}, Certificate: tls.Certificate{Leaf: &x509.Certificate{DNSNames: []string{"example.com"}}}} + certCache[""] = defaultCert + certCache["example.com"] = defaultCert + if cert, err := GetCertificate(hello); err != nil { + t.Errorf("Got an error but shouldn't have, when cert exists in cache: %v", err) + } else if cert.Leaf.DNSNames[0] != "example.com" { + t.Errorf("Got wrong certificate with exact match; expected 'example.com', got: %v", cert) + } + if cert, err := GetCertificate(helloNoSNI); err != nil { + t.Errorf("Got an error with no SNI but shouldn't have, when cert exists in cache: %v", err) + } else if cert.Leaf.DNSNames[0] != "example.com" { + t.Errorf("Got wrong certificate for no SNI; expected 'example.com' as default, got: %v", cert) + } + + // When retrieving wildcard certificate + certCache["*.example.com"] = Certificate{Names: []string{"*.example.com"}, Certificate: tls.Certificate{Leaf: &x509.Certificate{DNSNames: []string{"*.example.com"}}}} + if cert, err := GetCertificate(helloSub); err != nil { + t.Errorf("Didn't get wildcard cert, got: cert=%v, err=%v ", cert, err) + } else if cert.Leaf.DNSNames[0] != "*.example.com" { + t.Errorf("Got wrong certificate, expected wildcard: %v", cert) + } + + // When no certificate matches, the default is returned + if cert, err := GetCertificate(helloNoMatch); err != nil { + t.Errorf("Expected default certificate with no error when no matches, got err: %v", err) + } else if cert.Leaf.DNSNames[0] != "example.com" { + t.Errorf("Expected default cert with no matches, got: %v", cert) + } +} From ecf913e58db4bf5c6fafe5d883c3840aab0b35b4 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 18 Feb 2016 20:57:38 -0700 Subject: [PATCH 29/52] Update change log --- dist/CHANGES.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/dist/CHANGES.txt b/dist/CHANGES.txt index 21fa64296..00b837f04 100644 --- a/dist/CHANGES.txt +++ b/dist/CHANGES.txt @@ -1,5 +1,19 @@ CHANGES + +- On-demand TLS can obtain certificates during handshake +- Built with Go 1.6 +- Process log (-log) is rotated when it gets large +- fastcgi: Allow scheme prefix before address +- markdown: Support for definition lists +- proxy: Allow proxy to insecure HTTPS backends +- proxy: Support proxy to unix socket +- templates: New .Markdown action to interpret included file as Markdown +- tls: max_certs setting to set hard limit on-demand TLS +- tls: load certificates from directory +- Multiple bug fixes and internal changes + + 0.8.1 (January 12, 2016) - Improved OCSP stapling - Better graceful reload when new hosts need certificates from Let's Encrypt @@ -14,6 +28,7 @@ CHANGES - tls: No longer allow HTTPS over port 80 - Dozens of bug fixes, improvements, and more tests across the board + 0.8.0 (December 4, 2015) - HTTPS by default via Let's Encrypt (certs & keys are fully managed) - Graceful restarts (on POSIX-compliant systems) From 5f2670fddefae16375979e1dd5e84dc2124023a8 Mon Sep 17 00:00:00 2001 From: Jason Chu Date: Sat, 20 Feb 2016 00:42:17 +0800 Subject: [PATCH 30/52] Fix missing Content-Type for certain errors And corrected an error in a copy and pasted comment --- caddy/setup/errors.go | 2 +- middleware/errors/errors.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/caddy/setup/errors.go b/caddy/setup/errors.go index 24e2b0bb8..b4c0ab697 100644 --- a/caddy/setup/errors.go +++ b/caddy/setup/errors.go @@ -12,7 +12,7 @@ import ( "github.com/mholt/caddy/middleware/errors" ) -// Errors configures a new gzip middleware instance. +// Errors configures a new errors middleware instance. func Errors(c *Controller) (middleware.Middleware, error) { handler, err := errorsParse(c) if err != nil { diff --git a/middleware/errors/errors.go b/middleware/errors/errors.go index e9eef90e7..33a152692 100644 --- a/middleware/errors/errors.go +++ b/middleware/errors/errors.go @@ -34,6 +34,7 @@ func (h ErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, er if h.Debug { // Write error to response instead of to log + w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(status) fmt.Fprintln(w, errMsg) return 0, err // returning < 400 signals that a response has been written @@ -124,6 +125,7 @@ func (h ErrorHandler) recovery(w http.ResponseWriter, r *http.Request) { // Write error and stack trace to the response rather than to a log var stackBuf [4096]byte stack := stackBuf[:runtime.Stack(stackBuf[:], false)] + w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, "%s\n\n%s", panicMsg, stack) } else { From 09a7af8cae2537556e24baa657bfe873b76d92ff Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Fri, 19 Feb 2016 10:33:01 -0700 Subject: [PATCH 31/52] https: Wait as long as possible to create ACME client at startup (fixes #617) --- caddy/https/https.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/caddy/https/https.go b/caddy/https/https.go index 50ed53d62..824de541b 100644 --- a/caddy/https/https.go +++ b/caddy/https/https.go @@ -117,16 +117,26 @@ func ObtainCerts(configs []server.Config, allowPrompts, proxyACME bool) error { groupedConfigs := groupConfigsByEmail(configs, allowPrompts) for email, group := range groupedConfigs { - client, err := NewACMEClient(email, allowPrompts) - if err != nil { - return errors.New("error creating client: " + err.Error()) - } + // Wait as long as we can before creating the client, because it + // may not be needed, for example, if we already have what we + // need on disk. Creating a client involves the network and + // potentially prompting the user, etc., so only do if necessary. + var client *ACMEClient for _, cfg := range group { if cfg.Host == "" || existingCertAndKey(cfg.Host) { continue } + // Now we definitely do need a client + if client == nil { + var err error + client, err = NewACMEClient(email, allowPrompts) + if err != nil { + return errors.New("error creating client: " + err.Error()) + } + } + // c.Configure assumes that allowPrompts == !proxyACME, // but that's not always true. For example, a restart where // the user isn't present and we're not listening on port 80. From f7b5187bf323c7a6e3cb71a0a124c09e0f6158d3 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Fri, 19 Feb 2016 13:34:54 -0700 Subject: [PATCH 32/52] server: Add "Referer" to log entry when host not found --- server/server.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 963692fe7..3a336f3b0 100644 --- a/server/server.go +++ b/server/server.go @@ -331,7 +331,8 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) fmt.Fprintf(w, "No such host at %s", s.Server.Addr) - log.Printf("[INFO] %s - No such host at %s (requested by %s)", host, s.Server.Addr, remoteHost) + log.Printf("[INFO] %s - No such host at %s (Remote: %s, Referer: %s)", + host, s.Server.Addr, remoteHost, r.Header.Get("Referer")) } } From 09b7ce6c93305adca6da4d3962129798c4e1af5b Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Fri, 19 Feb 2016 18:07:48 -0700 Subject: [PATCH 33/52] Try to get Go 1.6 on appveyor --- appveyor.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a486bc24d..b370a1135 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,12 +9,18 @@ environment: CGO_ENABLED: 0 install: - - go get golang.org/x/tools/cmd/vet - - echo %GOPATH% + - rmdir c:\go /s /q + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.windows-amd64.zip + - 7z x go1.6.windows-amd64.zip -y -oC:\ > NUL - go version - go env + - go get golang.org/x/tools/cmd/vet - go get -t ./... -build_script: +build: off + +test_script: - go vet ./... - - go test ./... \ No newline at end of file + - go test ./... + +deploy: off From bec130a5638c72298b2d0104a47bdff64920736f Mon Sep 17 00:00:00 2001 From: Benoit Benedetti Date: Sat, 20 Feb 2016 22:52:42 +0100 Subject: [PATCH 34/52] Recorder: Exporting ResponseRecorder #614 --- middleware/recorder.go | 22 ++++++++++++++++------ middleware/replacer.go | 2 +- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/middleware/recorder.go b/middleware/recorder.go index 0477cce88..481b60dff 100644 --- a/middleware/recorder.go +++ b/middleware/recorder.go @@ -14,7 +14,7 @@ import ( // to be written, however, in which case 200 must be assumed. // It is best to have the constructor initialize this type // with that default status code. -type responseRecorder struct { +type ResponseRecorder struct { http.ResponseWriter status int size int @@ -27,8 +27,8 @@ type responseRecorder struct { // Because a status is not set unless WriteHeader is called // explicitly, this constructor initializes with a status code // of 200 to cover the default case. -func NewResponseRecorder(w http.ResponseWriter) *responseRecorder { - return &responseRecorder{ +func NewResponseRecorder(w http.ResponseWriter) *ResponseRecorder { + return &ResponseRecorder{ ResponseWriter: w, status: http.StatusOK, start: time.Now(), @@ -37,14 +37,14 @@ func NewResponseRecorder(w http.ResponseWriter) *responseRecorder { // WriteHeader records the status code and calls the // underlying ResponseWriter's WriteHeader method. -func (r *responseRecorder) WriteHeader(status int) { +func (r *ResponseRecorder) WriteHeader(status int) { r.status = status r.ResponseWriter.WriteHeader(status) } // Write is a wrapper that records the size of the body // that gets written. -func (r *responseRecorder) Write(buf []byte) (int, error) { +func (r *ResponseRecorder) Write(buf []byte) (int, error) { n, err := r.ResponseWriter.Write(buf) if err == nil { r.size += n @@ -52,9 +52,19 @@ func (r *responseRecorder) Write(buf []byte) (int, error) { return n, err } +// Size is a Getter to size property +func (r *ResponseRecorder) Size() int { + return r.size +} + +// Status is a Getter to status property +func (r *ResponseRecorder) Status() int { + return r.status +} + // Hijacker is a wrapper of http.Hijacker underearth if any, // otherwise it just returns an error. -func (r *responseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { +func (r *ResponseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { if hj, ok := r.ResponseWriter.(http.Hijacker); ok { return hj.Hijack() } diff --git a/middleware/replacer.go b/middleware/replacer.go index 8a3d202a2..d53fdc660 100644 --- a/middleware/replacer.go +++ b/middleware/replacer.go @@ -30,7 +30,7 @@ type replacer struct { // values into the replacer. rr may be nil if it is not // available. emptyValue should be the string that is used // in place of empty string (can still be empty string). -func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Replacer { +func NewReplacer(r *http.Request, rr *ResponseRecorder, emptyValue string) Replacer { rep := replacer{ replacements: map[string]string{ "{method}": r.Method, From c7674e2060c9c2a0dec658aacc3e240cf91ea86e Mon Sep 17 00:00:00 2001 From: Maxim Kupriianov Date: Mon, 22 Feb 2016 13:53:47 +0300 Subject: [PATCH 35/52] Implement .DocFlags directive and tests. It holds all the boolean-typed front matter values. --- middleware/markdown/markdown_test.go | 33 ++++++++++ middleware/markdown/metadata.go | 21 +++++-- middleware/markdown/metadata_test.go | 61 +++++++++++++++---- middleware/markdown/process.go | 14 +++-- .../markdown/testdata/docflags/template.txt | 4 ++ middleware/markdown/testdata/docflags/test.md | 4 ++ 6 files changed, 114 insertions(+), 23 deletions(-) create mode 100644 middleware/markdown/testdata/docflags/template.txt create mode 100644 middleware/markdown/testdata/docflags/test.md diff --git a/middleware/markdown/markdown_test.go b/middleware/markdown/markdown_test.go index fbbe845d7..bcc041c4d 100644 --- a/middleware/markdown/markdown_test.go +++ b/middleware/markdown/markdown_test.go @@ -32,6 +32,18 @@ func TestMarkdown(t *testing.T) { StaticDir: DefaultStaticDir, StaticFiles: make(map[string]string), }, + { + Renderer: blackfriday.HtmlRenderer(0, "", ""), + PathScope: "/docflags", + Extensions: []string{".md"}, + Styles: []string{}, + Scripts: []string{}, + Templates: map[string]string{ + DefaultTemplate: "testdata/docflags/template.txt", + }, + StaticDir: DefaultStaticDir, + StaticFiles: make(map[string]string), + }, { Renderer: blackfriday.HtmlRenderer(0, "", ""), PathScope: "/log", @@ -114,6 +126,26 @@ Welcome to A Caddy website! t.Fatalf("Expected body: %v got: %v", expectedBody, respBody) } + req, err = http.NewRequest("GET", "/docflags/test.md", nil) + if err != nil { + t.Fatalf("Could not create HTTP request: %v", err) + } + rec = httptest.NewRecorder() + + md.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("Wrong status, expected: %d and got %d", http.StatusOK, rec.Code) + } + respBody = rec.Body.String() + expectedBody = `Doc.var_string hello +Doc.var_bool +DocFlags.var_string +DocFlags.var_bool true` + + if !equalStrings(respBody, expectedBody) { + t.Fatalf("Expected body: %v got: %v", expectedBody, respBody) + } + req, err = http.NewRequest("GET", "/log/test.md", nil) if err != nil { t.Fatalf("Could not create HTTP request: %v", err) @@ -190,6 +222,7 @@ Welcome to title! expectedLinks := []string{ "/blog/test.md", + "/docflags/test.md", "/log/test.md", } diff --git a/middleware/markdown/metadata.go b/middleware/markdown/metadata.go index 07c00801b..9b5c416a8 100644 --- a/middleware/markdown/metadata.go +++ b/middleware/markdown/metadata.go @@ -23,6 +23,9 @@ type Metadata struct { // Variables to be used with Template Variables map[string]string + + // Flags to be used with Template + Flags map[string]bool } // load loads parsed values in parsedMap into Metadata @@ -40,8 +43,11 @@ func (m *Metadata) load(parsedMap map[string]interface{}) { } // store everything as a variable for key, val := range parsedMap { - if v, ok := val.(string); ok { + switch v := val.(type) { + case string: m.Variables[key] = v + case bool: + m.Flags[key] = v } } } @@ -219,11 +225,18 @@ func findParser(b []byte) MetadataParser { return nil } +func newMetadata() Metadata { + return Metadata{ + Variables: make(map[string]string), + Flags: make(map[string]bool), + } +} + // parsers returns all available parsers func parsers() []MetadataParser { return []MetadataParser{ - &JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, - &TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, - &YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, + &JSONMetadataParser{metadata: newMetadata()}, + &TOMLMetadataParser{metadata: newMetadata()}, + &YAMLMetadataParser{metadata: newMetadata()}, } } diff --git a/middleware/markdown/metadata_test.go b/middleware/markdown/metadata_test.go index 285caf972..a4b7565b5 100644 --- a/middleware/markdown/metadata_test.go +++ b/middleware/markdown/metadata_test.go @@ -18,11 +18,15 @@ var TOML = [5]string{` title = "A title" template = "default" name = "value" +positive = true +negative = false `, `+++ title = "A title" template = "default" name = "value" +positive = true +negative = false +++ Page content `, @@ -30,12 +34,16 @@ Page content title = "A title" template = "default" name = "value" +positive = true +negative = false `, `title = "A title" template = "default" [variables] name = "value"`, `+++ title = "A title" template = "default" name = "value" +positive = true +negative = false +++ `, } @@ -44,11 +52,15 @@ var YAML = [5]string{` title : A title template : default name : value +positive : true +negative : false `, `--- title : A title template : default name : value +positive : true +negative : false --- Page content `, @@ -57,11 +69,13 @@ title : A title template : default name : value `, - `title : A title template : default variables : name : value`, + `title : A title template : default variables : name : value : positive : true : negative : false`, `--- title : A title template : default name : value +positive : true +negative : false --- `, } @@ -69,12 +83,16 @@ name : value var JSON = [5]string{` "title" : "A title", "template" : "default", - "name" : "value" + "name" : "value", + "positive" : true, + "negative" : false `, `{ "title" : "A title", "template" : "default", - "name" : "value" + "name" : "value", + "positive" : true, + "negative" : false } Page content `, @@ -82,19 +100,25 @@ Page content { "title" : "A title", "template" : "default", - "name" : "value" + "name" : "value", + "positive" : true, + "negative" : false `, ` { "title" :: "A title", "template" : "default", - "name" : "value" + "name" : "value", + "positive" : true, + "negative" : false } `, `{ "title" : "A title", "template" : "default", - "name" : "value" + "name" : "value", + "positive" : true, + "negative" : false } `, } @@ -108,6 +132,10 @@ func TestParsers(t *testing.T) { "title": "A title", "template": "default", }, + Flags: map[string]bool{ + "positive": true, + "negative": false, + }, } compare := func(m Metadata) bool { if m.Title != expected.Title { @@ -121,7 +149,14 @@ func TestParsers(t *testing.T) { return false } } - return len(m.Variables) == len(expected.Variables) + for k, v := range m.Flags { + if v != expected.Flags[k] { + return false + } + } + varLenOK := len(m.Variables) == len(expected.Variables) + flagLenOK := len(m.Flags) == len(expected.Flags) + return varLenOK && flagLenOK } data := []struct { @@ -129,9 +164,9 @@ func TestParsers(t *testing.T) { testData [5]string name string }{ - {&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"}, - {&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, YAML, "yaml"}, - {&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, TOML, "toml"}, + {&JSONMetadataParser{metadata: newMetadata()}, JSON, "json"}, + {&YAMLMetadataParser{metadata: newMetadata()}, YAML, "yaml"}, + {&TOMLMetadataParser{metadata: newMetadata()}, TOML, "toml"}, } for _, v := range data { @@ -207,9 +242,9 @@ Mycket olika byggnader har man i de nordiska rikena: pyramidformiga, kilformiga, testData string name string }{ - {&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"}, - {&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, YAML, "yaml"}, - {&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, TOML, "toml"}, + {&JSONMetadataParser{metadata: newMetadata()}, JSON, "json"}, + {&YAMLMetadataParser{metadata: newMetadata()}, YAML, "yaml"}, + {&TOMLMetadataParser{metadata: newMetadata()}, TOML, "toml"}, } for _, v := range data { // metadata without identifiers diff --git a/middleware/markdown/process.go b/middleware/markdown/process.go index 807ae47c2..a0d1ae68c 100644 --- a/middleware/markdown/process.go +++ b/middleware/markdown/process.go @@ -23,14 +23,15 @@ const ( // Data represents a markdown document. type Data struct { middleware.Context - Doc map[string]string - Links []PageLink + Doc map[string]string + DocFlags map[string]bool + Links []PageLink } // Process processes the contents of a page in b. It parses the metadata // (if any) and uses the template (if found). func (md Markdown) Process(c *Config, requestPath string, b []byte, ctx middleware.Context) ([]byte, error) { - var metadata = Metadata{Variables: make(map[string]string)} + var metadata = newMetadata() var markdown []byte var err error @@ -100,9 +101,10 @@ func (md Markdown) processTemplate(c *Config, requestPath string, tmpl []byte, m return nil, err } mdData := Data{ - Context: ctx, - Doc: metadata.Variables, - Links: c.Links, + Context: ctx, + Doc: metadata.Variables, + DocFlags: metadata.Flags, + Links: c.Links, } c.RLock() diff --git a/middleware/markdown/testdata/docflags/template.txt b/middleware/markdown/testdata/docflags/template.txt new file mode 100644 index 000000000..2760d18d1 --- /dev/null +++ b/middleware/markdown/testdata/docflags/template.txt @@ -0,0 +1,4 @@ +Doc.var_string {{.Doc.var_string}} +Doc.var_bool {{.Doc.var_bool}} +DocFlags.var_string {{.DocFlags.var_string}} +DocFlags.var_bool {{.DocFlags.var_bool}} diff --git a/middleware/markdown/testdata/docflags/test.md b/middleware/markdown/testdata/docflags/test.md new file mode 100644 index 000000000..64ca7f78d --- /dev/null +++ b/middleware/markdown/testdata/docflags/test.md @@ -0,0 +1,4 @@ +--- +var_string: hello +var_bool: true +--- From 2ea6c95ac4845b67364c96bb16784ac77455c39a Mon Sep 17 00:00:00 2001 From: Nathan Probst Date: Mon, 22 Feb 2016 15:30:55 -0700 Subject: [PATCH 36/52] Allow rewrite status codes to be 2xx and 4xx. --- caddy/setup/rewrite.go | 4 ++-- caddy/setup/rewrite_test.go | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/caddy/setup/rewrite.go b/caddy/setup/rewrite.go index ab997d278..b270c93dd 100644 --- a/caddy/setup/rewrite.go +++ b/caddy/setup/rewrite.go @@ -80,8 +80,8 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { return nil, c.ArgErr() } status, _ = strconv.Atoi(c.Val()) - if status < 400 || status > 499 { - return nil, c.Err("status must be 4xx") + if status < 200 || (status > 299 && status < 400) || status > 499 { + return nil, c.Err("status must be 2xx or 4xx") } default: return nil, c.ArgErr() diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index 29bfe9975..1ce96aeec 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -137,6 +137,11 @@ func TestRewriteParse(t *testing.T) { }`, false, []rewrite.Rule{ &rewrite.ComplexRule{Base: "/", To: "/to", Ifs: []rewrite.If{{A: "{path}", Operator: "is", B: "a"}}}, }}, + {`rewrite { + status 500 + }`, true, []rewrite.Rule{ + &rewrite.ComplexRule{}, + }}, {`rewrite { status 400 }`, false, []rewrite.Rule{ @@ -153,6 +158,22 @@ func TestRewriteParse(t *testing.T) { }`, true, []rewrite.Rule{ &rewrite.ComplexRule{}, }}, + {`rewrite { + status 200 + }`, false, []rewrite.Rule{ + &rewrite.ComplexRule{Base: "/", Regexp: regexp.MustCompile(".*"), Status: 200}, + }}, + {`rewrite { + to /to + status 200 + }`, false, []rewrite.Rule{ + &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile(".*"), Status: 200}, + }}, + {`rewrite { + status 199 + }`, true, []rewrite.Rule{ + &rewrite.ComplexRule{}, + }}, {`rewrite { status 0 }`, true, []rewrite.Rule{ From a541eb7899dc57224a489115f62bcf5216ac84a8 Mon Sep 17 00:00:00 2001 From: elcore Date: Tue, 23 Feb 2016 01:43:40 +0100 Subject: [PATCH 37/52] Adding new cipher suites --- caddy/https/setup.go | 6 ++++++ caddy/https/setup_test.go | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/caddy/https/setup.go b/caddy/https/setup.go index b5b53454f..566bc94e6 100644 --- a/caddy/https/setup.go +++ b/caddy/https/setup.go @@ -268,6 +268,8 @@ var supportedProtocols = map[string]uint16{ // // This map, like any map, is NOT ORDERED. Do not range over this map. var supportedCiphersMap = map[string]uint16{ + "ECDHE-RSA-AES256-GCM-SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + "ECDHE-ECDSA-AES256-GCM-SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, "ECDHE-RSA-AES128-GCM-SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, "ECDHE-ECDSA-AES128-GCM-SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, "ECDHE-RSA-AES128-CBC-SHA": tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, @@ -287,6 +289,8 @@ var supportedCiphersMap = map[string]uint16{ // Note that TLS_FALLBACK_SCSV is not in this list since it is always // added manually. var supportedCiphers = []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, @@ -301,6 +305,8 @@ var supportedCiphers = []uint16{ // List of all the ciphers we want to use by default var defaultCiphers = []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, diff --git a/caddy/https/setup_test.go b/caddy/https/setup_test.go index 4ca57b823..047ccd57e 100644 --- a/caddy/https/setup_test.go +++ b/caddy/https/setup_test.go @@ -57,6 +57,8 @@ func TestSetupParseBasic(t *testing.T) { // Cipher checks expectedCiphers := []uint16{ tls.TLS_FALLBACK_SCSV, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, @@ -97,7 +99,7 @@ func TestSetupParseIncompleteParams(t *testing.T) { func TestSetupParseWithOptionalParams(t *testing.T) { params := `tls ` + certFile + ` ` + keyFile + ` { protocols ssl3.0 tls1.2 - ciphers RSA-3DES-EDE-CBC-SHA RSA-AES256-CBC-SHA ECDHE-RSA-AES128-GCM-SHA256 + ciphers RSA-AES256-CBC-SHA ECDHE-RSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 }` c := setup.NewTestController(params) From f4bb43781ccebce0ec69385c591680162c19cf01 Mon Sep 17 00:00:00 2001 From: Nathan Probst Date: Wed, 24 Feb 2016 10:28:06 -0700 Subject: [PATCH 38/52] Remove unneeded Regexp from tests. --- caddy/setup/rewrite_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index 1ce96aeec..d252ed904 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -145,13 +145,13 @@ func TestRewriteParse(t *testing.T) { {`rewrite { status 400 }`, false, []rewrite.Rule{ - &rewrite.ComplexRule{Base: "/", Regexp: regexp.MustCompile(".*"), Status: 400}, + &rewrite.ComplexRule{Base: "/", Status: 400}, }}, {`rewrite { to /to status 400 }`, false, []rewrite.Rule{ - &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile(".*"), Status: 400}, + &rewrite.ComplexRule{Base: "/", To: "/to", Status: 400}, }}, {`rewrite { status 399 @@ -161,13 +161,13 @@ func TestRewriteParse(t *testing.T) { {`rewrite { status 200 }`, false, []rewrite.Rule{ - &rewrite.ComplexRule{Base: "/", Regexp: regexp.MustCompile(".*"), Status: 200}, + &rewrite.ComplexRule{Base: "/", Status: 200}, }}, {`rewrite { to /to status 200 }`, false, []rewrite.Rule{ - &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile(".*"), Status: 200}, + &rewrite.ComplexRule{Base: "/", To: "/to", Status: 200}, }}, {`rewrite { status 199 From 05957b4965413bccac8a7e24dc8f5314f15a278b Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 24 Feb 2016 12:23:15 -0700 Subject: [PATCH 39/52] gzip: Implement http.Hijacker (fixes #635) --- middleware/gzip/gzip.go | 11 +++++++++++ middleware/recorder.go | 6 +++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index d7ed835d5..9d75b351e 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -3,10 +3,12 @@ package gzip import ( + "bufio" "compress/gzip" "fmt" "io" "io/ioutil" + "net" "net/http" "strings" @@ -130,3 +132,12 @@ func (w *gzipResponseWriter) Write(b []byte) (int, error) { n, err := w.Writer.Write(b) return n, err } + +// Hijack implements http.Hijacker. It simply wraps the underlying +// ResponseWriter's Hijack method if there is one, or returns an error. +func (w *gzipResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if hj, ok := w.ResponseWriter.(http.Hijacker); ok { + return hj.Hijack() + } + return nil, nil, fmt.Errorf("not a Hijacker") +} diff --git a/middleware/recorder.go b/middleware/recorder.go index 481b60dff..d3e65dc62 100644 --- a/middleware/recorder.go +++ b/middleware/recorder.go @@ -62,11 +62,11 @@ func (r *ResponseRecorder) Status() int { return r.status } -// Hijacker is a wrapper of http.Hijacker underearth if any, -// otherwise it just returns an error. +// Hijack implements http.Hijacker. It simply wraps the underlying +// ResponseWriter's Hijack method if there is one, or returns an error. func (r *ResponseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { if hj, ok := r.ResponseWriter.(http.Hijacker); ok { return hj.Hijack() } - return nil, nil, errors.New("I'm not a Hijacker") + return nil, nil, errors.New("not a Hijacker") } From ef5f9c771d54696687b35081436a3abb516014b1 Mon Sep 17 00:00:00 2001 From: Benoit Benedetti Date: Wed, 24 Feb 2016 19:35:21 +0100 Subject: [PATCH 40/52] FastCGI: Explicitly set Content-Length #626 --- middleware/fastcgi/fastcgi.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) mode change 100755 => 100644 middleware/fastcgi/fastcgi.go diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go old mode 100755 new mode 100644 index bddb04705..3d01c4162 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -4,6 +4,7 @@ package fastcgi import ( + "bytes" "errors" "io" "net/http" @@ -105,13 +106,21 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) return http.StatusBadGateway, err } + // Write the response body to a buffer + // To explicitly set Content-Length + // For FastCGI app that don't set it + var buf bytes.Buffer + io.Copy(&buf, resp.Body) + if r.Header.Get("Content-Length") == "" { + w.Header().Set("Content-Length", strconv.Itoa(buf.Len())) + } writeHeader(w, resp) // Write the response body // TODO: If this has an error, the response will already be // partly written. We should copy out of resp.Body into a buffer // first, then write it to the response... - _, err = io.Copy(w, resp.Body) + _, err = io.Copy(w, &buf) if err != nil { return http.StatusBadGateway, err } From 737c7c437204922822d94bf41d6f374ccb13b935 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 24 Feb 2016 16:41:45 -0700 Subject: [PATCH 41/52] fastcgi: Only perform extra copy if necessary; added tests --- middleware/fastcgi/fastcgi.go | 44 ++++++++++--------- middleware/fastcgi/fastcgi_test.go | 68 ++++++++++++++++++++++++------ 2 files changed, 79 insertions(+), 33 deletions(-) diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go index 3d01c4162..c4ca935e9 100644 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -72,7 +72,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) // Connect to FastCGI gateway network, address := rule.parseAddress() - fcgi, err := Dial(network, address) + fcgiBackend, err := Dial(network, address) if err != nil { return http.StatusBadGateway, err } @@ -81,19 +81,19 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) contentLength, _ := strconv.Atoi(r.Header.Get("Content-Length")) switch r.Method { case "HEAD": - resp, err = fcgi.Head(env) + resp, err = fcgiBackend.Head(env) case "GET": - resp, err = fcgi.Get(env) + resp, err = fcgiBackend.Get(env) case "OPTIONS": - resp, err = fcgi.Options(env) + resp, err = fcgiBackend.Options(env) case "POST": - resp, err = fcgi.Post(env, r.Header.Get("Content-Type"), r.Body, contentLength) + resp, err = fcgiBackend.Post(env, r.Header.Get("Content-Type"), r.Body, contentLength) case "PUT": - resp, err = fcgi.Put(env, r.Header.Get("Content-Type"), r.Body, contentLength) + resp, err = fcgiBackend.Put(env, r.Header.Get("Content-Type"), r.Body, contentLength) case "PATCH": - resp, err = fcgi.Patch(env, r.Header.Get("Content-Type"), r.Body, contentLength) + resp, err = fcgiBackend.Patch(env, r.Header.Get("Content-Type"), r.Body, contentLength) case "DELETE": - resp, err = fcgi.Delete(env, r.Header.Get("Content-Type"), r.Body, contentLength) + resp, err = fcgiBackend.Delete(env, r.Header.Get("Content-Type"), r.Body, contentLength) default: return http.StatusMethodNotAllowed, nil } @@ -106,29 +106,35 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) return http.StatusBadGateway, err } - // Write the response body to a buffer - // To explicitly set Content-Length - // For FastCGI app that don't set it - var buf bytes.Buffer - io.Copy(&buf, resp.Body) + var responseBody io.Reader = resp.Body if r.Header.Get("Content-Length") == "" { + // If the upstream app didn't set a Content-Length (shame on them), + // we need to do it to prevent error messages being appended to + // an already-written response, and other problematic behavior. + // So we copy it to a buffer and read its size before flushing + // the response out to the client. See issues #567 and #614. + buf := new(bytes.Buffer) + _, err := io.Copy(buf, resp.Body) + if err != nil { + return http.StatusBadGateway, err + } w.Header().Set("Content-Length", strconv.Itoa(buf.Len())) + responseBody = buf } + + // Write the status code and header fields writeHeader(w, resp) // Write the response body - // TODO: If this has an error, the response will already be - // partly written. We should copy out of resp.Body into a buffer - // first, then write it to the response... - _, err = io.Copy(w, &buf) + _, err = io.Copy(w, responseBody) if err != nil { return http.StatusBadGateway, err } // FastCGI stderr outputs - if fcgi.stderr.Len() != 0 { + if fcgiBackend.stderr.Len() != 0 { // Remove trailing newline, error logger already does this. - err = LogError(strings.TrimSuffix(fcgi.stderr.String(), "\n")) + err = LogError(strings.TrimSuffix(fcgiBackend.stderr.String(), "\n")) } return resp.StatusCode, err diff --git a/middleware/fastcgi/fastcgi_test.go b/middleware/fastcgi/fastcgi_test.go index c33f47af9..5fbba23f1 100644 --- a/middleware/fastcgi/fastcgi_test.go +++ b/middleware/fastcgi/fastcgi_test.go @@ -1,13 +1,61 @@ package fastcgi import ( + "net" "net/http" + "net/http/fcgi" + "net/http/httptest" "net/url" + "strconv" "testing" ) -func TestRuleParseAddress(t *testing.T) { +func TestServeHTTPContentLength(t *testing.T) { + testWithBackend := func(body string, setContentLength bool) { + bodyLenStr := strconv.Itoa(len(body)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("BackendSetsContentLength=%v: Unable to create listener for test: %v", setContentLength, err) + } + defer listener.Close() + go fcgi.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if setContentLength { + w.Header().Set("Content-Length", bodyLenStr) + } + w.Write([]byte(body)) + })) + handler := Handler{ + Next: nil, + Rules: []Rule{{Path: "/", Address: listener.Addr().String()}}, + } + r, err := http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatalf("BackendSetsContentLength=%v: Unable to create request: %v", setContentLength, err) + } + w := httptest.NewRecorder() + + status, err := handler.ServeHTTP(w, r) + + if got, want := status, http.StatusOK; got != want { + t.Errorf("BackendSetsContentLength=%v: Expected returned status code to be %d, got %d", setContentLength, want, got) + } + if err != nil { + t.Errorf("BackendSetsContentLength=%v: Expected nil error, got: %v", setContentLength, err) + } + if got, want := w.Header().Get("Content-Length"), bodyLenStr; got != want { + t.Errorf("BackendSetsContentLength=%v: Expected Content-Length to be '%s', got: '%s'", setContentLength, want, got) + } + if got, want := w.Body.String(), body; got != want { + t.Errorf("BackendSetsContentLength=%v: Expected response body to be '%s', got: '%s'", setContentLength, want, got) + } + } + + testWithBackend("Backend does NOT set Content-Length", false) + testWithBackend("Backend sets Content-Length", true) +} + +func TestRuleParseAddress(t *testing.T) { getClientTestTable := []struct { rule *Rule expectednetwork string @@ -27,28 +75,21 @@ func TestRuleParseAddress(t *testing.T) { if _, actualaddress := entry.rule.parseAddress(); actualaddress != entry.expectedaddress { t.Errorf("Unexpected parsed address for address string %v. Got %v, expected %v", entry.rule.Address, actualaddress, entry.expectedaddress) } - } - } func TestBuildEnv(t *testing.T) { - - buildEnvSingle := func(r *http.Request, rule Rule, fpath string, envExpected map[string]string, t *testing.T) { - - h := Handler{} - + testBuildEnv := func(r *http.Request, rule Rule, fpath string, envExpected map[string]string) { + var h Handler env, err := h.buildEnv(r, rule, fpath) if err != nil { t.Error("Unexpected error:", err.Error()) } - for k, v := range envExpected { if env[k] != v { t.Errorf("Unexpected %v. Got %v, expected %v", k, env[k], v) } } - } rule := Rule{} @@ -80,16 +121,15 @@ func TestBuildEnv(t *testing.T) { } // 1. Test for full canonical IPv6 address - buildEnvSingle(&r, rule, fpath, envExpected, t) + testBuildEnv(&r, rule, fpath, envExpected) // 2. Test for shorthand notation of IPv6 address r.RemoteAddr = "[::1]:51688" envExpected["REMOTE_ADDR"] = "[::1]" - buildEnvSingle(&r, rule, fpath, envExpected, t) + testBuildEnv(&r, rule, fpath, envExpected) // 3. Test for IPv4 address r.RemoteAddr = "192.168.0.10:51688" envExpected["REMOTE_ADDR"] = "192.168.0.10" - buildEnvSingle(&r, rule, fpath, envExpected, t) - + testBuildEnv(&r, rule, fpath, envExpected) } From c37ad7f677b9122544ab2d2384ef8ef2b76bd970 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 24 Feb 2016 19:50:46 -0700 Subject: [PATCH 42/52] Only write error message/page if body not already written (fixes #567) Based on work started in, and replaces, #614 --- middleware/errors/errors.go | 4 +++- middleware/errors/errors_test.go | 11 +++++++++++ middleware/fastcgi/fastcgi.go | 7 ++++++- middleware/log/log.go | 2 +- server/server.go | 32 +------------------------------- 5 files changed, 22 insertions(+), 34 deletions(-) diff --git a/middleware/errors/errors.go b/middleware/errors/errors.go index 33a152692..ccd7e6af8 100644 --- a/middleware/errors/errors.go +++ b/middleware/errors/errors.go @@ -43,7 +43,9 @@ func (h ErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, er } if status >= 400 { - h.errorPage(w, r, status) + if w.Header().Get("Content-Length") == "" { + h.errorPage(w, r, status) + } return 0, err } diff --git a/middleware/errors/errors_test.go b/middleware/errors/errors_test.go index 8afa6bff5..c0cf63259 100644 --- a/middleware/errors/errors_test.go +++ b/middleware/errors/errors_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" @@ -78,6 +79,13 @@ func TestErrors(t *testing.T) { expectedLog: "", expectedErr: nil, }, + { + next: genErrorHandler(http.StatusNotFound, nil, "normal"), + expectedCode: 0, + expectedBody: "normal", + expectedLog: "", + expectedErr: nil, + }, { next: genErrorHandler(http.StatusForbidden, nil, ""), expectedCode: 0, @@ -158,6 +166,9 @@ func TestVisibleErrorWithPanic(t *testing.T) { func genErrorHandler(status int, err error, body string) middleware.Handler { return middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { + if len(body) > 0 { + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + } fmt.Fprint(w, body) return status, err }) diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go index c4ca935e9..fa9a6c469 100644 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -107,7 +107,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) } var responseBody io.Reader = resp.Body - if r.Header.Get("Content-Length") == "" { + if resp.Header.Get("Content-Length") == "" { // If the upstream app didn't set a Content-Length (shame on them), // we need to do it to prevent error messages being appended to // an already-written response, and other problematic behavior. @@ -137,6 +137,11 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) err = LogError(strings.TrimSuffix(fcgiBackend.stderr.String(), "\n")) } + // Normally we should only return a status >= 400 if no response + // body is written yet, however, upstream apps don't know about + // this contract and we still want the correct code logged, so error + // handling code in our stack needs to check Content-Length before + // writing an error message... oh well. return resp.StatusCode, err } } diff --git a/middleware/log/log.go b/middleware/log/log.go index feb6182ad..acb695c5e 100644 --- a/middleware/log/log.go +++ b/middleware/log/log.go @@ -26,7 +26,7 @@ func (l Logger) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { // The error must be handled here so the log entry will record the response size. if l.ErrorFunc != nil { l.ErrorFunc(responseRecorder, r, status) - } else { + } else if responseRecorder.Header().Get("Content-Length") == "" { // ensure no body written since proxy backends may write an error page // Default failover error handler responseRecorder.WriteHeader(status) fmt.Fprintf(responseRecorder, "%d %s", status, http.StatusText(status)) diff --git a/server/server.go b/server/server.go index 3a336f3b0..2df0deac3 100644 --- a/server/server.go +++ b/server/server.go @@ -319,7 +319,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { status, _ := vh.stack.ServeHTTP(w, r) // Fallback error response in case error handling wasn't chained in - if status >= 400 { + if status >= 400 && w.Header().Get("Content-Length") == "" { DefaultErrorFunc(w, r, status) } } else { @@ -417,36 +417,6 @@ func (ln tcpKeepAliveListener) File() (*os.File, error) { return ln.TCPListener.File() } -// copied from net/http/transport.go -/* - TODO - remove - not necessary? -func cloneTLSConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - SessionTicketsDisabled: cfg.SessionTicketsDisabled, - SessionTicketKey: cfg.SessionTicketKey, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, - } -}*/ - // ShutdownCallbacks executes all the shutdown callbacks // for all the virtualhosts in servers, and returns all the // errors generated during their execution. In other words, From 2ecc837020cb215f834cceec0e2d8a01a92a5e7b Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 24 Feb 2016 20:32:26 -0700 Subject: [PATCH 43/52] templates: .Truncate can truncate from end of string if length is negative --- middleware/context.go | 12 +++++++++--- middleware/context_test.go | 26 +++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/middleware/context.go b/middleware/context.go index 4b61da290..7cea124eb 100644 --- a/middleware/context.go +++ b/middleware/context.go @@ -132,10 +132,16 @@ func (c Context) PathMatches(pattern string) bool { return Path(c.Req.URL.Path).Matches(pattern) } -// Truncate truncates the input string to the given length. If -// input is shorter than length, the entire string is returned. +// Truncate truncates the input string to the given length. +// If length is negative, it returns that many characters +// starting from the end of the string. If the absolute value +// of length is greater than len(input), the whole input is +// returned. func (c Context) Truncate(input string, length int) string { - if len(input) > length { + if length < 0 && len(input)+length > 0 { + return input[len(input)+length:] + } + if length >= 0 && len(input) > length { return input[:length] } return input diff --git a/middleware/context_test.go b/middleware/context_test.go index 5c6473e9e..689c47c13 100644 --- a/middleware/context_test.go +++ b/middleware/context_test.go @@ -459,12 +459,36 @@ func TestTruncate(t *testing.T) { inputLength: 10, expected: "string", }, + // Test 3 - zero length + { + inputString: "string", + inputLength: 0, + expected: "", + }, + // Test 4 - negative, smaller length + { + inputString: "string", + inputLength: -5, + expected: "tring", + }, + // Test 5 - negative, exact length + { + inputString: "string", + inputLength: -6, + expected: "string", + }, + // Test 6 - negative, bigger length + { + inputString: "string", + inputLength: -7, + expected: "string", + }, } for i, test := range tests { actual := context.Truncate(test.inputString, test.inputLength) if actual != test.expected { - t.Errorf(getTestPrefix(i)+"Expected %s, found %s. Input was Truncate(%q, %d)", test.expected, actual, test.inputString, test.inputLength) + t.Errorf(getTestPrefix(i)+"Expected '%s', found '%s'. Input was Truncate(%q, %d)", test.expected, actual, test.inputString, test.inputLength) } } } From c827a71d5ddb28969aae19f388ac4b7d737fece4 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 25 Feb 2016 10:26:42 -0700 Subject: [PATCH 44/52] Version 0.8.2 --- dist/CHANGES.txt | 12 ++++++++---- dist/README.txt | 9 ++++++++- main.go | 4 ++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/dist/CHANGES.txt b/dist/CHANGES.txt index 00b837f04..34b69d7d5 100644 --- a/dist/CHANGES.txt +++ b/dist/CHANGES.txt @@ -1,16 +1,20 @@ CHANGES - -- On-demand TLS can obtain certificates during handshake +0.8.2 (February 25, 2016) +- On-demand TLS can obtain certificates during handshakes - Built with Go 1.6 - Process log (-log) is rotated when it gets large +- Managed certificates get renewed 30 days early instead of just 14 - fastcgi: Allow scheme prefix before address - markdown: Support for definition lists - proxy: Allow proxy to insecure HTTPS backends - proxy: Support proxy to unix socket +- rewrite: Status code can be 2xx or 4xx - templates: New .Markdown action to interpret included file as Markdown -- tls: max_certs setting to set hard limit on-demand TLS -- tls: load certificates from directory +- templates: .Truncate now truncates from end of string when length is negative +- tls: Set hard limit for certificates obtained with on-demand TLS +- tls: Load certificates from directory +- tls: Add SHA384 cipher suites - Multiple bug fixes and internal changes diff --git a/dist/README.txt b/dist/README.txt index 532e93f46..e2ec8a24b 100644 --- a/dist/README.txt +++ b/dist/README.txt @@ -1,16 +1,23 @@ -CADDY 0.8.1 +CADDY 0.8.2 Website https://caddyserver.com + +Twitter @caddyserver Source Code https://github.com/mholt/caddy + https://github.com/caddyserver For instructions on using Caddy, please see the user guide on the website. For a list of what's new in this version, see CHANGES.txt. +Please consider donating to the project if you think it is helpful, +especially if your company is using Caddy. There are also sponsorship +opportunities available! + If you have a question, bug report, or would like to contribute, please open an issue or submit a pull request on GitHub. Your contributions do not go unnoticed! diff --git a/main.go b/main.go index ed4449fdd..3d2bae760 100644 --- a/main.go +++ b/main.go @@ -28,7 +28,7 @@ var ( const ( appName = "Caddy" - appVersion = "0.8.1" + appVersion = "0.8.2" ) func init() { @@ -40,7 +40,7 @@ func init() { flag.StringVar(&https.DefaultEmail, "email", "", "Default Let's Encrypt account email address") flag.DurationVar(&caddy.GracefulTimeout, "grace", 5*time.Second, "Maximum duration of graceful shutdown") flag.StringVar(&caddy.Host, "host", caddy.DefaultHost, "Default host") - flag.BoolVar(&caddy.HTTP2, "http2", true, "HTTP/2 support") + flag.BoolVar(&caddy.HTTP2, "http2", true, "Use HTTP/2") flag.StringVar(&logfile, "log", "", "Process log file") flag.StringVar(&caddy.PidFile, "pidfile", "", "Path to write pid file") flag.StringVar(&caddy.Port, "port", caddy.DefaultPort, "Default port") From da08c94a8cb34fc4c1a933d9ba13b9455abfed28 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Fri, 26 Feb 2016 00:21:20 -0700 Subject: [PATCH 45/52] Implant version information with -ldflags with help of build script Without -ldflags, the verison information needs to be updated manually, which is never done between releases, so development builds appear indiscernable from stable builds using `caddy -version`. This is part of a set of changes intended to relieve the burden of always updating version information manually and distributing binaries that look stable but actually may not be. A stable build is defined as one which is produced at a git tag with a clean working directory (no uncommitted changes). A dev build is anything else. With this build script, `caddy -version` will now reveal whether it is a development build and, if so, the base version, the latest commit, the date and time of build, and the names of files with changes as well as how many changes were made. The output of `caddy -version` for stable builds remains the same. --- build.bash | 55 +++++++++++++++++++++++++++++++++++++++++++++++ main.go | 60 ++++++++++++++++++++++++++++++++++++++++------------ main_test.go | 31 +++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 14 deletions(-) create mode 100755 build.bash diff --git a/build.bash b/build.bash new file mode 100755 index 000000000..b7c97d1ec --- /dev/null +++ b/build.bash @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# Caddy build script. Automates proper versioning. +# +# Usage: +# +# $ ./build.bash [output_filename] +# +# Outputs compiled program in current directory. +# Default file name is 'ecaddy'. +# +set -e + +output="$1" +if [ -z "$output" ]; then + output="ecaddy" +fi + +pkg=main + +# Timestamp of build +builddate_id=$pkg.buildDate +builddate=`date -u` + +# Current tag, if HEAD is on a tag +tag_id=$pkg.gitTag +set +e +tag=`git describe --exact-match HEAD 2> /dev/null` +set -e + +# Nearest tag on branch +lasttag_id=$pkg.gitNearestTag +lasttag=`git describe --abbrev=0 --tags HEAD` + +# Commit SHA +commit_id=$pkg.gitCommit +commit=`git rev-parse --short HEAD` + +# Summary of uncommited changes +shortstat_id=$pkg.gitShortStat +shortstat=`git diff-index --shortstat HEAD` + +# List of modified files +files_id=$pkg.gitFilesModified +files=`git diff-index --name-only HEAD` + + +go build -ldflags " + -X \"$builddate_id=$builddate\" + -X \"$tag_id=$tag\" + -X \"$lasttag_id=$lasttag\" + -X \"$commit_id=$commit\" + -X \"$shortstat_id=$shortstat\" + -X \"$files_id=$files\" +" -o "$output" diff --git a/main.go b/main.go index 3d2bae760..b509b12fd 100644 --- a/main.go +++ b/main.go @@ -18,21 +18,9 @@ import ( "gopkg.in/natefinch/lumberjack.v2" ) -var ( - conf string - cpu string - logfile string - revoke string - version bool -) - -const ( - appName = "Caddy" - appVersion = "0.8.2" -) - func init() { caddy.TrapSignals() + setVersion() flag.BoolVar(&https.Agreed, "agree", false, "Agree to Let's Encrypt Subscriber Agreement") flag.StringVar(&https.CAUrl, "ca", "https://acme-v01.api.letsencrypt.org/directory", "Certificate authority ACME server") flag.StringVar(&conf, "conf", "", "Configuration file to use (default="+caddy.DefaultConfigFile+")") @@ -83,7 +71,10 @@ func main() { os.Exit(0) } if version { - fmt.Printf("%s %s\n", caddy.AppName, caddy.AppVersion) + fmt.Printf("%s %s\n", appName, appVersion) + if devBuild && gitShortStat != "" { + fmt.Printf("%s\n%s\n", gitShortStat, gitFilesModified) + } os.Exit(0) } @@ -199,3 +190,44 @@ func setCPU(cpu string) error { runtime.GOMAXPROCS(numCPU) return nil } + +// setVersion figures out the version information based on +// variables set by -ldflags. +func setVersion() { + // A development build is one that's not at a tag or has uncommitted changes + devBuild = gitTag == "" || gitShortStat != "" + + // Only set the appVersion if -ldflags was used + if gitNearestTag != "" || gitTag != "" { + if devBuild && gitNearestTag != "" { + appVersion = fmt.Sprintf("%s (+%s %s)", + strings.TrimPrefix(gitNearestTag, "v"), gitCommit, buildDate) + } else if gitTag != "" { + appVersion = strings.TrimPrefix(gitTag, "v") + } + } +} + +const appName = "Caddy" + +// Flags that control program flow or startup +var ( + conf string + cpu string + logfile string + revoke string + version bool +) + +// Build information obtained with the help of -ldflags +var ( + appVersion = "(untracked dev build)" // inferred at startup + devBuild = true // inferred at startup + + buildDate string // date -u + gitTag string // git describe --exact-match HEAD 2> /dev/null + gitNearestTag string // git describe --abbrev=0 --tags HEAD + gitCommit string // git rev-parse HEAD + gitShortStat string // git diff-index --shortstat + gitFilesModified string // git diff-index --name-only HEAD +) diff --git a/main_test.go b/main_test.go index 311673164..01722ed60 100644 --- a/main_test.go +++ b/main_test.go @@ -42,3 +42,34 @@ func TestSetCPU(t *testing.T) { runtime.GOMAXPROCS(currentCPU) } } + +func TestSetVersion(t *testing.T) { + setVersion() + if !devBuild { + t.Error("Expected default to assume development build, but it didn't") + } + if got, want := appVersion, "(untracked dev build)"; got != want { + t.Errorf("Expected appVersion='%s', got: '%s'", want, got) + } + + gitTag = "v1.1" + setVersion() + if devBuild { + t.Error("Expected a stable build if gitTag is set with no changes") + } + if got, want := appVersion, "1.1"; got != want { + t.Errorf("Expected appVersion='%s', got: '%s'", want, got) + } + + gitTag = "" + gitNearestTag = "v1.0" + gitCommit = "deadbeef" + buildDate = "Fri Feb 26 06:53:17 UTC 2016" + setVersion() + if !devBuild { + t.Error("Expected inferring a dev build when gitTag is empty") + } + if got, want := appVersion, "1.0 (+deadbeef Fri Feb 26 06:53:17 UTC 2016)"; got != want { + t.Errorf("Expected appVersion='%s', got: '%s'", want, got) + } +} From 49c2807ba14bbac97376f3c5f4ece85b51386a55 Mon Sep 17 00:00:00 2001 From: Henrik Jonsson Date: Sat, 27 Feb 2016 17:49:19 +0100 Subject: [PATCH 46/52] Fix build after https://github.com/xenolf/lego/commit/0e26b Fix up last-second changes Fixes #640 --- caddy/https/client.go | 11 ++++++++++- caddy/https/crypto_test.go | 4 +++- caddy/https/user.go | 3 ++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/caddy/https/client.go b/caddy/https/client.go index b47fd57f3..40e7a7098 100644 --- a/caddy/https/client.go +++ b/caddy/https/client.go @@ -34,7 +34,16 @@ var NewACMEClient = func(email string, allowPrompts bool) (*ACMEClient, error) { } // The client facilitates our communication with the CA server. - client, err := acme.NewClient(CAUrl, &leUser, rsaKeySizeToUse) + var kt acme.KeyType + if rsaKeySizeToUse == Rsa2048 { + kt = acme.RSA2048 + } else if rsaKeySizeToUse == Rsa4096 { + kt = acme.RSA4096 + } else { + // TODO(hkjn): Support more types? Current changes are quick fix for #640. + return nil, fmt.Errorf("https: unsupported keysize") + } + client, err := acme.NewClient(CAUrl, &leUser, kt) if err != nil { return nil, err } diff --git a/caddy/https/crypto_test.go b/caddy/https/crypto_test.go index 875f2d217..39cd27b53 100644 --- a/caddy/https/crypto_test.go +++ b/caddy/https/crypto_test.go @@ -11,7 +11,9 @@ import ( ) func init() { - rsaKeySizeToUse = 128 // make tests faster; small key size OK for testing + rsaKeySizeToUse = 2048 // TODO(hkjn): Bring back support for small + // keys to speed up tests? Current changes + // are quick fix for #640. } func TestSaveAndLoadRSAPrivateKey(t *testing.T) { diff --git a/caddy/https/user.go b/caddy/https/user.go index c5a742526..203e07a27 100644 --- a/caddy/https/user.go +++ b/caddy/https/user.go @@ -2,6 +2,7 @@ package https import ( "bufio" + "crypto" "crypto/rand" "crypto/rsa" "encoding/json" @@ -34,7 +35,7 @@ func (u User) GetRegistration() *acme.RegistrationResource { } // GetPrivateKey gets u's private key. -func (u User) GetPrivateKey() *rsa.PrivateKey { +func (u User) GetPrivateKey() crypto.PrivateKey { return u.key } From 741880a38be0104fb2023b944ea3b9afbc130e83 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 1 Mar 2016 12:27:46 -0700 Subject: [PATCH 47/52] Only obtain certificate and enable TLS if host qualifies (fixes #638) --- caddy/https/https.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caddy/https/https.go b/caddy/https/https.go index 824de541b..90022ed5a 100644 --- a/caddy/https/https.go +++ b/caddy/https/https.go @@ -124,7 +124,7 @@ func ObtainCerts(configs []server.Config, allowPrompts, proxyACME bool) error { var client *ACMEClient for _, cfg := range group { - if cfg.Host == "" || existingCertAndKey(cfg.Host) { + if !HostQualifies(cfg.Host) || existingCertAndKey(cfg.Host) { continue } @@ -190,7 +190,7 @@ func EnableTLS(configs []server.Config, loadCertificates bool) error { continue } configs[i].TLS.Enabled = true - if loadCertificates && configs[i].Host != "" { + if loadCertificates && HostQualifies(configs[i].Host) { _, err := cacheManagedCertificate(configs[i].Host, false) if err != nil { return err From 2a46f2a14eb647513fda843ac9b0efac1dbabd43 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 2 Mar 2016 11:33:40 -0700 Subject: [PATCH 48/52] Revert recent Content-Length-related changes and fix fastcgi return fastcgi's ServeHTTP method originally returned the correct value (0) in b51e8bc191da8c84999797caeb0c998d4305d088. Later, I mistakenly suggested we change that to return the status code because I forgot that status codes aren't logged by the return value. So fastcgi broke due in 3966936bd6f01462fb8b41198bf36a83e17ad6e7 due to my error. We later had to try to make up for this with ugly Content-Length checks like in c37ad7f677b9122544ab2d2384ef8ef2b76bd970. Turns out that all we had to do was fix the returned status here back to 0. The proxy middleware behaves the same way, and returning 0 is correct. We should only return a status code if the response has not been written, but with upstream servers, we do write a response; they do not know about our error handler. Also clarifed this in the middleware.Handler documentation. --- middleware/errors/errors.go | 4 +- middleware/errors/errors_test.go | 9 +--- middleware/fastcgi/fastcgi.go | 36 ++++---------- middleware/fastcgi/fastcgi_test.go | 75 ++++++++++++++---------------- middleware/log/log.go | 2 +- middleware/middleware.go | 36 ++++++-------- server/server.go | 2 +- 7 files changed, 64 insertions(+), 100 deletions(-) mode change 100644 => 100755 middleware/fastcgi/fastcgi.go diff --git a/middleware/errors/errors.go b/middleware/errors/errors.go index ccd7e6af8..33a152692 100644 --- a/middleware/errors/errors.go +++ b/middleware/errors/errors.go @@ -43,9 +43,7 @@ func (h ErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, er } if status >= 400 { - if w.Header().Get("Content-Length") == "" { - h.errorPage(w, r, status) - } + h.errorPage(w, r, status) return 0, err } diff --git a/middleware/errors/errors_test.go b/middleware/errors/errors_test.go index c0cf63259..49af3e4f4 100644 --- a/middleware/errors/errors_test.go +++ b/middleware/errors/errors_test.go @@ -79,13 +79,6 @@ func TestErrors(t *testing.T) { expectedLog: "", expectedErr: nil, }, - { - next: genErrorHandler(http.StatusNotFound, nil, "normal"), - expectedCode: 0, - expectedBody: "normal", - expectedLog: "", - expectedErr: nil, - }, { next: genErrorHandler(http.StatusForbidden, nil, ""), expectedCode: 0, @@ -168,8 +161,8 @@ func genErrorHandler(status int, err error, body string) middleware.Handler { return middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { if len(body) > 0 { w.Header().Set("Content-Length", strconv.Itoa(len(body))) + fmt.Fprint(w, body) } - fmt.Fprint(w, body) return status, err }) } diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go old mode 100644 new mode 100755 index fa9a6c469..33b21d435 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -4,7 +4,6 @@ package fastcgi import ( - "bytes" "errors" "io" "net/http" @@ -106,43 +105,28 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) return http.StatusBadGateway, err } - var responseBody io.Reader = resp.Body - if resp.Header.Get("Content-Length") == "" { - // If the upstream app didn't set a Content-Length (shame on them), - // we need to do it to prevent error messages being appended to - // an already-written response, and other problematic behavior. - // So we copy it to a buffer and read its size before flushing - // the response out to the client. See issues #567 and #614. - buf := new(bytes.Buffer) - _, err := io.Copy(buf, resp.Body) - if err != nil { - return http.StatusBadGateway, err - } - w.Header().Set("Content-Length", strconv.Itoa(buf.Len())) - responseBody = buf - } - - // Write the status code and header fields + // Write response header writeHeader(w, resp) // Write the response body - _, err = io.Copy(w, responseBody) + _, err = io.Copy(w, resp.Body) if err != nil { return http.StatusBadGateway, err } - // FastCGI stderr outputs + // Log any stderr output from upstream if fcgiBackend.stderr.Len() != 0 { // Remove trailing newline, error logger already does this. err = LogError(strings.TrimSuffix(fcgiBackend.stderr.String(), "\n")) } - // Normally we should only return a status >= 400 if no response - // body is written yet, however, upstream apps don't know about - // this contract and we still want the correct code logged, so error - // handling code in our stack needs to check Content-Length before - // writing an error message... oh well. - return resp.StatusCode, err + // Normally we would return the status code if it is an error status (>= 400), + // however, upstream FastCGI apps don't know about our contract and have + // probably already written an error page. So we just return 0, indicating + // that the response body is already written. However, we do return any + // error value so it can be logged. + // Note that the proxy middleware works the same way, returning status=0. + return 0, err } } diff --git a/middleware/fastcgi/fastcgi_test.go b/middleware/fastcgi/fastcgi_test.go index 5fbba23f1..001f38721 100644 --- a/middleware/fastcgi/fastcgi_test.go +++ b/middleware/fastcgi/fastcgi_test.go @@ -10,49 +10,44 @@ import ( "testing" ) -func TestServeHTTPContentLength(t *testing.T) { - testWithBackend := func(body string, setContentLength bool) { - bodyLenStr := strconv.Itoa(len(body)) - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatalf("BackendSetsContentLength=%v: Unable to create listener for test: %v", setContentLength, err) - } - defer listener.Close() - go fcgi.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if setContentLength { - w.Header().Set("Content-Length", bodyLenStr) - } - w.Write([]byte(body)) - })) +func TestServeHTTP(t *testing.T) { + body := "This is some test body content" - handler := Handler{ - Next: nil, - Rules: []Rule{{Path: "/", Address: listener.Addr().String()}}, - } - r, err := http.NewRequest("GET", "/", nil) - if err != nil { - t.Fatalf("BackendSetsContentLength=%v: Unable to create request: %v", setContentLength, err) - } - w := httptest.NewRecorder() - - status, err := handler.ServeHTTP(w, r) - - if got, want := status, http.StatusOK; got != want { - t.Errorf("BackendSetsContentLength=%v: Expected returned status code to be %d, got %d", setContentLength, want, got) - } - if err != nil { - t.Errorf("BackendSetsContentLength=%v: Expected nil error, got: %v", setContentLength, err) - } - if got, want := w.Header().Get("Content-Length"), bodyLenStr; got != want { - t.Errorf("BackendSetsContentLength=%v: Expected Content-Length to be '%s', got: '%s'", setContentLength, want, got) - } - if got, want := w.Body.String(), body; got != want { - t.Errorf("BackendSetsContentLength=%v: Expected response body to be '%s', got: '%s'", setContentLength, want, got) - } + bodyLenStr := strconv.Itoa(len(body)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Unable to create listener for test: %v", err) } + defer listener.Close() + go fcgi.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", bodyLenStr) + w.Write([]byte(body)) + })) - testWithBackend("Backend does NOT set Content-Length", false) - testWithBackend("Backend sets Content-Length", true) + handler := Handler{ + Next: nil, + Rules: []Rule{{Path: "/", Address: listener.Addr().String()}}, + } + r, err := http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatalf("Unable to create request: %v", err) + } + w := httptest.NewRecorder() + + status, err := handler.ServeHTTP(w, r) + + if got, want := status, 0; got != want { + t.Errorf("Expected returned status code to be %d, got %d", want, got) + } + if err != nil { + t.Errorf("Expected nil error, got: %v", err) + } + if got, want := w.Header().Get("Content-Length"), bodyLenStr; got != want { + t.Errorf("Expected Content-Length to be '%s', got: '%s'", want, got) + } + if got, want := w.Body.String(), body; got != want { + t.Errorf("Expected response body to be '%s', got: '%s'", want, got) + } } func TestRuleParseAddress(t *testing.T) { diff --git a/middleware/log/log.go b/middleware/log/log.go index acb695c5e..feb6182ad 100644 --- a/middleware/log/log.go +++ b/middleware/log/log.go @@ -26,7 +26,7 @@ func (l Logger) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { // The error must be handled here so the log entry will record the response size. if l.ErrorFunc != nil { l.ErrorFunc(responseRecorder, r, status) - } else if responseRecorder.Header().Get("Content-Length") == "" { // ensure no body written since proxy backends may write an error page + } else { // Default failover error handler responseRecorder.WriteHeader(status) fmt.Fprintf(responseRecorder, "%d %s", status, http.StatusText(status)) diff --git a/middleware/middleware.go b/middleware/middleware.go index c7036f3c9..d91044ebe 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -13,30 +13,24 @@ type ( // passed the next Handler in the chain. Middleware func(Handler) Handler - // Handler is like http.Handler except ServeHTTP returns a status code - // and an error. The status code is for the client's benefit; the error - // value is for the server's benefit. The status code will be sent to - // the client while the error value will be logged privately. Sometimes, - // an error status code (4xx or 5xx) may be returned with a nil error - // when there is no reason to log the error on the server. + // Handler is like http.Handler except ServeHTTP may return a status + // code and/or error. // - // If a HandlerFunc returns an error (status >= 400), it should NOT - // write to the response. This philosophy makes middleware.Handler - // different from http.Handler: error handling should happen at the - // application layer or in dedicated error-handling middleware only - // rather than with an "every middleware for itself" paradigm. + // If ServeHTTP writes to the response body, it should return a status + // code of 0. This signals to other handlers above it that the response + // body is already written, and that they should not write to it also. // - // The application or error-handling middleware should incorporate logic - // to ensure that the client always gets a proper response according to - // the status code. For security reasons, it should probably not reveal - // the actual error message. (Instead it should be logged, for example.) + // If ServeHTTP encounters an error, it should return the error value + // so it can be logged by designated error-handling middleware. // - // Handlers which do write to the response should return a status value - // < 400 as a signal that a response has been written. In other words, - // only error-handling middleware or the application will write to the - // response for a status code >= 400. When ANY handler writes to the - // response, it should return a status code < 400 to signal others to - // NOT write to the response again, which would be erroneous. + // If writing a response after calling another ServeHTTP method, the + // returned status code SHOULD be used when writing the response. + // + // If handling errors after calling another ServeHTTP method, the + // returned error value SHOULD be logged or handled accordingly. + // + // Otherwise, return values should be propagated down the middleware + // chain by returning them unchanged. Handler interface { ServeHTTP(http.ResponseWriter, *http.Request) (int, error) } diff --git a/server/server.go b/server/server.go index 2df0deac3..d687b8378 100644 --- a/server/server.go +++ b/server/server.go @@ -319,7 +319,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { status, _ := vh.stack.ServeHTTP(w, r) // Fallback error response in case error handling wasn't chained in - if status >= 400 && w.Header().Get("Content-Length") == "" { + if status >= 400 { DefaultErrorFunc(w, r, status) } } else { From 36b440c04b33cb443b045ee1751917beecb3cce3 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 2 Mar 2016 11:34:39 -0700 Subject: [PATCH 49/52] https: Refuse start only if renewal fails on expired cert (closes #642) --- caddy/https/maintain.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/caddy/https/maintain.go b/caddy/https/maintain.go index 49fc1c169..28fa2fe6c 100644 --- a/caddy/https/maintain.go +++ b/caddy/https/maintain.go @@ -89,8 +89,13 @@ func renewManagedCertificates(allowPrompts bool) (err error) { err := client.Renew(cert.Names[0]) // managed certs better have only one name if err != nil { - if client.AllowPrompts { - // User is present, so stop immediately and report the error + if client.AllowPrompts && timeLeft < 0 { + // Certificate renewal failed, the operator is present, and the certificate + // is already expired; we should stop immediately and return the error. Note + // that we used to do this any time a renewal failed at startup. However, + // after discussion in https://github.com/mholt/caddy/issues/642 we decided to + // only stop startup if the certificate is expired. We still log the error + // otherwise. certCacheMu.RUnlock() return err } From 9099375b11b7b5e62b831627c2927d1c4c666071 Mon Sep 17 00:00:00 2001 From: elcore Date: Wed, 2 Mar 2016 14:34:33 +0100 Subject: [PATCH 50/52] Support ECC certificates --- caddy/https/client.go | 11 +----- caddy/https/crypto.go | 38 ++++++++++++++++--- caddy/https/crypto_test.go | 76 ++++++++++++++++++++++++++++++-------- caddy/https/https.go | 15 +------- caddy/https/user.go | 11 +++--- caddy/https/user_test.go | 2 +- 6 files changed, 102 insertions(+), 51 deletions(-) diff --git a/caddy/https/client.go b/caddy/https/client.go index 40e7a7098..762e58aa1 100644 --- a/caddy/https/client.go +++ b/caddy/https/client.go @@ -34,16 +34,7 @@ var NewACMEClient = func(email string, allowPrompts bool) (*ACMEClient, error) { } // The client facilitates our communication with the CA server. - var kt acme.KeyType - if rsaKeySizeToUse == Rsa2048 { - kt = acme.RSA2048 - } else if rsaKeySizeToUse == Rsa4096 { - kt = acme.RSA4096 - } else { - // TODO(hkjn): Support more types? Current changes are quick fix for #640. - return nil, fmt.Errorf("https: unsupported keysize") - } - client, err := acme.NewClient(CAUrl, &leUser, kt) + client, err := acme.NewClient(CAUrl, &leUser, KeyType) if err != nil { return nil, err } diff --git a/caddy/https/crypto.go b/caddy/https/crypto.go index efc40d434..bc0ff6373 100644 --- a/caddy/https/crypto.go +++ b/caddy/https/crypto.go @@ -1,26 +1,52 @@ package https import ( + "crypto" + "crypto/ecdsa" "crypto/rsa" "crypto/x509" "encoding/pem" + "errors" "io/ioutil" "os" ) -// loadRSAPrivateKey loads a PEM-encoded RSA private key from file. -func loadRSAPrivateKey(file string) (*rsa.PrivateKey, error) { +// loadPrivateKey loads a PEM-encoded ECC/RSA private key from file. +func loadPrivateKey(file string) (crypto.PrivateKey, error) { keyBytes, err := ioutil.ReadFile(file) if err != nil { return nil, err } keyBlock, _ := pem.Decode(keyBytes) - return x509.ParsePKCS1PrivateKey(keyBlock.Bytes) + + switch keyBlock.Type { + case "RSA PRIVATE KEY": + return x509.ParsePKCS1PrivateKey(keyBlock.Bytes) + case "EC PRIVATE KEY": + return x509.ParseECPrivateKey(keyBlock.Bytes) + } + + return nil, errors.New("unknown private key type") } -// saveRSAPrivateKey saves a PEM-encoded RSA private key to file. -func saveRSAPrivateKey(key *rsa.PrivateKey, file string) error { - pemKey := pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)} +// savePrivateKey saves a PEM-encoded ECC/RSA private key to file. +func savePrivateKey(key crypto.PrivateKey, file string) error { + var pemType string + var keyBytes []byte + switch key := key.(type) { + case *ecdsa.PrivateKey: + var err error + pemType = "EC" + keyBytes, err = x509.MarshalECPrivateKey(key) + if err != nil { + return err + } + case *rsa.PrivateKey: + pemType = "RSA" + keyBytes = x509.MarshalPKCS1PrivateKey(key) + } + + pemKey := pem.Block{Type: pemType + " PRIVATE KEY", Bytes: keyBytes} keyOut, err := os.Create(file) if err != nil { return err diff --git a/caddy/https/crypto_test.go b/caddy/https/crypto_test.go index 39cd27b53..c1f32b27d 100644 --- a/caddy/https/crypto_test.go +++ b/caddy/https/crypto_test.go @@ -2,6 +2,9 @@ package https import ( "bytes" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" "crypto/rand" "crypto/rsa" "crypto/x509" @@ -10,23 +13,17 @@ import ( "testing" ) -func init() { - rsaKeySizeToUse = 2048 // TODO(hkjn): Bring back support for small - // keys to speed up tests? Current changes - // are quick fix for #640. -} - func TestSaveAndLoadRSAPrivateKey(t *testing.T) { keyFile := "test.key" defer os.Remove(keyFile) - privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeySizeToUse) + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { t.Fatal(err) } // test save - err = saveRSAPrivateKey(privateKey, keyFile) + err = savePrivateKey(privateKey, keyFile) if err != nil { t.Fatal("error saving private key:", err) } @@ -45,23 +42,70 @@ func TestSaveAndLoadRSAPrivateKey(t *testing.T) { } // test load - loadedKey, err := loadRSAPrivateKey(keyFile) + loadedKey, err := loadPrivateKey(keyFile) if err != nil { t.Error("error loading private key:", err) } // verify loaded key is correct - if !rsaPrivateKeysSame(privateKey, loadedKey) { + if !PrivateKeysSame(privateKey, loadedKey) { t.Error("Expected key bytes to be the same, but they weren't") } } -// rsaPrivateKeysSame compares the bytes of a and b and returns true if they are the same. -func rsaPrivateKeysSame(a, b *rsa.PrivateKey) bool { - return bytes.Equal(rsaPrivateKeyBytes(a), rsaPrivateKeyBytes(b)) +func TestSaveAndLoadECCPrivateKey(t *testing.T) { + keyFile := "test.key" + defer os.Remove(keyFile) + + privateKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + t.Fatal(err) + } + + // test save + err = savePrivateKey(privateKey, keyFile) + if err != nil { + t.Fatal("error saving private key:", err) + } + + // it doesn't make sense to test file permission on windows + if runtime.GOOS != "windows" { + // get info of the key file + info, err := os.Stat(keyFile) + if err != nil { + t.Fatal("error stating private key:", err) + } + // verify permission of key file is correct + if info.Mode().Perm() != 0600 { + t.Error("Expected key file to have permission 0600, but it wasn't") + } + } + + // test load + loadedKey, err := loadPrivateKey(keyFile) + if err != nil { + t.Error("error loading private key:", err) + } + + // verify loaded key is correct + if !PrivateKeysSame(privateKey, loadedKey) { + t.Error("Expected key bytes to be the same, but they weren't") + } } -// rsaPrivateKeyBytes returns the bytes of DER-encoded key. -func rsaPrivateKeyBytes(key *rsa.PrivateKey) []byte { - return x509.MarshalPKCS1PrivateKey(key) +// PrivateKeysSame compares the bytes of a and b and returns true if they are the same. +func PrivateKeysSame(a, b crypto.PrivateKey) bool { + return bytes.Equal(PrivateKeyBytes(a), PrivateKeyBytes(b)) +} + +// PrivateKeyBytes returns the bytes of DER-encoded key. +func PrivateKeyBytes(key crypto.PrivateKey) []byte { + var keyBytes []byte + switch key := key.(type) { + case *rsa.PrivateKey: + keyBytes = x509.MarshalPKCS1PrivateKey(key) + case *ecdsa.PrivateKey: + keyBytes, _ = x509.MarshalECPrivateKey(key) + } + return keyBytes } diff --git a/caddy/https/https.go b/caddy/https/https.go index 90022ed5a..76e5e3129 100644 --- a/caddy/https/https.go +++ b/caddy/https/https.go @@ -401,21 +401,10 @@ var ( // default port for the challenge must be forwarded to this one. const AlternatePort = "5033" -// KeySize represents the length of a key in bits. -type KeySize int - -// Key sizes are used to determine the strength of a key. -const ( - Ecc224 KeySize = 224 - Ecc256 = 256 - Rsa2048 = 2048 - Rsa4096 = 4096 -) - -// rsaKeySizeToUse is the size to use for new RSA keys. +// KeyType is the type to use for new keys. // This shouldn't need to change except for in tests; // the size can be drastically reduced for speed. -var rsaKeySizeToUse = Rsa2048 +var KeyType = acme.EC384 // stopChan is used to signal the maintenance goroutine // to terminate. diff --git a/caddy/https/user.go b/caddy/https/user.go index 203e07a27..a7e6e5f62 100644 --- a/caddy/https/user.go +++ b/caddy/https/user.go @@ -3,8 +3,9 @@ package https import ( "bufio" "crypto" + "crypto/ecdsa" + "crypto/elliptic" "crypto/rand" - "crypto/rsa" "encoding/json" "errors" "fmt" @@ -21,7 +22,7 @@ import ( type User struct { Email string Registration *acme.RegistrationResource - key *rsa.PrivateKey + key crypto.PrivateKey } // GetEmail gets u's email. @@ -64,7 +65,7 @@ func getUser(email string) (User, error) { } // load their private key - user.key, err = loadRSAPrivateKey(storage.UserKeyFile(email)) + user.key, err = loadPrivateKey(storage.UserKeyFile(email)) if err != nil { return user, err } @@ -83,7 +84,7 @@ func saveUser(user User) error { } // save private key file - err = saveRSAPrivateKey(user.key, storage.UserKeyFile(user.Email)) + err = savePrivateKey(user.key, storage.UserKeyFile(user.Email)) if err != nil { return err } @@ -104,7 +105,7 @@ func saveUser(user User) error { // instead. It does NOT prompt the user. func newUser(email string) (User, error) { user := User{Email: email} - privateKey, err := rsa.GenerateKey(rand.Reader, rsaKeySizeToUse) + privateKey, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) if err != nil { return user, errors.New("error generating private key: " + err.Error()) } diff --git a/caddy/https/user_test.go b/caddy/https/user_test.go index 5bc28b04c..c1d115e1f 100644 --- a/caddy/https/user_test.go +++ b/caddy/https/user_test.go @@ -114,7 +114,7 @@ func TestGetUserAlreadyExists(t *testing.T) { } // Assert keys are the same - if !rsaPrivateKeysSame(user.key, user2.key) { + if !PrivateKeysSame(user.key, user2.key) { t.Error("Expected private key to be the same after loading, but it wasn't") } From f52b1e80f5123845251ac3298ebf98914fa468d1 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 7 Mar 2016 12:07:34 -0700 Subject: [PATCH 51/52] Update contributing doc and add issue template --- CONTRIBUTING.md | 72 +++++++++++++++++++++++++++++++++++++------------ ISSUE_TEMPLATE | 20 ++++++++++++++ 2 files changed, 75 insertions(+), 17 deletions(-) create mode 100644 ISSUE_TEMPLATE diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 44eb8638a..bf8505a0e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,28 +1,66 @@ ## Contributing to Caddy -**[Join our dev chat on Gitter](https://gitter.im/mholt/caddy)** to chat with -other Caddy developers! (Dev chat only; try our -[support room](https://gitter.im/caddyserver/support) for help or -[general](https://gitter.im/caddyserver/general) for anything else.) - -This project gladly accepts contributions and we encourage interested users to -get involved! +Welcome! Our community focuses on helping others and making Caddy the best it +can be. We gladly accept contributions and encourage you to get involved! -#### For small tweaks, bug fixes, and tests +### Join us in chat -Submit [pull requests](https://github.com/mholt/caddy/pulls) at any time. -Bug fixes should be under test to assert correct behavior. Thank you for -helping out in simple ways! +Please direct your discussion to the correct room: + +- **Dev Chat:** [gitter.im/mholt/caddy](https://gitter.im/mholt/caddy) - to chat +with other Caddy developers +- **Support:** +[gitter.im/caddyserver/support](https://gitter.im/caddyserver/support) - to give +and get help +- **General:** +[gitter.im/caddyserver/general](https://gitter.im/caddyserver/general) - for +anything about Web development -#### Ideas, questions, bug reports +### Bug reports + +First, please [search this repository](https://github.com/mholt/caddy/search?q=&type=Issues&utf8=%E2%9C%93) +with a variety of keywords to ensure your bug is not already reported. + +If not, [open an issue](https://github.com/mholt/caddy/issues) and answer the +questions so we can understand and reproduce the problematic behavior. + +The burden is on you to convince us that it is actually a bug in Caddy. This is +easiest to do when you write clear, concise instructions so we can reproduce +the behavior (even if it seems obvious). The more detailed and specific you are, +the faster we will be able to help you. Check out +[How to Report Bugs Effectively](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html). + +Please be kind. :smile: Remember that Caddy comes at no cost to you, and you're +getting free help. If we helped you, please consider +[donating](https://caddyserver.com/donate) - it keeps us motivated! + + + +### Minor improvements and new tests + +Submit [pull requests](https://github.com/mholt/caddy/pulls) at any time. Make +sure to write tests to assert your change is working properly and is thoroughly +covered. + + +### Proposals, suggestions, ideas, new features + +First, please [search](https://github.com/mholt/caddy/search?q=&type=Issues&utf8=%E2%9C%93) +with a variety of keywords to ensure your suggestion/proposal is new. + +If so, you may open either an issue or a pull request for discussion and +feedback. + +The advantage of issues is that you don't have to spend time actually +implementing your idea, but you should still describe it thoroughly. The +advantage of a pull request is that we can immediately see the impact the change +will have on the project, what the code will look like, and how to improve it. +The disadvantage of pull requests is that they are unlikely to get accepted +without significant changes, or it may be rejected entirely. Don't worry, that +won't happen without an open discussion first. -Feel free to [open an issue](https://github.com/mholt/caddy/issues) with your -ideas, questions, and bug reports, if one does not already exist for it. Bug -reports should state expected behavior and contain clear instructions for -isolating and reproducing the problem. -See [How to Report Bugs Effectively](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html). #### New features diff --git a/ISSUE_TEMPLATE b/ISSUE_TEMPLATE new file mode 100644 index 000000000..f9d55a2db --- /dev/null +++ b/ISSUE_TEMPLATE @@ -0,0 +1,20 @@ +*If you are filing a bug report, please answer these questions. If your issue is not a bug report, you do not need to use this template. Either way, please consider donating if we've helped you. Thanks!* + +#### 1. What version of Caddy are you running (`caddy -version`)? + + +#### 2. What are you trying to do? + + +#### 3. What is your entire Caddyfile? +```text +(Put Caddyfile here) +``` + +#### 4. How did you run Caddy (give the full command and describe the execution environment)? + + +#### 5. What did you expect to see? + + +#### 6. What did you see instead (give full error messages and/or log)? From 88e3a26c99cba3ba080f31f8aa109dc4f6113d39 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 7 Mar 2016 12:10:26 -0700 Subject: [PATCH 52/52] Full changes to contributing doc That was weird, only half of the file got committed... --- CONTRIBUTING.md | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bf8505a0e..346c6dcb9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,7 +37,6 @@ getting free help. If we helped you, please consider [donating](https://caddyserver.com/donate) - it keeps us motivated! - ### Minor improvements and new tests Submit [pull requests](https://github.com/mholt/caddy/pulls) at any time. Make @@ -61,19 +60,12 @@ The disadvantage of pull requests is that they are unlikely to get accepted without significant changes, or it may be rejected entirely. Don't worry, that won't happen without an open discussion first. +If you are going to spend significant time implementing code for a pull request, +best to open an issue first and "claim" it and get feedback before you invest +a lot of time. -#### New features - -Before submitting a pull request, please open an issue first to discuss it and -claim it. This prevents overlapping efforts and keeps the project in-line with -its goals. If you prefer to discuss the feature privately, you can reach other -developers on Gitter or you may email me directly. (My email address is below.) - -And don't forget to write tests for new features! - - -#### Vulnerabilities +### Vulnerabilities If you've found a vulnerability that is serious, please email me: Matthew dot Holt at Gmail. If it's not a big deal, a pull request will probably be faster. @@ -81,4 +73,5 @@ Holt at Gmail. If it's not a big deal, a pull request will probably be faster. ## Thank you -Thanks for your help! Caddy would not be what it is today without your contributions. +Thanks for your help! Caddy would not be what it is today without your +contributions.