diff --git a/caddyhttp/httpserver/server.go b/caddyhttp/httpserver/server.go
index a9b0e8465..2006ecfcb 100644
--- a/caddyhttp/httpserver/server.go
+++ b/caddyhttp/httpserver/server.go
@@ -93,9 +93,11 @@ func NewServer(addr string, group []*SiteConfig) (*Server, error) {
 
 	s.tlsConfig = tlsConfigs
 
-	s.Server.TLSConfig = &tls.Config{
-		GetConfigForClient: s.tlsConfig.GetConfigForClient,
-		GetCertificate:     s.tlsConfig.GetCertificate,
+	if caddytls.HasTLSEnabled(allConfigs) {
+		s.Server.TLSConfig = &tls.Config{
+			GetConfigForClient: s.tlsConfig.GetConfigForClient,
+			GetCertificate:     s.tlsConfig.GetCertificate,
+		}
 	}
 
 	// As of Go 1.7, HTTP/2 is enabled only if NextProtos includes the string "h2"
diff --git a/caddytls/config.go b/caddytls/config.go
index 33a16fcc7..840cfebdf 100644
--- a/caddytls/config.go
+++ b/caddytls/config.go
@@ -230,14 +230,22 @@ func (cfg *Config) Build(group ConfigGroup) error {
 		return err
 	}
 
-	cfg.tlsConfig = config
-	cfg.tlsConfig.GetCertificate = group.GetCertificate
+	if config != nil {
+		cfg.tlsConfig = config
+		cfg.tlsConfig.GetCertificate = group.GetCertificate
+	}
+
 	return nil
+
 }
 
 func (cfg *Config) build() (*tls.Config, error) {
 	config := new(tls.Config)
 
+	if !cfg.Enabled {
+		return nil, nil
+	}
+
 	ciphersAdded := make(map[uint16]struct{})
 	curvesAdded := make(map[tls.CurveID]struct{})
 
@@ -337,6 +345,16 @@ func CheckConfigs(configs []*Config) error {
 	return nil
 }
 
+func HasTLSEnabled(configs []*Config) bool {
+	for _, config := range configs {
+		if config.Enabled {
+			return true
+		}
+	}
+
+	return false
+}
+
 // ConfigGetter gets a Config keyed by key.
 type ConfigGetter func(c *caddy.Controller) *Config