diff --git a/config/setup/tls.go b/config/setup/tls.go
index ea2675345..93e9324d4 100644
--- a/config/setup/tls.go
+++ b/config/setup/tls.go
@@ -3,7 +3,7 @@ package setup
 import "github.com/mholt/caddy/middleware"
 
 func TLS(c *Controller) (middleware.Middleware, error) {
-	c.TLS.Enabled = true
+	c.TLS.Enabled = c.Port != "http"
 
 	for c.Next() {
 		if !c.NextArg() {