From fbc18c5b8560508920efaec89fa4967128904422 Mon Sep 17 00:00:00 2001 From: Tw Date: Sun, 22 Nov 2015 10:50:31 +0800 Subject: [PATCH 01/44] markdown: fix json format parse issue We can't use json meta parser's remaining buffered data as the markdown body because it may not contain the entire original content. Now we adopt the way like toml and yaml parser's way to extract the meta content at first. Also when spilting the meta data and content body, additional io.Copy is unnecessary. Fix issue #355 Signed-off-by: Tw --- middleware/markdown/metadata.go | 71 ++++++++++++--------------------- 1 file changed, 26 insertions(+), 45 deletions(-) diff --git a/middleware/markdown/metadata.go b/middleware/markdown/metadata.go index 103b97b23..2a7e434f0 100644 --- a/middleware/markdown/metadata.go +++ b/middleware/markdown/metadata.go @@ -1,11 +1,9 @@ package markdown import ( - "bufio" "bytes" "encoding/json" "fmt" - "io" "github.com/BurntSushi/toml" "gopkg.in/yaml.v2" @@ -73,23 +71,20 @@ type JSONMetadataParser struct { // Parse the metadata func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) { + b, markdown, err := extractMetadata(j, b) + if err != nil { + return markdown, err + } m := make(map[string]interface{}) // Read the preceding JSON object decoder := json.NewDecoder(bytes.NewReader(b)) if err := decoder.Decode(&m); err != nil { - return b, err + return markdown, err } j.metadata.load(m) - // Retrieve remaining bytes after decoding - buf := make([]byte, len(b)) - n, err := decoder.Buffered().Read(buf) - if err != nil { - return b, err - } - - return buf[:n], nil + return markdown, nil } // Metadata returns parsed metadata. It should be called @@ -183,43 +178,29 @@ func (y *YAMLMetadataParser) Closing() []byte { // It returns the metadata, the remaining bytes (markdown), and an error, if any. func extractMetadata(parser MetadataParser, b []byte) (metadata []byte, markdown []byte, err error) { b = bytes.TrimSpace(b) - reader := bufio.NewReader(bytes.NewBuffer(b)) - - // Read first line, which should indicate metadata or not - line, err := reader.ReadBytes('\n') - if err != nil || !bytes.Equal(bytes.TrimSpace(line), parser.Opening()) { + openingLine := append(parser.Opening(), '\n') + closingLine := append(parser.Closing(), '\n') + if !bytes.HasPrefix(b, openingLine) { return nil, b, fmt.Errorf("first line missing expected metadata identifier") } - - // buffer for metadata contents - metaBuf := bytes.Buffer{} - - // Read remaining lines until closing identifier is found - for { - line, err := reader.ReadBytes('\n') - if err != nil && err != io.EOF { - return nil, nil, err - } - - // if closing identifier found, the remaining bytes must be markdown content - if bytes.Equal(bytes.TrimSpace(line), parser.Closing()) { - break - } - - // if file ended, by this point no closing identifier was found - if err == io.EOF { - return nil, nil, fmt.Errorf("metadata not closed ('%s' not found)", parser.Closing()) - } - - metaBuf.Write(line) - metaBuf.WriteString("\r\n") + metaStart := len(openingLine) + if _, ok := parser.(*JSONMetadataParser); ok { + metaStart = 0 } - - // By now, the rest of the buffer contains markdown content - contentBuf := new(bytes.Buffer) - io.Copy(contentBuf, reader) - - return metaBuf.Bytes(), contentBuf.Bytes(), nil + metaEnd := bytes.Index(b[metaStart:], closingLine) + if metaEnd == -1 { + return nil, nil, fmt.Errorf("metadata not closed ('%s' not found)", parser.Closing()) + } + metaEnd += metaStart + if _, ok := parser.(*JSONMetadataParser); ok { + metaEnd += len(closingLine) + } + metadata = b[metaStart:metaEnd] + markdown = b[metaEnd:] + if _, ok := parser.(*JSONMetadataParser); !ok { + markdown = b[metaEnd+len(closingLine):] + } + return metadata, markdown, nil } // findParser finds the parser using line that contains opening identifier From 19c6bbf6a2275dab38a6307dc84b45c9a52fd27a Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 2 Dec 2015 11:42:50 -0700 Subject: [PATCH 02/44] Update changelist (env vars) --- dist/CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/CHANGES.txt b/dist/CHANGES.txt index e98535b58..01657febf 100644 --- a/dist/CHANGES.txt +++ b/dist/CHANGES.txt @@ -14,6 +14,7 @@ CHANGES - New -grace flag to customize the graceful shutdown timeout - New support for SIGHUP, SIGTERM, and SIGQUIT signals - browse: Render filenames with multiple whitespace properly +- core: Use environment variables in Caddyfile - markdown: Include Last-Modified header in response - markdown: Render tables, strikethrough, and fenced code blocks - proxy: Ability to exclude/ignore paths from proxying From 9002db2ae03906fdbfd3ede2fc32825388889ee9 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 2 Dec 2015 12:40:57 -0700 Subject: [PATCH 03/44] letsencrypt: Remote duplicate hosts from certificate request Domain names must be unique in cert bundle request or really bad things happen (like, um, a panic) --- caddy/letsencrypt/letsencrypt.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/caddy/letsencrypt/letsencrypt.go b/caddy/letsencrypt/letsencrypt.go index 16312bd6a..feeac2975 100644 --- a/caddy/letsencrypt/letsencrypt.go +++ b/caddy/letsencrypt/letsencrypt.go @@ -79,9 +79,19 @@ func Activate(configs []server.Config) ([]server.Config, error) { } // little bit of housekeeping; gather the hostnames into a slice - hosts := make([]string, len(cfgIndexes)) - for i, idx := range cfgIndexes { - hosts[i] = configs[idx].Host + var hosts []string + for _, idx := range cfgIndexes { + // don't allow duplicates (happens when serving same host on multiple ports!) + var duplicate bool + for _, otherHost := range hosts { + if configs[idx].Host == otherHost { + duplicate = true + break + } + } + if !duplicate { + hosts = append(hosts, configs[idx].Host) + } } // client is ready, so let's get free, trusted SSL certificates! From 6f4835f91a4c18a4acd7c5d8977761b8fdf71b0f Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Thu, 3 Dec 2015 00:41:12 +0100 Subject: [PATCH 04/44] Markdown: Fix "metadata not closed" bug. More tests. --- middleware/markdown/metadata.go | 6 +- middleware/markdown/metadata_test.go | 172 +++++++++++++++++---------- 2 files changed, 109 insertions(+), 69 deletions(-) diff --git a/middleware/markdown/metadata.go b/middleware/markdown/metadata.go index 2a7e434f0..07c00801b 100644 --- a/middleware/markdown/metadata.go +++ b/middleware/markdown/metadata.go @@ -4,10 +4,10 @@ import ( "bytes" "encoding/json" "fmt" + "time" "github.com/BurntSushi/toml" "gopkg.in/yaml.v2" - "time" ) // Metadata stores a page's metadata @@ -178,8 +178,8 @@ func (y *YAMLMetadataParser) Closing() []byte { // It returns the metadata, the remaining bytes (markdown), and an error, if any. func extractMetadata(parser MetadataParser, b []byte) (metadata []byte, markdown []byte, err error) { b = bytes.TrimSpace(b) - openingLine := append(parser.Opening(), '\n') - closingLine := append(parser.Closing(), '\n') + openingLine := parser.Opening() + closingLine := parser.Closing() if !bytes.HasPrefix(b, openingLine) { return nil, b, fmt.Errorf("first line missing expected metadata identifier") } diff --git a/middleware/markdown/metadata_test.go b/middleware/markdown/metadata_test.go index b93c9ddcb..b80670169 100644 --- a/middleware/markdown/metadata_test.go +++ b/middleware/markdown/metadata_test.go @@ -8,72 +8,6 @@ import ( "testing" ) -var TOML = [4]string{` -title = "A title" -template = "default" -name = "value" -`, - `+++ -title = "A title" -template = "default" -name = "value" -+++ -Page content -`, - `+++ -title = "A title" -template = "default" -name = "value" - `, - `title = "A title" template = "default" [variables] name = "value"`, -} - -var YAML = [4]string{` -title : A title -template : default -name : value -`, - `--- -title : A title -template : default -name : value ---- -Page content -`, - `--- -title : A title -template : default -name : value -`, - `title : A title template : default variables : name : value`, -} -var JSON = [4]string{` - "title" : "A title", - "template" : "default", - "name" : "value" -`, - `{ - "title" : "A title", - "template" : "default", - "name" : "value" -} -Page content -`, - ` -{ - "title" : "A title", - "template" : "default", - "name" : "value" -`, - ` -{{ - "title" : "A title", - "template" : "default", - "name" : "value" -} - `, -} - func check(t *testing.T, err error) { if err != nil { t.Fatal(err) @@ -81,6 +15,71 @@ func check(t *testing.T, err error) { } func TestParsers(t *testing.T) { + var TOML = [4]string{` +title = "A title" +template = "default" +name = "value" +`, + `+++ +title = "A title" +template = "default" +name = "value" ++++ +Page content + `, + `+++ +title = "A title" +template = "default" +name = "value" + `, + `title = "A title" template = "default" [variables] name = "value"`, + } + + var YAML = [4]string{` +title : A title +template : default +name : value +`, + `--- +title : A title +template : default +name : value +--- + Page content + `, + `--- +title : A title +template : default +name : value + `, + `title : A title template : default variables : name : value`, + } + var JSON = [4]string{` +"title" : "A title", +"template" : "default", +"name" : "value" +`, + `{ + "title" : "A title", + "template" : "default", + "name" : "value" +} +Page content + `, + ` +{ + "title" : "A title", + "template" : "default", + "name" : "value" + `, + ` +{{ + "title" : "A title", + "template" : "default", + "name" : "value" +} + `, + } expected := Metadata{ Title: "A title", Template: "default", @@ -153,3 +152,44 @@ func TestParsers(t *testing.T) { } } + +func TestLargeBody(t *testing.T) { + var JSON = `{ +"template": "chapter" +} + +Mycket olika byggnader har man i de nordiska rikena: pyramidformiga, kilformiga, välvda, runda och fyrkantiga. De pyramidformiga består helt enkelt av träribbor, som upptill löper samman och nedtill bildar en vidare krets; de är avsedda att användas av hantverkarna under sommaren, för att de inte ska plågas av solen, på samma gång som de besväras av rök och eld. De kilformiga husen är i regel försedda med höga tak, för att de täta och tunga snömassorna fortare ska kunna blåsa av och inte tynga ned taken. Dessa är täckta av björknäver, tegel eller kluvet spån av furu - för kådans skull -, gran, ek eller bok; taken på de förmögnas hus däremot med plåtar av koppar eller bly, i likhet med kyrktaken. Valvbyggnaderna uppförs ganska konstnärligt till skydd mot våldsamma vindar och snöfall, görs av sten eller trä, och är avsedda för olika alldagliga viktiga ändamål. Liknande byggnader kan finnas i stormännens gårdar där de används som förvaringsrum för husgeråd och jordbruksredskap. De runda byggnaderna - som för övrigt är de högst sällsynta - används av konstnärer, som vid sitt arbete behöver ett jämnt fördelat ljus från taket. Vanligast är de fyrkantiga husen, vars grova bjälkar är synnerligen väl hopfogade i hörnen - ett sant mästerverk av byggnadskonst; även dessa har fönster högt uppe i taken, för att dagsljuset skall kunna strömma in och ge alla därinne full belysning. Stenhusen har dörröppningar i förhållande till byggnadens storlek, men smala fönstergluggar, som skydd mot den stränga kölden, frosten och snön. Vore de större och vidare, såsom fönstren i Italien, skulle husen i följd av den fint yrande snön, som röres upp av den starka blåsten, precis som dammet av virvelvinden, snart nog fyllas med massor av snö och inte kunna stå emot dess tryck, utan störta samman. + + ` + var TOML = `+++ +template = "chapter" ++++ + +Mycket olika byggnader har man i de nordiska rikena: pyramidformiga, kilformiga, välvda, runda och fyrkantiga. De pyramidformiga består helt enkelt av träribbor, som upptill löper samman och nedtill bildar en vidare krets; de är avsedda att användas av hantverkarna under sommaren, för att de inte ska plågas av solen, på samma gång som de besväras av rök och eld. De kilformiga husen är i regel försedda med höga tak, för att de täta och tunga snömassorna fortare ska kunna blåsa av och inte tynga ned taken. Dessa är täckta av björknäver, tegel eller kluvet spån av furu - för kådans skull -, gran, ek eller bok; taken på de förmögnas hus däremot med plåtar av koppar eller bly, i likhet med kyrktaken. Valvbyggnaderna uppförs ganska konstnärligt till skydd mot våldsamma vindar och snöfall, görs av sten eller trä, och är avsedda för olika alldagliga viktiga ändamål. Liknande byggnader kan finnas i stormännens gårdar där de används som förvaringsrum för husgeråd och jordbruksredskap. De runda byggnaderna - som för övrigt är de högst sällsynta - används av konstnärer, som vid sitt arbete behöver ett jämnt fördelat ljus från taket. Vanligast är de fyrkantiga husen, vars grova bjälkar är synnerligen väl hopfogade i hörnen - ett sant mästerverk av byggnadskonst; även dessa har fönster högt uppe i taken, för att dagsljuset skall kunna strömma in och ge alla därinne full belysning. Stenhusen har dörröppningar i förhållande till byggnadens storlek, men smala fönstergluggar, som skydd mot den stränga kölden, frosten och snön. Vore de större och vidare, såsom fönstren i Italien, skulle husen i följd av den fint yrande snön, som röres upp av den starka blåsten, precis som dammet av virvelvinden, snart nog fyllas med massor av snö och inte kunna stå emot dess tryck, utan störta samman. + + ` + var YAML = `--- +template : chapter +--- + +Mycket olika byggnader har man i de nordiska rikena: pyramidformiga, kilformiga, välvda, runda och fyrkantiga. De pyramidformiga består helt enkelt av träribbor, som upptill löper samman och nedtill bildar en vidare krets; de är avsedda att användas av hantverkarna under sommaren, för att de inte ska plågas av solen, på samma gång som de besväras av rök och eld. De kilformiga husen är i regel försedda med höga tak, för att de täta och tunga snömassorna fortare ska kunna blåsa av och inte tynga ned taken. Dessa är täckta av björknäver, tegel eller kluvet spån av furu - för kådans skull -, gran, ek eller bok; taken på de förmögnas hus däremot med plåtar av koppar eller bly, i likhet med kyrktaken. Valvbyggnaderna uppförs ganska konstnärligt till skydd mot våldsamma vindar och snöfall, görs av sten eller trä, och är avsedda för olika alldagliga viktiga ändamål. Liknande byggnader kan finnas i stormännens gårdar där de används som förvaringsrum för husgeråd och jordbruksredskap. De runda byggnaderna - som för övrigt är de högst sällsynta - används av konstnärer, som vid sitt arbete behöver ett jämnt fördelat ljus från taket. Vanligast är de fyrkantiga husen, vars grova bjälkar är synnerligen väl hopfogade i hörnen - ett sant mästerverk av byggnadskonst; även dessa har fönster högt uppe i taken, för att dagsljuset skall kunna strömma in och ge alla därinne full belysning. Stenhusen har dörröppningar i förhållande till byggnadens storlek, men smala fönstergluggar, som skydd mot den stränga kölden, frosten och snön. Vore de större och vidare, såsom fönstren i Italien, skulle husen i följd av den fint yrande snön, som röres upp av den starka blåsten, precis som dammet av virvelvinden, snart nog fyllas med massor av snö och inte kunna stå emot dess tryck, utan störta samman. + + ` + var expectedBody = `Mycket olika byggnader har man i de nordiska rikena: pyramidformiga, kilformiga, välvda, runda och fyrkantiga. De pyramidformiga består helt enkelt av träribbor, som upptill löper samman och nedtill bildar en vidare krets; de är avsedda att användas av hantverkarna under sommaren, för att de inte ska plågas av solen, på samma gång som de besväras av rök och eld. De kilformiga husen är i regel försedda med höga tak, för att de täta och tunga snömassorna fortare ska kunna blåsa av och inte tynga ned taken. Dessa är täckta av björknäver, tegel eller kluvet spån av furu - för kådans skull -, gran, ek eller bok; taken på de förmögnas hus däremot med plåtar av koppar eller bly, i likhet med kyrktaken. Valvbyggnaderna uppförs ganska konstnärligt till skydd mot våldsamma vindar och snöfall, görs av sten eller trä, och är avsedda för olika alldagliga viktiga ändamål. Liknande byggnader kan finnas i stormännens gårdar där de används som förvaringsrum för husgeråd och jordbruksredskap. De runda byggnaderna - som för övrigt är de högst sällsynta - används av konstnärer, som vid sitt arbete behöver ett jämnt fördelat ljus från taket. Vanligast är de fyrkantiga husen, vars grova bjälkar är synnerligen väl hopfogade i hörnen - ett sant mästerverk av byggnadskonst; även dessa har fönster högt uppe i taken, för att dagsljuset skall kunna strömma in och ge alla därinne full belysning. Stenhusen har dörröppningar i förhållande till byggnadens storlek, men smala fönstergluggar, som skydd mot den stränga kölden, frosten och snön. Vore de större och vidare, såsom fönstren i Italien, skulle husen i följd av den fint yrande snön, som röres upp av den starka blåsten, precis som dammet av virvelvinden, snart nog fyllas med massor av snö och inte kunna stå emot dess tryck, utan störta samman. +` + data := []struct { + parser MetadataParser + testData string + name string + }{ + {&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"}, + {&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, YAML, "yaml"}, + {&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, TOML, "toml"}, + } + for _, v := range data { + // metadata without identifiers + if md, err := v.parser.Parse([]byte(v.testData)); err != nil || strings.TrimSpace(string(md)) != strings.TrimSpace(expectedBody) { + t.Fatalf("Error not expected and/or markdown not equal for %v", v.name) + } + } +} From fd14f257df1b1e1862c24705628cb79ab4a68a8e Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 2 Dec 2015 17:04:58 -0700 Subject: [PATCH 05/44] markdown: Add (currently failing) test for empty body --- middleware/markdown/metadata_test.go | 69 +++++++++++++++++++--------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/middleware/markdown/metadata_test.go b/middleware/markdown/metadata_test.go index b80670169..285caf972 100644 --- a/middleware/markdown/metadata_test.go +++ b/middleware/markdown/metadata_test.go @@ -14,72 +14,92 @@ func check(t *testing.T, err error) { } } -func TestParsers(t *testing.T) { - var TOML = [4]string{` +var TOML = [5]string{` title = "A title" template = "default" name = "value" `, - `+++ + `+++ title = "A title" template = "default" name = "value" +++ Page content `, - `+++ + `+++ title = "A title" template = "default" name = "value" `, - `title = "A title" template = "default" [variables] name = "value"`, - } + `title = "A title" template = "default" [variables] name = "value"`, + `+++ +title = "A title" +template = "default" +name = "value" ++++ +`, +} - var YAML = [4]string{` +var YAML = [5]string{` title : A title template : default name : value `, - `--- + `--- title : A title template : default name : value --- Page content `, - `--- + `--- title : A title template : default name : value `, - `title : A title template : default variables : name : value`, - } - var JSON = [4]string{` -"title" : "A title", -"template" : "default", -"name" : "value" + `title : A title template : default variables : name : value`, + `--- +title : A title +template : default +name : value +--- `, - `{ +} + +var JSON = [5]string{` + "title" : "A title", + "template" : "default", + "name" : "value" +`, + `{ "title" : "A title", "template" : "default", "name" : "value" } Page content `, - ` + ` { "title" : "A title", "template" : "default", "name" : "value" `, - ` -{{ + ` +{ + "title" :: "A title", + "template" : "default", + "name" : "value" +} + `, + `{ "title" : "A title", "template" : "default", "name" : "value" } - `, - } +`, +} + +func TestParsers(t *testing.T) { expected := Metadata{ Title: "A title", Template: "default", @@ -106,7 +126,7 @@ Page content data := []struct { parser MetadataParser - testData [4]string + testData [5]string name string }{ {&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"}, @@ -149,6 +169,11 @@ Page content if md, err = v.parser.Parse([]byte(v.testData[3])); err == nil { t.Fatalf("Expected error for invalid metadata for %v", v.name) } + + // front matter but no body + if md, err = v.parser.Parse([]byte(v.testData[4])); err != nil { + t.Fatalf("Unexpected error for valid metadata but no body for %v", v.name) + } } } From 5b93799a6230357827335497b7a09f8006ce9b57 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 3 Dec 2015 19:52:15 -0700 Subject: [PATCH 06/44] Version 0.8.0 --- dist/CHANGES.txt | 6 +++--- dist/README.txt | 2 +- main.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dist/CHANGES.txt b/dist/CHANGES.txt index 01657febf..97542ec7f 100644 --- a/dist/CHANGES.txt +++ b/dist/CHANGES.txt @@ -1,8 +1,8 @@ CHANGES -0.8 beta -- Let's Encrypt (free, automatic, fully-managed HTTPS for your sites) -- Graceful restarts (for POSIX-compatible systems) +0.8.0 (December 4, 2015) +- HTTPS by default via Let's Encrypt (certs & keys are fully managed) +- Graceful restarts (on POSIX-compliant systems) - Major internal refactoring to allow use of Caddy as library - New directive 'mime' to customize Content-Type based on file extension - New -accept flag to accept Let's Encrypt SA without prompt diff --git a/dist/README.txt b/dist/README.txt index 0fd9d1a03..aa0a0e784 100644 --- a/dist/README.txt +++ b/dist/README.txt @@ -1,4 +1,4 @@ -CADDY 0.8 beta 4 +CADDY 0.8 Website https://caddyserver.com diff --git a/main.go b/main.go index b08d101ae..f0055d422 100644 --- a/main.go +++ b/main.go @@ -26,13 +26,13 @@ var ( const ( appName = "Caddy" - appVersion = "0.8 beta 4" + appVersion = "0.8" ) func init() { caddy.TrapSignals() flag.BoolVar(&letsencrypt.Agreed, "agree", false, "Agree to Let's Encrypt Subscriber Agreement") - flag.StringVar(&letsencrypt.CAUrl, "ca", "https://acme-staging.api.letsencrypt.org/directory", "Certificate authority ACME server") + flag.StringVar(&letsencrypt.CAUrl, "ca", "https://acme-v01.api.letsencrypt.org/directory", "Certificate authority ACME server") flag.StringVar(&conf, "conf", "", "Configuration file to use (default="+caddy.DefaultConfigFile+")") flag.StringVar(&cpu, "cpu", "100%", "CPU cap") flag.StringVar(&letsencrypt.DefaultEmail, "email", "", "Default Let's Encrypt account email address") From b3f5e4d4ad08a22900c4f11b641c0aadfcd2eee9 Mon Sep 17 00:00:00 2001 From: ReadmeCritic Date: Fri, 4 Dec 2015 07:20:56 -0800 Subject: [PATCH 07/44] Update README URLs based on HTTP redirects --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6698dccf6..cb81c1751 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ You may also be interested in the [developer guide] ## Running from Source -Note: You will need **[Go 1.4](https://golang.org/dl)** or a later version. +Note: You will need **[Go 1.4](https://golang.org/dl/)** or a later version. 1. `$ go get github.com/mholt/caddy` 2. `cd` into your website's directory @@ -120,11 +120,11 @@ ports < 1024 like 80 and 443. Caddy is available as a Docker container from any of these sources: -- [abiosoft/caddy](https://registry.hub.docker.com/u/abiosoft/caddy/) -- [darron/caddy](https://registry.hub.docker.com/u/darron/caddy/) -- [joshix/caddy](https://registry.hub.docker.com/u/joshix/caddy/) -- [jumanjiman/caddy](https://registry.hub.docker.com/u/jumanjiman/caddy/) -- [zenithar/nano-caddy](https://registry.hub.docker.com/u/zenithar/nano-caddy/) +- [abiosoft/caddy](https://hub.docker.com/r/abiosoft/caddy/) +- [darron/caddy](https://hub.docker.com/r/darron/caddy/) +- [joshix/caddy](https://hub.docker.com/r/joshix/caddy/) +- [jumanjiman/caddy](https://hub.docker.com/r/jumanjiman/caddy/) +- [zenithar/nano-caddy](https://hub.docker.com/r/zenithar/nano-caddy/) From 12f594779c9b8a23b69aefe5363e1eac087db646 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Fri, 4 Dec 2015 23:04:12 +0100 Subject: [PATCH 08/44] Added support magic characters in import pattern Import now allows to use the star wildcard, question mark and square brackets as used by filepath.Glob --- caddy/parse/parsing.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index 03d9d800a..e71a03f5b 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -184,11 +184,26 @@ func (p *parser) doImport() error { if !p.NextArg() { return p.ArgErr() } - importFile := p.Val() + importPattern := p.Val() if p.NextArg() { return p.Err("Import allows only one file to import") } + matches, err := filepath.Glob(importPattern) + if err != nil { + return p.Errf("Failed to use import pattern %s - %s", importPattern, err.Error()) + } + + for _, importFile := range matches { + if err := p.doSingleImport(importFile); err != nil { + return err + } + } + + return nil +} + +func (p *parser) doSingleImport(importFile string) error { file, err := os.Open(importFile) if err != nil { return p.Errf("Could not import %s - %v", importFile, err) From d1216f409dd8f7278757be951277758347e66172 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Fri, 4 Dec 2015 23:21:28 +0100 Subject: [PATCH 09/44] Handle no matches --- caddy/parse/parsing.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index e71a03f5b..8aa711c3f 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -194,6 +194,10 @@ func (p *parser) doImport() error { return p.Errf("Failed to use import pattern %s - %s", importPattern, err.Error()) } + if len(matches) == 0 { + return p.Errf("No files matching the import pattern %s", importPattern) + } + for _, importFile := range matches { if err := p.doSingleImport(importFile); err != nil { return err From e17a18365d4ef83d8865065b6cad772c825a3c83 Mon Sep 17 00:00:00 2001 From: Carsten Hagemann Date: Sat, 5 Dec 2015 13:05:51 +0100 Subject: [PATCH 10/44] Latest Go 1.5 version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 514e3a1ca..19ba6dbab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.4.3 - - 1.5.1 + - 1.5.2 - tip install: From d56a9a1c5dcbfa3191de905884c7016feb168936 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Sun, 6 Dec 2015 23:49:21 +0100 Subject: [PATCH 11/44] Correct position of the newly imported tokens --- caddy/parse/parsing.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index 8aa711c3f..e011fd8cc 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -198,12 +198,23 @@ func (p *parser) doImport() error { return p.Errf("No files matching the import pattern %s", importPattern) } + // Splice out the import directive and its argument (2 tokens total) + // and insert the imported tokens in their place. + tokensBefore := p.tokens[:p.cursor-1] + tokensAfter := p.tokens[p.cursor+1:] + // cursor was advanced one position to read filename; rewind it + p.cursor-- + + p.tokens = tokensBefore + for _, importFile := range matches { if err := p.doSingleImport(importFile); err != nil { return err } } + p.tokens = append(p.tokens, append(tokensAfter)...) + return nil } @@ -222,10 +233,7 @@ func (p *parser) doSingleImport(importFile string) error { // Splice out the import directive and its argument (2 tokens total) // and insert the imported tokens in their place. - tokensBefore := p.tokens[:p.cursor-1] - tokensAfter := p.tokens[p.cursor+1:] - p.tokens = append(tokensBefore, append(importedTokens, tokensAfter...)...) - p.cursor-- // cursor was advanced one position to read the filename; rewind it + p.tokens = append(p.tokens, append(importedTokens)...) return nil } From ab5087e215271bb9b6d64f84ee7d69194eec2827 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Mon, 7 Dec 2015 23:17:05 +0100 Subject: [PATCH 12/44] Gzip: support for min_length. --- caddy/setup/gzip.go | 30 ++++++-- caddy/setup/gzip_test.go | 12 ++++ middleware/gzip/gzip.go | 20 ++++-- middleware/gzip/gzip_test.go | 2 +- .../gzip/{filter.go => request_filter.go} | 8 +-- ...{filter_test.go => request_filter_test.go} | 4 +- middleware/gzip/response_filter.go | 62 ++++++++++++++++ middleware/gzip/response_filter_test.go | 70 +++++++++++++++++++ 8 files changed, 192 insertions(+), 16 deletions(-) rename middleware/gzip/{filter.go => request_filter.go} (91%) rename middleware/gzip/{filter_test.go => request_filter_test.go} (95%) create mode 100644 middleware/gzip/response_filter.go create mode 100644 middleware/gzip/response_filter_test.go diff --git a/caddy/setup/gzip.go b/caddy/setup/gzip.go index a81c5f170..40c81252f 100644 --- a/caddy/setup/gzip.go +++ b/caddy/setup/gzip.go @@ -27,9 +27,13 @@ func gzipParse(c *Controller) ([]gzip.Config, error) { for c.Next() { config := gzip.Config{} + // Request Filters pathFilter := gzip.PathFilter{IgnoredPaths: make(gzip.Set)} extFilter := gzip.ExtFilter{Exts: make(gzip.Set)} + // Response Filters + lengthFilter := gzip.LengthFilter(0) + // No extra args expected if len(c.RemainingArgs()) > 0 { return configs, c.ArgErr() @@ -68,24 +72,42 @@ func gzipParse(c *Controller) ([]gzip.Config, error) { } level, _ := strconv.Atoi(c.Val()) config.Level = level + case "min_length": + if !c.NextArg() { + return configs, c.ArgErr() + } + length, err := strconv.ParseInt(c.Val(), 10, 64) + if err != nil { + return configs, err + } else if length == 0 { + return configs, fmt.Errorf(`gzip: min_length must be greater than 0`) + } + lengthFilter = gzip.LengthFilter(length) default: return configs, c.ArgErr() } } - config.Filters = []gzip.Filter{} + // Request Filters + config.RequestFilters = []gzip.RequestFilter{} // If ignored paths are specified, put in front to filter with path first if len(pathFilter.IgnoredPaths) > 0 { - config.Filters = []gzip.Filter{pathFilter} + config.RequestFilters = []gzip.RequestFilter{pathFilter} } // Then, if extensions are specified, use those to filter. // Otherwise, use default extensions filter. if len(extFilter.Exts) > 0 { - config.Filters = append(config.Filters, extFilter) + config.RequestFilters = append(config.RequestFilters, extFilter) } else { - config.Filters = append(config.Filters, gzip.DefaultExtFilter()) + config.RequestFilters = append(config.RequestFilters, gzip.DefaultExtFilter()) + } + + // Response Filters + // If min_length is specified, use it. + if int64(lengthFilter) != 0 { + config.ResponseFilters = append(config.ResponseFilters, lengthFilter) } configs = append(configs, config) diff --git a/caddy/setup/gzip_test.go b/caddy/setup/gzip_test.go index 22d01d7a1..36eeb0aea 100644 --- a/caddy/setup/gzip_test.go +++ b/caddy/setup/gzip_test.go @@ -73,6 +73,18 @@ func TestGzip(t *testing.T) { level 1 } `, false}, + {`gzip { not /file + ext * + level 1 + min_length ab + } + `, true}, + {`gzip { not /file + ext * + level 1 + min_length 1000 + } + `, false}, } for i, test := range tests { c := NewTestController(test.input) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index e2d447753..b5866f682 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -23,8 +23,9 @@ type Gzip struct { // Config holds the configuration for Gzip middleware type Config struct { - Filters []Filter // Filters to use - Level int // Compression level + RequestFilters []RequestFilter + ResponseFilters []ResponseFilter + Level int // Compression level } // ServeHTTP serves a gzipped response if the client supports it. @@ -36,8 +37,8 @@ func (g Gzip) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { outer: for _, c := range g.Configs { - // Check filters to determine if gzipping is permitted for this request - for _, filter := range c.Filters { + // Check request filters to determine if gzipping is permitted for this request + for _, filter := range c.RequestFilters { if !filter.ShouldCompress(r) { continue outer } @@ -56,8 +57,17 @@ outer: defer gzipWriter.Close() gz := gzipResponseWriter{Writer: gzipWriter, ResponseWriter: w} + var rw http.ResponseWriter + // if no response filter is used + if len(c.ResponseFilters) == 0 { + rw = gz + } else { + // wrap gzip writer with ResponseFilterWriter + rw = NewResponseFilterWriter(c.ResponseFilters, gz) + } + // Any response in forward middleware will now be compressed - status, err := g.Next.ServeHTTP(gz, r) + status, err := g.Next.ServeHTTP(rw, r) // If there was an error that remained unhandled, we need // to send something back before gzipWriter gets closed at diff --git a/middleware/gzip/gzip_test.go b/middleware/gzip/gzip_test.go index 3e7bed996..11ce6b209 100644 --- a/middleware/gzip/gzip_test.go +++ b/middleware/gzip/gzip_test.go @@ -21,7 +21,7 @@ func TestGzipHandler(t *testing.T) { extFilter.Exts.Add(e) } gz := Gzip{Configs: []Config{ - {Filters: []Filter{pathFilter, extFilter}}, + {RequestFilters: []RequestFilter{pathFilter, extFilter}}, }} w := httptest.NewRecorder() diff --git a/middleware/gzip/filter.go b/middleware/gzip/request_filter.go similarity index 91% rename from middleware/gzip/filter.go rename to middleware/gzip/request_filter.go index f4039da35..76a9ef83c 100644 --- a/middleware/gzip/filter.go +++ b/middleware/gzip/request_filter.go @@ -7,8 +7,8 @@ import ( "github.com/mholt/caddy/middleware" ) -// Filter determines if a request should be gzipped. -type Filter interface { +// RequestFilter determines if a request should be gzipped. +type RequestFilter interface { // ShouldCompress tells if gzip compression // should be done on the request. ShouldCompress(*http.Request) bool @@ -26,7 +26,7 @@ func DefaultExtFilter() ExtFilter { return m } -// ExtFilter is Filter for file name extensions. +// ExtFilter is RequestFilter for file name extensions. type ExtFilter struct { // Exts is the file name extensions to accept Exts Set @@ -43,7 +43,7 @@ func (e ExtFilter) ShouldCompress(r *http.Request) bool { return e.Exts.Contains(ExtWildCard) || e.Exts.Contains(ext) } -// PathFilter is Filter for request path. +// PathFilter is RequestFilter for request path. type PathFilter struct { // IgnoredPaths is the paths to ignore IgnoredPaths Set diff --git a/middleware/gzip/filter_test.go b/middleware/gzip/request_filter_test.go similarity index 95% rename from middleware/gzip/filter_test.go rename to middleware/gzip/request_filter_test.go index f6537b9e2..ce31d7faf 100644 --- a/middleware/gzip/filter_test.go +++ b/middleware/gzip/request_filter_test.go @@ -47,7 +47,7 @@ func TestSet(t *testing.T) { } func TestExtFilter(t *testing.T) { - var filter Filter = ExtFilter{make(Set)} + var filter RequestFilter = ExtFilter{make(Set)} for _, e := range []string{".txt", ".html", ".css", ".md"} { filter.(ExtFilter).Exts.Add(e) } @@ -86,7 +86,7 @@ func TestPathFilter(t *testing.T) { paths := []string{ "/a", "/b", "/c", "/de", } - var filter Filter = PathFilter{make(Set)} + var filter RequestFilter = PathFilter{make(Set)} for _, p := range paths { filter.(PathFilter).IgnoredPaths.Add(p) } diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go new file mode 100644 index 000000000..8793dabe1 --- /dev/null +++ b/middleware/gzip/response_filter.go @@ -0,0 +1,62 @@ +package gzip + +import ( + "net/http" + "strconv" +) + +// ResponseFilter determines if the response should be gzipped. +type ResponseFilter interface { + ShouldCompress(http.ResponseWriter) bool +} + +// LengthFilter is ResponseFilter for minimum content length. +type LengthFilter int64 + +// ShouldCompress returns if content length is greater than or +// equals to minimum length. +func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { + contentLength := (w.Header().Get("Content-Length")) + length, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil || length == 0 { + return false + } + return l != 0 && int64(l) <= length +} + +// ResponseFilterWriter validates ResponseFilters. It writes +// gzip compressed data if ResponseFilters are satisfied or +// uncompressed data otherwise. +type ResponseFilterWriter struct { + filters []ResponseFilter + validated bool + shouldCompress bool + gzipResponseWriter +} + +// NewResponseFilterWriter creates and initializes a new ResponseFilterWriter. +func NewResponseFilterWriter(filters []ResponseFilter, gz gzipResponseWriter) *ResponseFilterWriter { + return &ResponseFilterWriter{filters: filters, gzipResponseWriter: gz} +} + +// Write wraps underlying Write method and compresses if filters +// are satisfied +func (r *ResponseFilterWriter) Write(b []byte) (int, error) { + // One time validation to determine if compression should + // be used or not. + if !r.validated { + r.shouldCompress = true + for _, filter := range r.filters { + if !filter.ShouldCompress(r) { + r.shouldCompress = false + break + } + } + r.validated = true + } + + if r.shouldCompress { + return r.gzipResponseWriter.Write(b) + } + return r.ResponseWriter.Write(b) +} diff --git a/middleware/gzip/response_filter_test.go b/middleware/gzip/response_filter_test.go new file mode 100644 index 000000000..1a5a1b4f3 --- /dev/null +++ b/middleware/gzip/response_filter_test.go @@ -0,0 +1,70 @@ +package gzip + +import ( + "compress/gzip" + "fmt" + "net/http/httptest" + "testing" +) + +func TestLengthFilter(t *testing.T) { + var filters []ResponseFilter = []ResponseFilter{ + LengthFilter(100), + LengthFilter(1000), + LengthFilter(0), + } + + var tests = []struct { + length int64 + shouldCompress [3]bool + }{ + {20, [3]bool{false, false, false}}, + {50, [3]bool{false, false, false}}, + {100, [3]bool{true, false, false}}, + {500, [3]bool{true, false, false}}, + {1000, [3]bool{true, true, false}}, + {1500, [3]bool{true, true, false}}, + } + + for i, ts := range tests { + for j, filter := range filters { + r := httptest.NewRecorder() + r.Header().Set("Content-Length", fmt.Sprint(ts.length)) + if filter.ShouldCompress(r) != ts.shouldCompress[j] { + t.Errorf("Test %v: Expected %v found %v", i, ts.shouldCompress[j], filter.ShouldCompress(r)) + } + } + } +} + +func TestResponseFilterWriter(t *testing.T) { + tests := []struct { + body string + shouldCompress bool + }{ + {"Hello\t\t\t\n", false}, + {"Hello the \t\t\t world is\n\n\n great", true}, + {"Hello \t\t\nfrom gzip", true}, + {"Hello gzip\n", false}, + } + filters := []ResponseFilter{ + LengthFilter(15), + } + for i, ts := range tests { + w := httptest.NewRecorder() + w.Header().Set("Content-Length", fmt.Sprint(len(ts.body))) + gz := gzipResponseWriter{gzip.NewWriter(w), w} + rw := NewResponseFilterWriter(filters, gz) + rw.Write([]byte(ts.body)) + resp := w.Body.String() + if !ts.shouldCompress { + if resp != ts.body { + t.Errorf("Test %v: No compression expected, found %v", i, resp) + } + } else { + if resp == ts.body { + t.Errorf("Test %v: Compression expected, found %v", i, resp) + } + } + } +} From 8631f33940a8c50b154da2756ce7d4954b925f0a Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Mon, 7 Dec 2015 23:27:57 +0100 Subject: [PATCH 13/44] remove minor ugly parenthesis --- middleware/gzip/response_filter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index 8793dabe1..fcfc8be51 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -16,7 +16,7 @@ type LengthFilter int64 // ShouldCompress returns if content length is greater than or // equals to minimum length. func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { - contentLength := (w.Header().Get("Content-Length")) + contentLength := w.Header().Get("Content-Length") length, err := strconv.ParseInt(contentLength, 10, 64) if err != nil || length == 0 { return false From 23631cfaca8526192a7d18ce619a4fd8e46b4289 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 8 Dec 2015 12:01:24 +0100 Subject: [PATCH 14/44] Fix deleted Content-Length header bug. --- middleware/gzip/gzip.go | 17 ++++++++--- middleware/gzip/gzip_test.go | 2 ++ middleware/gzip/response_filter.go | 38 ++++++++++++++++--------- middleware/gzip/response_filter_test.go | 30 +++++++++++++++---- 4 files changed, 65 insertions(+), 22 deletions(-) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index b5866f682..65256b2ff 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -3,6 +3,7 @@ package gzip import ( + "bytes" "compress/gzip" "fmt" "io" @@ -47,9 +48,13 @@ outer: // Delete this header so gzipping is not repeated later in the chain r.Header.Del("Accept-Encoding") - w.Header().Set("Content-Encoding", "gzip") - w.Header().Set("Vary", "Accept-Encoding") - gzipWriter, err := newWriter(c, w) + // gzipWriter modifies underlying writer at init, + // use a buffer instead to leave ResponseWriter in + // original form. + var buf = &bytes.Buffer{} + defer buf.Reset() + + gzipWriter, err := newWriter(c, buf) if err != nil { // should not happen return http.StatusInternalServerError, err @@ -60,6 +65,8 @@ outer: var rw http.ResponseWriter // if no response filter is used if len(c.ResponseFilters) == 0 { + // replace buffer with ResponseWriter + gzipWriter.Reset(w) rw = gz } else { // wrap gzip writer with ResponseFilterWriter @@ -88,7 +95,7 @@ outer: // newWriter create a new Gzip Writer based on the compression level. // If the level is valid (i.e. between 1 and 9), it uses the level. // Otherwise, it uses default compression level. -func newWriter(c Config, w http.ResponseWriter) (*gzip.Writer, error) { +func newWriter(c Config, w io.Writer) (*gzip.Writer, error) { if c.Level >= gzip.BestSpeed && c.Level <= gzip.BestCompression { return gzip.NewWriterLevel(w, c.Level) } @@ -108,6 +115,8 @@ type gzipResponseWriter struct { // be wrong because it doesn't know it's being gzipped. func (w gzipResponseWriter) WriteHeader(code int) { w.Header().Del("Content-Length") + w.Header().Set("Content-Encoding", "gzip") + w.Header().Set("Vary", "Accept-Encoding") w.ResponseWriter.WriteHeader(code) } diff --git a/middleware/gzip/gzip_test.go b/middleware/gzip/gzip_test.go index 11ce6b209..c35c99c63 100644 --- a/middleware/gzip/gzip_test.go +++ b/middleware/gzip/gzip_test.go @@ -80,6 +80,8 @@ func TestGzipHandler(t *testing.T) { func nextFunc(shouldGzip bool) middleware.Handler { return middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { + w.WriteHeader(200) + w.Write([]byte("test")) if shouldGzip { if r.Header.Get("Accept-Encoding") != "" { return 0, fmt.Errorf("Accept-Encoding header not expected") diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index fcfc8be51..034fd6123 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -1,6 +1,7 @@ package gzip import ( + "compress/gzip" "net/http" "strconv" ) @@ -29,7 +30,6 @@ func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { // uncompressed data otherwise. type ResponseFilterWriter struct { filters []ResponseFilter - validated bool shouldCompress bool gzipResponseWriter } @@ -40,21 +40,33 @@ func NewResponseFilterWriter(filters []ResponseFilter, gz gzipResponseWriter) *R } // Write wraps underlying Write method and compresses if filters -// are satisfied -func (r *ResponseFilterWriter) Write(b []byte) (int, error) { - // One time validation to determine if compression should - // be used or not. - if !r.validated { - r.shouldCompress = true - for _, filter := range r.filters { - if !filter.ShouldCompress(r) { - r.shouldCompress = false - break - } +// are satisfied. +func (r *ResponseFilterWriter) WriteHeader(code int) { + // Determine if compression should be used or not. + r.shouldCompress = true + for _, filter := range r.filters { + if !filter.ShouldCompress(r) { + r.shouldCompress = false + break } - r.validated = true } + if r.shouldCompress { + // replace buffer with ResponseWriter + if gzWriter, ok := r.gzipResponseWriter.Writer.(*gzip.Writer); ok { + gzWriter.Reset(r.ResponseWriter) + } + // use gzip WriteHeader to include and delete + // necessary headers + r.gzipResponseWriter.WriteHeader(code) + } else { + r.ResponseWriter.WriteHeader(code) + } +} + +// Write wraps underlying Write method and compresses if filters +// are satisfied +func (r *ResponseFilterWriter) Write(b []byte) (int, error) { if r.shouldCompress { return r.gzipResponseWriter.Write(b) } diff --git a/middleware/gzip/response_filter_test.go b/middleware/gzip/response_filter_test.go index 1a5a1b4f3..cd7c71917 100644 --- a/middleware/gzip/response_filter_test.go +++ b/middleware/gzip/response_filter_test.go @@ -3,8 +3,11 @@ package gzip import ( "compress/gzip" "fmt" + "net/http" "net/http/httptest" "testing" + + "github.com/mholt/caddy/middleware" ) func TestLengthFilter(t *testing.T) { @@ -30,7 +33,8 @@ func TestLengthFilter(t *testing.T) { for j, filter := range filters { r := httptest.NewRecorder() r.Header().Set("Content-Length", fmt.Sprint(ts.length)) - if filter.ShouldCompress(r) != ts.shouldCompress[j] { + wWriter := NewResponseFilterWriter([]ResponseFilter{filter}, gzipResponseWriter{gzip.NewWriter(r), r}) + if filter.ShouldCompress(wWriter) != ts.shouldCompress[j] { t.Errorf("Test %v: Expected %v found %v", i, ts.shouldCompress[j], filter.ShouldCompress(r)) } } @@ -47,16 +51,32 @@ func TestResponseFilterWriter(t *testing.T) { {"Hello \t\t\nfrom gzip", true}, {"Hello gzip\n", false}, } + filters := []ResponseFilter{ LengthFilter(15), } + + server := Gzip{Configs: []Config{ + {ResponseFilters: filters}, + }} + for i, ts := range tests { + server.Next = middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { + w.Header().Set("Content-Length", fmt.Sprint(len(ts.body))) + w.WriteHeader(200) + w.Write([]byte(ts.body)) + return 200, nil + }) + + r := urlRequest("/") + r.Header.Set("Accept-Encoding", "gzip") + w := httptest.NewRecorder() - w.Header().Set("Content-Length", fmt.Sprint(len(ts.body))) - gz := gzipResponseWriter{gzip.NewWriter(w), w} - rw := NewResponseFilterWriter(filters, gz) - rw.Write([]byte(ts.body)) + + server.ServeHTTP(w, r) + resp := w.Body.String() + if !ts.shouldCompress { if resp != ts.body { t.Errorf("Test %v: No compression expected, found %v", i, resp) From b6c4178f0ab7c49abd6e795708beddaf7b766137 Mon Sep 17 00:00:00 2001 From: Pavel Pavlenko Date: Wed, 9 Dec 2015 11:10:55 +0300 Subject: [PATCH 15/44] Remove ECDHE-RSA-3DES-EDE-CBC-SHA and RSA-3DES-EDE-CBC-SHA from the default TLS config --- caddy/setup/tls.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/caddy/setup/tls.go b/caddy/setup/tls.go index 4e5f7f9d2..abec2d523 100644 --- a/caddy/setup/tls.go +++ b/caddy/setup/tls.go @@ -91,6 +91,9 @@ func SetDefaultTLSParams(c *server.Config) { // If no ciphers provided, use all that Caddy supports for the protocol if len(c.TLS.Ciphers) == 0 { c.TLS.Ciphers = supportedCiphers + + // Remove ECDHE-RSA-3DES-EDE-CBC-SHA and RSA-3DES-EDE-CBC-SHA from the default TLS config + c.TLS.Ciphers = c.TLS.Ciphers[:len(c.TLS.Ciphers)-2] } // Not a cipher suite, but still important for mitigating protocol downgrade attacks From e4ff77ed079a2a993b389097ff8dca259ea70404 Mon Sep 17 00:00:00 2001 From: Pavel Pavlenko Date: Wed, 9 Dec 2015 11:27:59 +0300 Subject: [PATCH 16/44] fix tls_test.go --- caddy/setup/tls_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/caddy/setup/tls_test.go b/caddy/setup/tls_test.go index fdea1e0c7..629937016 100644 --- a/caddy/setup/tls_test.go +++ b/caddy/setup/tls_test.go @@ -42,15 +42,15 @@ func TestTLSParseBasic(t *testing.T) { tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, tls.TLS_RSA_WITH_AES_256_CBC_SHA, tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + //tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + //tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, tls.TLS_FALLBACK_SCSV, } // Ensure count is correct (plus one for TLS_FALLBACK_SCSV) - if len(c.TLS.Ciphers) != len(supportedCiphers)+1 { + if len(c.TLS.Ciphers) != len(supportedCiphers)-1 { t.Errorf("Expected %v Ciphers (including TLS_FALLBACK_SCSV), got %v", - len(supportedCiphers)+1, len(c.TLS.Ciphers)) + len(supportedCiphers)-1, len(c.TLS.Ciphers)) } // Ensure ordering is correct From a44d59f1e553ed23a489da63c64f0932c7ccc295 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 9 Dec 2015 18:44:25 +0100 Subject: [PATCH 17/44] Use ioutil.Discard instead for unneeded bytes. --- middleware/gzip/gzip.go | 11 ++++------- middleware/gzip/response_filter.go | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index 65256b2ff..147d739f8 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -3,10 +3,10 @@ package gzip import ( - "bytes" "compress/gzip" "fmt" "io" + "io/ioutil" "net/http" "strings" @@ -49,12 +49,9 @@ outer: r.Header.Del("Accept-Encoding") // gzipWriter modifies underlying writer at init, - // use a buffer instead to leave ResponseWriter in + // use a discard writer instead to leave ResponseWriter in // original form. - var buf = &bytes.Buffer{} - defer buf.Reset() - - gzipWriter, err := newWriter(c, buf) + gzipWriter, err := newWriter(c, ioutil.Discard) if err != nil { // should not happen return http.StatusInternalServerError, err @@ -65,7 +62,7 @@ outer: var rw http.ResponseWriter // if no response filter is used if len(c.ResponseFilters) == 0 { - // replace buffer with ResponseWriter + // replace discard writer with ResponseWriter gzipWriter.Reset(w) rw = gz } else { diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index 034fd6123..c599b3e1b 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -52,7 +52,7 @@ func (r *ResponseFilterWriter) WriteHeader(code int) { } if r.shouldCompress { - // replace buffer with ResponseWriter + // replace discard writer with ResponseWriter if gzWriter, ok := r.gzipResponseWriter.Writer.(*gzip.Writer); ok { gzWriter.Reset(r.ResponseWriter) } From 59dbea768c75a7ce07453010fcf6c392738771b8 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 9 Dec 2015 19:22:20 +0100 Subject: [PATCH 18/44] Fix race condition on AppVeyor. Increase timeout a bit. --- middleware/markdown/watcher_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/middleware/markdown/watcher_test.go b/middleware/markdown/watcher_test.go index d09d07c64..2f7ca4cba 100644 --- a/middleware/markdown/watcher_test.go +++ b/middleware/markdown/watcher_test.go @@ -18,7 +18,7 @@ func TestWatcher(t *testing.T) { out += fmt.Sprint(i) }) // wait little more because of concurrency - time.Sleep(interval * 9) + time.Sleep(interval * 12) stopChan <- struct{}{} if !strings.HasPrefix(out, expected) { t.Fatalf("Expected to have prefix %v, found %v", expected, out) @@ -32,7 +32,7 @@ func TestWatcher(t *testing.T) { out += fmt.Sprint(i) mu.Unlock() }) - time.Sleep(interval * 10) + time.Sleep(interval * 15) mu.Lock() res := out mu.Unlock() From 2e295b51b3344ead5a980827a8f26d9ec9c9b1de Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 9 Dec 2015 20:18:18 +0100 Subject: [PATCH 19/44] Use channel instead for a synchronous interval. --- middleware/markdown/watcher_test.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/middleware/markdown/watcher_test.go b/middleware/markdown/watcher_test.go index 2f7ca4cba..8a89e007f 100644 --- a/middleware/markdown/watcher_test.go +++ b/middleware/markdown/watcher_test.go @@ -13,14 +13,14 @@ func TestWatcher(t *testing.T) { interval := time.Millisecond * 100 i := 0 out := "" + syncChan := make(chan struct{}) stopChan := TickerFunc(interval, func() { i++ out += fmt.Sprint(i) + syncChan <- struct{}{} }) - // wait little more because of concurrency - time.Sleep(interval * 12) - stopChan <- struct{}{} - if !strings.HasPrefix(out, expected) { + sleepInSync(8, syncChan, stopChan) + if out != expected { t.Fatalf("Expected to have prefix %v, found %v", expected, out) } out = "" @@ -31,8 +31,9 @@ func TestWatcher(t *testing.T) { mu.Lock() out += fmt.Sprint(i) mu.Unlock() + syncChan <- struct{}{} }) - time.Sleep(interval * 15) + sleepInSync(9, syncChan, stopChan) mu.Lock() res := out mu.Unlock() @@ -40,3 +41,10 @@ func TestWatcher(t *testing.T) { t.Fatalf("expected (%v) must be a proper prefix of out(%v).", expected, out) } } + +func sleepInSync(times int, syncChan chan struct{}, stopChan chan struct{}) { + for i := 0; i < times; i++ { + <-syncChan + } + stopChan <- struct{}{} +} From afbda595f6c031ebb09b013858895167b79803c9 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Fri, 11 Dec 2015 21:47:38 +0100 Subject: [PATCH 20/44] import glob tests --- caddy/parse/import_glob0.txt | 6 ++++++ caddy/parse/import_glob1.txt | 4 ++++ caddy/parse/import_glob2.txt | 3 +++ caddy/parse/parsing_test.go | 7 +++++++ 4 files changed, 20 insertions(+) create mode 100644 caddy/parse/import_glob0.txt create mode 100644 caddy/parse/import_glob1.txt create mode 100644 caddy/parse/import_glob2.txt diff --git a/caddy/parse/import_glob0.txt b/caddy/parse/import_glob0.txt new file mode 100644 index 000000000..e610b5e7c --- /dev/null +++ b/caddy/parse/import_glob0.txt @@ -0,0 +1,6 @@ +glob0.host0 { + dir2 arg1 +} + +glob0.host1 { +} diff --git a/caddy/parse/import_glob1.txt b/caddy/parse/import_glob1.txt new file mode 100644 index 000000000..111eb044d --- /dev/null +++ b/caddy/parse/import_glob1.txt @@ -0,0 +1,4 @@ +glob1.host0 { + dir1 + dir2 arg1 +} diff --git a/caddy/parse/import_glob2.txt b/caddy/parse/import_glob2.txt new file mode 100644 index 000000000..c09f784ec --- /dev/null +++ b/caddy/parse/import_glob2.txt @@ -0,0 +1,3 @@ +glob2.host0 { + dir2 arg1 +} diff --git a/caddy/parse/parsing_test.go b/caddy/parse/parsing_test.go index 97c86808a..bda6b29bc 100644 --- a/caddy/parse/parsing_test.go +++ b/caddy/parse/parsing_test.go @@ -329,6 +329,13 @@ func TestParseAll(t *testing.T) { []address{{"host1.com", "http"}, {"host2.com", "http"}}, []address{{"host3.com", "https"}, {"host4.com", "https"}}, }}, + + {`import import_glob*.txt`, false, [][]address{ + []address{{"glob0.host0", ""}}, + []address{{"glob0.host1", ""}}, + []address{{"glob1.host0", ""}}, + []address{{"glob2.host0", ""}}, + }}, } { p := testParser(test.input) blocks, err := p.parseAll() From eb48885d4d24305e183223dbe3e9d947125d3274 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Fri, 11 Dec 2015 22:02:31 +0100 Subject: [PATCH 21/44] Updated comments --- caddy/parse/parsing.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index e011fd8cc..db4b7e425 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -176,10 +176,11 @@ func (p *parser) directives() error { } // doImport swaps out the import directive and its argument -// (a total of 2 tokens) with the tokens in the file specified. -// When the function returns, the cursor is on the token before -// where the import directive was. In other words, call Next() -// to access the first token that was imported. +// (a total of 2 tokens) with the tokens in the specified file +// or globbing pattern. When the function returns, the cursor +// is on the token before where the import directive was. In +// other words, call Next() to access the first token that was +// imported. func (p *parser) doImport() error { if !p.NextArg() { return p.ArgErr() @@ -218,6 +219,8 @@ func (p *parser) doImport() error { return nil } +// doSingleImport lexes the individual files matching the +// globbing pattern from of the import directive. func (p *parser) doSingleImport(importFile string) error { file, err := os.Open(importFile) if err != nil { From b7fd1f4e9ea044566e246b0768f2254238400d3e Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Sat, 12 Dec 2015 14:04:48 +0100 Subject: [PATCH 22/44] FastCGI: separate standard and error output responses. --- middleware/fastcgi/fastcgi.go | 15 ++++++++++++++- middleware/fastcgi/fcgiclient.go | 21 +++++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go index 77fb668f1..b7a86b66a 100755 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -115,7 +115,12 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) return http.StatusBadGateway, err } - return 0, nil + // FastCGI stderr outputs + if fcgi.stderr.Len() != 0 { + err = LogError(fcgi.stderr.String()) + } + + return 0, err } } @@ -281,3 +286,11 @@ var ( // ErrIndexMissingSplit describes an index configuration error. ErrIndexMissingSplit = errors.New("configured index file(s) must include split value") ) + +// LogError is a non fatal error that allows requests to go through. +type LogError string + +// Error satisfies error interface. +func (l LogError) Error() string { + return string(l) +} diff --git a/middleware/fastcgi/fcgiclient.go b/middleware/fastcgi/fcgiclient.go index b8b03fe76..97dc61532 100644 --- a/middleware/fastcgi/fcgiclient.go +++ b/middleware/fastcgi/fcgiclient.go @@ -164,6 +164,7 @@ type FCGIClient struct { rwc io.ReadWriteCloser h header buf bytes.Buffer + stderr bytes.Buffer keepAlive bool reqID uint16 } @@ -346,10 +347,22 @@ func (w *streamReader) Read(p []byte) (n int, err error) { if len(p) > 0 { if len(w.buf) == 0 { - rec := &record{} - w.buf, err = rec.read(w.c.rwc) - if err != nil { - return + + // filter outputs for error log + for { + rec := &record{} + var buf []byte + buf, err = rec.read(w.c.rwc) + if err != nil { + return + } + // standard error output + if rec.h.Type == Stderr { + w.c.stderr.Write(buf) + continue + } + w.buf = buf + break } } From 3966936bd6f01462fb8b41198bf36a83e17ad6e7 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Sun, 13 Dec 2015 12:58:21 +0100 Subject: [PATCH 23/44] Remove trailing new line for cleaner log output. Return correct status code. --- middleware/fastcgi/fastcgi.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go index b7a86b66a..21404c2ea 100755 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -117,10 +117,11 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) // FastCGI stderr outputs if fcgi.stderr.Len() != 0 { - err = LogError(fcgi.stderr.String()) + // Remove trailing newline, error logger already does this. + err = LogError(strings.TrimSuffix(fcgi.stderr.String(), "\n")) } - return 0, err + return resp.StatusCode, err } } From 34d3cd7c927e113c7ec65735c4694fb21d2d51ae Mon Sep 17 00:00:00 2001 From: Craig Peterson Date: Tue, 15 Dec 2015 08:56:44 -0700 Subject: [PATCH 24/44] Gzip: Append to Vary header instead of replacing. --- middleware/gzip/gzip.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index 147d739f8..39a922663 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -113,7 +113,7 @@ type gzipResponseWriter struct { func (w gzipResponseWriter) WriteHeader(code int) { w.Header().Del("Content-Length") w.Header().Set("Content-Encoding", "gzip") - w.Header().Set("Vary", "Accept-Encoding") + w.Header().Add("Vary", "Accept-Encoding") w.ResponseWriter.WriteHeader(code) } From 5eadea6615b4667a003cc977b8f1c3d8ef1ce47c Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 17 Dec 2015 08:37:16 -0700 Subject: [PATCH 25/44] Slack -> Gitter --- CONTRIBUTING.md | 22 ++++++++++++---------- README.md | 8 +++++--- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b4d3f3f2..b05a6e0f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,9 @@ ## Contributing to Caddy -**[Join us on Slack](https://gophers.slack.com/messages/caddy/)** to chat with -other Caddy developers! ([Request an invite](http://bit.ly/go-slack-signup), -then join the #caddy channel.) +**[Join our dev chat on Gitter](https://gitter.im/mholt/caddy)** to chat with +other Caddy developers! (Dev chat only; try our +[support room](https://gitter.im/caddyserver/support) for help or +[general](https://gitter.im/caddyserver/general) for anything else.) This project gladly accepts contributions and we encourage interested users to get involved! @@ -11,24 +12,25 @@ get involved! #### For small tweaks, bug fixes, and tests Submit [pull requests](https://github.com/mholt/caddy/pulls) at any time. -Thank you for helping out in simple ways! Bug fixes should be under test to -assert correct behavior. +Bug fixes should be under test to assert correct behavior. Thank you for +helping out in simple ways! #### Ideas, questions, bug reports -You should totally [open an issue](https://github.com/mholt/caddy/issues) with -your ideas, questions, and bug reports, if one does not already exist for it. -Bug reports should state expected behavior and contain clear instructions for -reproducing the problem. +Feek free to [open an issue](https://github.com/mholt/caddy/issues) with your +ideas, questions, and bug reports, if one does not already exist for it. Bug +reports should state expected behavior and contain clear instructions for +isolating and reproducing the problem. See [How to Report Bugs Effectively](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html). + #### New features Before submitting a pull request, please open an issue first to discuss it and claim it. This prevents overlapping efforts and keeps the project in-line with its goals. If you prefer to discuss the feature privately, you can reach other -developers on Slack or you may email me directly. (My email address is below.) +developers on Gitter or you may email me directly. (My email address is below.) And don't forget to write tests for new features! diff --git a/README.md b/README.md index cb81c1751..6aa9510a1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Caddy](https://caddyserver.com/resources/images/caddy-boxed.png)](https://caddyserver.com) +[![Dev Chat](https://img.shields.io/badge/dev%20chat-gitter-ff69b4.svg?style=flat-square&label=dev+chat&color=ff69b4)](https://gitter.im/mholt/caddy) [![Documentation](https://img.shields.io/badge/godoc-reference-blue.svg?style=flat-square)](https://godoc.org/github.com/mholt/caddy) [![Linux Build Status](https://img.shields.io/travis/mholt/caddy.svg?style=flat-square&label=linux+build)](https://travis-ci.org/mholt/caddy) [![Windows Build Status](https://img.shields.io/appveyor/ci/mholt/caddy.svg?style=flat-square&label=windows+build)](https://ci.appveyor.com/project/mholt/caddy) @@ -139,9 +140,10 @@ packages that each Caddy package imports. ## Contributing -**[Join us on Slack](https://gophers.slack.com/messages/caddy/)** to chat with -other Caddy developers! ([Request an invite](http://bit.ly/go-slack-signup), -then join the #caddy channel.) +**[Join our dev chat on Gitter](https://gitter.im/mholt/caddy)** to chat with +other Caddy developers! (Dev chat only; try our +[support room](https://gitter.im/caddyserver/support) for help or +[general](https://gitter.im/caddyserver/general) for anything else.) This project would not be what it is without your help. Please see the [contributing guidelines](https://github.com/mholt/caddy/blob/master/CONTRIBUTING.md) From 8f23c430ae3a3f9101ea823a3b6a00b985dded3c Mon Sep 17 00:00:00 2001 From: jungle-boogie Date: Thu, 17 Dec 2015 13:12:22 -0800 Subject: [PATCH 26/44] Feek --> Feel unless you meant Freak --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b05a6e0f3..44eb8638a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ helping out in simple ways! #### Ideas, questions, bug reports -Feek free to [open an issue](https://github.com/mholt/caddy/issues) with your +Feel free to [open an issue](https://github.com/mholt/caddy/issues) with your ideas, questions, and bug reports, if one does not already exist for it. Bug reports should state expected behavior and contain clear instructions for isolating and reproducing the problem. From f04ff063ed62b8ce0b2c3cafdf6bcb90fffd4a4b Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Fri, 18 Dec 2015 20:58:23 +0100 Subject: [PATCH 27/44] Gzip: Fix missing gzip encoding headers. --- middleware/gzip/gzip.go | 11 ++++++++--- middleware/gzip/gzip_test.go | 4 ++-- middleware/gzip/response_filter.go | 4 ++-- middleware/gzip/response_filter_test.go | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index 147d739f8..99903ff03 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -57,7 +57,7 @@ outer: return http.StatusInternalServerError, err } defer gzipWriter.Close() - gz := gzipResponseWriter{Writer: gzipWriter, ResponseWriter: w} + gz := &gzipResponseWriter{Writer: gzipWriter, ResponseWriter: w} var rw http.ResponseWriter // if no response filter is used @@ -104,21 +104,26 @@ func newWriter(c Config, w io.Writer) (*gzip.Writer, error) { type gzipResponseWriter struct { io.Writer http.ResponseWriter + statusCodeWritten bool } // WriteHeader wraps the underlying WriteHeader method to prevent // problems with conflicting headers from proxied backends. For // example, a backend system that calculates Content-Length would // be wrong because it doesn't know it's being gzipped. -func (w gzipResponseWriter) WriteHeader(code int) { +func (w *gzipResponseWriter) WriteHeader(code int) { w.Header().Del("Content-Length") w.Header().Set("Content-Encoding", "gzip") w.Header().Set("Vary", "Accept-Encoding") w.ResponseWriter.WriteHeader(code) + w.statusCodeWritten = true } // Write wraps the underlying Write method to do compression. -func (w gzipResponseWriter) Write(b []byte) (int, error) { +func (w *gzipResponseWriter) Write(b []byte) (int, error) { + if !w.statusCodeWritten { + w.WriteHeader(http.StatusOK) + } if w.Header().Get("Content-Type") == "" { w.Header().Set("Content-Type", http.DetectContentType(b)) } diff --git a/middleware/gzip/gzip_test.go b/middleware/gzip/gzip_test.go index c35c99c63..b9bafc6d6 100644 --- a/middleware/gzip/gzip_test.go +++ b/middleware/gzip/gzip_test.go @@ -92,7 +92,7 @@ func nextFunc(shouldGzip bool) middleware.Handler { if w.Header().Get("Vary") != "Accept-Encoding" { return 0, fmt.Errorf("Vary must be Accept-Encoding, found %v", r.Header.Get("Vary")) } - if _, ok := w.(gzipResponseWriter); !ok { + if _, ok := w.(*gzipResponseWriter); !ok { return 0, fmt.Errorf("ResponseWriter should be gzipResponseWriter, found %T", w) } return 0, nil @@ -103,7 +103,7 @@ func nextFunc(shouldGzip bool) middleware.Handler { if w.Header().Get("Content-Encoding") == "gzip" { return 0, fmt.Errorf("Content-Encoding must not be gzip, found gzip") } - if _, ok := w.(gzipResponseWriter); ok { + if _, ok := w.(*gzipResponseWriter); ok { return 0, fmt.Errorf("ResponseWriter should not be gzipResponseWriter") } return 0, nil diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index c599b3e1b..87a34e603 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -31,11 +31,11 @@ func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { type ResponseFilterWriter struct { filters []ResponseFilter shouldCompress bool - gzipResponseWriter + *gzipResponseWriter } // NewResponseFilterWriter creates and initializes a new ResponseFilterWriter. -func NewResponseFilterWriter(filters []ResponseFilter, gz gzipResponseWriter) *ResponseFilterWriter { +func NewResponseFilterWriter(filters []ResponseFilter, gz *gzipResponseWriter) *ResponseFilterWriter { return &ResponseFilterWriter{filters: filters, gzipResponseWriter: gz} } diff --git a/middleware/gzip/response_filter_test.go b/middleware/gzip/response_filter_test.go index cd7c71917..73867e7fc 100644 --- a/middleware/gzip/response_filter_test.go +++ b/middleware/gzip/response_filter_test.go @@ -33,7 +33,7 @@ func TestLengthFilter(t *testing.T) { for j, filter := range filters { r := httptest.NewRecorder() r.Header().Set("Content-Length", fmt.Sprint(ts.length)) - wWriter := NewResponseFilterWriter([]ResponseFilter{filter}, gzipResponseWriter{gzip.NewWriter(r), r}) + wWriter := NewResponseFilterWriter([]ResponseFilter{filter}, &gzipResponseWriter{gzip.NewWriter(r), r, false}) if filter.ShouldCompress(wWriter) != ts.shouldCompress[j] { t.Errorf("Test %v: Expected %v found %v", i, ts.shouldCompress[j], filter.ShouldCompress(r)) } From a946d65fe6ba95fe02407b6f4a7c4478c46bd50b Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Fri, 18 Dec 2015 21:25:06 +0100 Subject: [PATCH 28/44] Oops. Tests. --- middleware/gzip/gzip_test.go | 1 - middleware/gzip/response_filter.go | 11 ++++++++--- middleware/gzip/response_filter_test.go | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/middleware/gzip/gzip_test.go b/middleware/gzip/gzip_test.go index b9bafc6d6..0e4df3388 100644 --- a/middleware/gzip/gzip_test.go +++ b/middleware/gzip/gzip_test.go @@ -80,7 +80,6 @@ func TestGzipHandler(t *testing.T) { func nextFunc(shouldGzip bool) middleware.Handler { return middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { - w.WriteHeader(200) w.Write([]byte("test")) if shouldGzip { if r.Header.Get("Accept-Encoding") != "" { diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index 87a34e603..b561649e5 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -29,8 +29,9 @@ func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { // gzip compressed data if ResponseFilters are satisfied or // uncompressed data otherwise. type ResponseFilterWriter struct { - filters []ResponseFilter - shouldCompress bool + filters []ResponseFilter + shouldCompress bool + statusCodeWritten bool *gzipResponseWriter } @@ -39,7 +40,7 @@ func NewResponseFilterWriter(filters []ResponseFilter, gz *gzipResponseWriter) * return &ResponseFilterWriter{filters: filters, gzipResponseWriter: gz} } -// Write wraps underlying Write method and compresses if filters +// Write wraps underlying WriteHeader method and compresses if filters // are satisfied. func (r *ResponseFilterWriter) WriteHeader(code int) { // Determine if compression should be used or not. @@ -62,11 +63,15 @@ func (r *ResponseFilterWriter) WriteHeader(code int) { } else { r.ResponseWriter.WriteHeader(code) } + r.statusCodeWritten = true } // Write wraps underlying Write method and compresses if filters // are satisfied func (r *ResponseFilterWriter) Write(b []byte) (int, error) { + if !r.statusCodeWritten { + r.WriteHeader(http.StatusOK) + } if r.shouldCompress { return r.gzipResponseWriter.Write(b) } diff --git a/middleware/gzip/response_filter_test.go b/middleware/gzip/response_filter_test.go index 73867e7fc..75f726922 100644 --- a/middleware/gzip/response_filter_test.go +++ b/middleware/gzip/response_filter_test.go @@ -63,7 +63,6 @@ func TestResponseFilterWriter(t *testing.T) { for i, ts := range tests { server.Next = middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { w.Header().Set("Content-Length", fmt.Sprint(len(ts.body))) - w.WriteHeader(200) w.Write([]byte(ts.body)) return 200, nil }) From 1e27b5be8907021b44b307f216f6b86df5db1adc Mon Sep 17 00:00:00 2001 From: Pavel Pavlenko Date: Sat, 19 Dec 2015 14:30:25 +0300 Subject: [PATCH 29/44] Remove ECDHE-RSA-3DES-EDE-CBC-SHA and RSA-3DES-EDE-CBC-SHA from the default TLS config --- caddy/setup/tls.go | 17 +++++++++++++---- caddy/setup/tls_test.go | 6 ++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/caddy/setup/tls.go b/caddy/setup/tls.go index abec2d523..79954de48 100644 --- a/caddy/setup/tls.go +++ b/caddy/setup/tls.go @@ -90,10 +90,7 @@ func TLS(c *Controller) (middleware.Middleware, error) { func SetDefaultTLSParams(c *server.Config) { // If no ciphers provided, use all that Caddy supports for the protocol if len(c.TLS.Ciphers) == 0 { - c.TLS.Ciphers = supportedCiphers - - // Remove ECDHE-RSA-3DES-EDE-CBC-SHA and RSA-3DES-EDE-CBC-SHA from the default TLS config - c.TLS.Ciphers = c.TLS.Ciphers[:len(c.TLS.Ciphers)-2] + c.TLS.Ciphers = defaultCiphers } // Not a cipher suite, but still important for mitigating protocol downgrade attacks @@ -162,3 +159,15 @@ var supportedCiphers = []uint16{ tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, } + +// List of all the ciphers we want to use by default +var defaultCiphers = []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, +} diff --git a/caddy/setup/tls_test.go b/caddy/setup/tls_test.go index 629937016..8e2ececed 100644 --- a/caddy/setup/tls_test.go +++ b/caddy/setup/tls_test.go @@ -42,15 +42,13 @@ func TestTLSParseBasic(t *testing.T) { tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, tls.TLS_RSA_WITH_AES_256_CBC_SHA, tls.TLS_RSA_WITH_AES_128_CBC_SHA, - //tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - //tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, tls.TLS_FALLBACK_SCSV, } // Ensure count is correct (plus one for TLS_FALLBACK_SCSV) - if len(c.TLS.Ciphers) != len(supportedCiphers)-1 { + if len(c.TLS.Ciphers) != len(defaultCiphers) { t.Errorf("Expected %v Ciphers (including TLS_FALLBACK_SCSV), got %v", - len(supportedCiphers)-1, len(c.TLS.Ciphers)) + len(defaultCiphers), len(c.TLS.Ciphers)) } // Ensure ordering is correct From 3dd4c0eb6a34e6fe0929296bd908fa7b31662406 Mon Sep 17 00:00:00 2001 From: Pavel Pavlenko Date: Sat, 19 Dec 2015 14:37:38 +0300 Subject: [PATCH 30/44] Fix TestTLSParseBasic --- caddy/setup/tls_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caddy/setup/tls_test.go b/caddy/setup/tls_test.go index 8e2ececed..e2d2e0155 100644 --- a/caddy/setup/tls_test.go +++ b/caddy/setup/tls_test.go @@ -46,9 +46,9 @@ func TestTLSParseBasic(t *testing.T) { } // Ensure count is correct (plus one for TLS_FALLBACK_SCSV) - if len(c.TLS.Ciphers) != len(defaultCiphers) { + if len(c.TLS.Ciphers) != len(expectedCiphers) { t.Errorf("Expected %v Ciphers (including TLS_FALLBACK_SCSV), got %v", - len(defaultCiphers), len(c.TLS.Ciphers)) + len(expectedCiphers), len(c.TLS.Ciphers)) } // Ensure ordering is correct From f639d3cd68bc8bf06fe7953b3278d206a2f17424 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Mon, 21 Dec 2015 11:57:20 +0100 Subject: [PATCH 31/44] FastCGI: Close client connections when done. --- middleware/fastcgi/fcgiclient.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/middleware/fastcgi/fcgiclient.go b/middleware/fastcgi/fcgiclient.go index 97dc61532..511a5219d 100644 --- a/middleware/fastcgi/fcgiclient.go +++ b/middleware/fastcgi/fcgiclient.go @@ -400,6 +400,15 @@ func (c *FCGIClient) Do(p map[string]string, req io.Reader) (r io.Reader, err er return } +// clientCloser is a io.ReadCloser. It wraps a io.Reader with a Closer +// that closes FCGIClient connection. +type clientCloser struct { + *FCGIClient + io.Reader +} + +func (f clientCloser) Close() error { return f.rwc.Close() } + // Request returns a HTTP Response with Header and Body // from fcgi responder func (c *FCGIClient) Request(p map[string]string, req io.Reader) (resp *http.Response, err error) { @@ -439,9 +448,9 @@ func (c *FCGIClient) Request(p map[string]string, req io.Reader) (resp *http.Res resp.ContentLength, _ = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) if chunked(resp.TransferEncoding) { - resp.Body = ioutil.NopCloser(httputil.NewChunkedReader(rb)) + resp.Body = clientCloser{c, httputil.NewChunkedReader(rb)} } else { - resp.Body = ioutil.NopCloser(rb) + resp.Body = clientCloser{c, ioutil.NopCloser(rb)} } return } From 4d867e848bbc6d4ff65f3a4b840ad2349579adcb Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 22 Dec 2015 13:32:27 +0100 Subject: [PATCH 32/44] Markdown: Fix panic on sitegen for request dependent template values. --- middleware/markdown/generator.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/middleware/markdown/generator.go b/middleware/markdown/generator.go index d3485e918..653f457fe 100644 --- a/middleware/markdown/generator.go +++ b/middleware/markdown/generator.go @@ -5,6 +5,8 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "net/http" + "net/url" "os" "path/filepath" "strings" @@ -104,7 +106,7 @@ func generateStaticHTML(md Markdown, cfg *Config) error { reqPath = "/" + reqPath // Generate the static file - ctx := middleware.Context{Root: md.FileSys} + ctx := middleware.Context{Root: md.FileSys, Req: new(http.Request), URL: new(url.URL)} _, err = md.Process(cfg, reqPath, body, ctx) if err != nil { return err From 9e163a655d379eb6d19120a5de3dfd034a0f7eec Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 22 Dec 2015 14:43:48 +0100 Subject: [PATCH 33/44] Use proper struct constructors instead. --- middleware/markdown/generator.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/middleware/markdown/generator.go b/middleware/markdown/generator.go index 653f457fe..d218f22b4 100644 --- a/middleware/markdown/generator.go +++ b/middleware/markdown/generator.go @@ -105,8 +105,12 @@ func generateStaticHTML(md Markdown, cfg *Config) error { reqPath = filepath.ToSlash(reqPath) reqPath = "/" + reqPath + // Create empty requests and url to cater for template values. + req, _ := http.NewRequest("", "/", nil) + urlVar, _ := url.Parse("/") + // Generate the static file - ctx := middleware.Context{Root: md.FileSys, Req: new(http.Request), URL: new(url.URL)} + ctx := middleware.Context{Root: md.FileSys, Req: req, URL: urlVar} _, err = md.Process(cfg, reqPath, body, ctx) if err != nil { return err From 98d8c0f81b6fbac4a818368861b44c8ee42d22a3 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 22 Dec 2015 23:19:22 +0100 Subject: [PATCH 34/44] Added new rewrite features. --- caddy/setup/rewrite.go | 22 +++++- caddy/setup/rewrite_test.go | 20 +++--- middleware/rewrite/condition.go | 110 +++++++++++++++++++++++++++++ middleware/rewrite/rewrite.go | 78 +++++++++----------- middleware/rewrite/rewrite_test.go | 5 +- middleware/rewrite/to.go | 68 ++++++++++++++++++ 6 files changed, 243 insertions(+), 60 deletions(-) create mode 100644 middleware/rewrite/condition.go create mode 100644 middleware/rewrite/to.go diff --git a/caddy/setup/rewrite.go b/caddy/setup/rewrite.go index b510a237b..c70a2f94d 100644 --- a/caddy/setup/rewrite.go +++ b/caddy/setup/rewrite.go @@ -1,6 +1,8 @@ package setup import ( + "net/http" + "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/middleware/rewrite" ) @@ -13,7 +15,11 @@ func Rewrite(c *Controller) (middleware.Middleware, error) { } return func(next middleware.Handler) middleware.Handler { - return rewrite.Rewrite{Next: next, Rules: rewrites} + return rewrite.Rewrite{ + Next: next, + FileSys: http.Dir(c.Root), + Rules: rewrites, + } }, nil } @@ -30,6 +36,8 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { args := c.RemainingArgs() + var ifs []rewrite.If + switch len(args) { case 2: rule = rewrite.NewSimpleRule(args[0], args[1]) @@ -56,6 +64,16 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { return nil, c.ArgErr() } ext = args1 + case "if": + args1 := c.RemainingArgs() + if len(args1) != 3 { + return nil, c.ArgErr() + } + ifCond, err := rewrite.NewIf(args1[0], args1[1], args1[2]) + if err != nil { + return nil, err + } + ifs = append(ifs, ifCond) default: return nil, c.ArgErr() } @@ -64,7 +82,7 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { if pattern == "" || to == "" { return nil, c.ArgErr() } - if rule, err = rewrite.NewRegexpRule(base, pattern, to, ext); err != nil { + if rule, err = rewrite.NewComplexRule(base, pattern, to, ext, ifs); err != nil { return nil, err } regexpRules = append(regexpRules, rule) diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index f54266788..ddef7cd49 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -98,14 +98,14 @@ func TestRewriteParse(t *testing.T) { r .* to /to }`, false, []rewrite.Rule{ - &rewrite.RegexpRule{Base: "/", To: "/to", Regexp: regexp.MustCompile(".*")}, + &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile(".*")}, }}, {`rewrite { regexp .* to /to ext / html txt }`, false, []rewrite.Rule{ - &rewrite.RegexpRule{Base: "/", To: "/to", Exts: []string{"/", "html", "txt"}, Regexp: regexp.MustCompile(".*")}, + &rewrite.ComplexRule{Base: "/", To: "/to", Exts: []string{"/", "html", "txt"}, Regexp: regexp.MustCompile(".*")}, }}, {`rewrite /path { r rr @@ -116,26 +116,26 @@ func TestRewriteParse(t *testing.T) { to /to } `, false, []rewrite.Rule{ - &rewrite.RegexpRule{Base: "/path", To: "/dest", Regexp: regexp.MustCompile("rr")}, - &rewrite.RegexpRule{Base: "/", To: "/to", Regexp: regexp.MustCompile("[a-z]+")}, + &rewrite.ComplexRule{Base: "/path", To: "/dest", Regexp: regexp.MustCompile("rr")}, + &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile("[a-z]+")}, }}, {`rewrite { to /to }`, true, []rewrite.Rule{ - &rewrite.RegexpRule{}, + &rewrite.ComplexRule{}, }}, {`rewrite { r .* }`, true, []rewrite.Rule{ - &rewrite.RegexpRule{}, + &rewrite.ComplexRule{}, }}, {`rewrite { }`, true, []rewrite.Rule{ - &rewrite.RegexpRule{}, + &rewrite.ComplexRule{}, }}, {`rewrite /`, true, []rewrite.Rule{ - &rewrite.RegexpRule{}, + &rewrite.ComplexRule{}, }}, } @@ -157,8 +157,8 @@ func TestRewriteParse(t *testing.T) { } for j, e := range test.expected { - actualRule := actual[j].(*rewrite.RegexpRule) - expectedRule := e.(*rewrite.RegexpRule) + actualRule := actual[j].(*rewrite.ComplexRule) + expectedRule := e.(*rewrite.ComplexRule) if actualRule.Base != expectedRule.Base { t.Errorf("Test %d, rule %d: Expected Base=%s, got %s", diff --git a/middleware/rewrite/condition.go b/middleware/rewrite/condition.go new file mode 100644 index 000000000..ab69ef4af --- /dev/null +++ b/middleware/rewrite/condition.go @@ -0,0 +1,110 @@ +package rewrite + +import ( + "fmt" + "github.com/mholt/caddy/middleware" + "net/http" + "regexp" + "strings" +) + +const ( + // Operators + Is = "is" + Not = "not" + Has = "has" + StartsWith = "starts_with" + EndsWith = "ends_with" + Match = "match" +) + +func operatorError(operator string) error { + return fmt.Errorf("Invalid operator", operator) +} + +func newReplacer(r *http.Request) middleware.Replacer { + return middleware.NewReplacer(r, nil, "") +} + +// condition is a rewrite condition. +type condition func(string, string) bool + +var conditions = map[string]condition{ + Is: isFunc, + Not: notFunc, + Has: hasFunc, + StartsWith: startsWithFunc, + EndsWith: endsWithFunc, + Match: matchFunc, +} + +// isFunc is condition for Is operator. +// It checks for equality. +func isFunc(a, b string) bool { + return a == b +} + +// notFunc is condition for Not operator. +// It checks for inequality. +func notFunc(a, b string) bool { + return a != b +} + +// hasFunc is condition for Has operator. +// It checks if b is a substring of a. +func hasFunc(a, b string) bool { + return strings.Contains(a, b) +} + +// startsWithFunc is condition for StartsWith operator. +// It checks if b is a prefix of a. +func startsWithFunc(a, b string) bool { + return strings.HasPrefix(a, b) +} + +// endsWithFunc is condition for EndsWith operator. +// It checks if b is a suffix of a. +func endsWithFunc(a, b string) bool { + return strings.HasSuffix(a, b) +} + +// matchFunc is condition for Match operator. +// It does regexp matching of +func matchFunc(a, b string) bool { + matched, _ := regexp.MatchString(b, a) + return matched +} + +// If is statement for a rewrite condition. +type If struct { + A string + Operator string + B string +} + +// True returns true if the condition is true and false otherwise. +// If r is not nil, it replaces placeholders before comparison. +func (i If) True(r *http.Request) bool { + if c, ok := conditions[i.Operator]; ok { + a, b := i.A, i.B + if r != nil { + replacer := newReplacer(r) + a = replacer.Replace(i.A) + b = replacer.Replace(i.B) + } + return c(a, b) + } + return false +} + +// NewIf creates a new If condition. +func NewIf(a, operator, b string) (If, error) { + if _, ok := conditions[operator]; !ok { + return If{}, operatorError(operator) + } + return If{ + A: a, + Operator: operator, + B: b, + }, nil +} diff --git a/middleware/rewrite/rewrite.go b/middleware/rewrite/rewrite.go index 88944f73d..cc60e82cd 100644 --- a/middleware/rewrite/rewrite.go +++ b/middleware/rewrite/rewrite.go @@ -5,7 +5,6 @@ package rewrite import ( "fmt" "net/http" - "net/url" "path" "path/filepath" "regexp" @@ -16,14 +15,15 @@ import ( // Rewrite is middleware to rewrite request locations internally before being handled. type Rewrite struct { - Next middleware.Handler - Rules []Rule + Next middleware.Handler + FileSys http.FileSystem + Rules []Rule } // ServeHTTP implements the middleware.Handler interface. func (rw Rewrite) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { for _, rule := range rw.Rules { - if ok := rule.Rewrite(r); ok { + if ok := rule.Rewrite(rw.FileSys, r); ok { break } } @@ -33,7 +33,7 @@ func (rw Rewrite) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) // Rule describes an internal location rewrite rule. type Rule interface { // Rewrite rewrites the internal location of the current request. - Rewrite(*http.Request) bool + Rewrite(http.FileSystem, *http.Request) bool } // SimpleRule is a simple rewrite rule. @@ -47,23 +47,20 @@ func NewSimpleRule(from, to string) SimpleRule { } // Rewrite rewrites the internal location of the current request. -func (s SimpleRule) Rewrite(r *http.Request) bool { +func (s SimpleRule) Rewrite(fs http.FileSystem, r *http.Request) bool { if s.From == r.URL.Path { // take note of this rewrite for internal use by fastcgi // all we need is the URI, not full URL r.Header.Set(headerFieldName, r.URL.RequestURI()) - // replace variables - to := path.Clean(middleware.NewReplacer(r, nil, "").Replace(s.To)) - - r.URL.Path = to - return true + // attempt rewrite + return To(fs, r, s.To) } return false } -// RegexpRule is a rewrite rule based on a regular expression -type RegexpRule struct { +// ComplexRule is a rewrite rule based on a regular expression +type ComplexRule struct { // Path base. Request to this path and subpaths will be rewritten Base string @@ -73,18 +70,26 @@ type RegexpRule struct { // Extensions to filter by Exts []string + // Rewrite conditions + Ifs []If + *regexp.Regexp } // NewRegexpRule creates a new RegexpRule. It returns an error if regexp // pattern (pattern) or extensions (ext) are invalid. -func NewRegexpRule(base, pattern, to string, ext []string) (*RegexpRule, error) { - r, err := regexp.Compile(pattern) - if err != nil { - return nil, err +func NewComplexRule(base, pattern, to string, ext []string, ifs []If) (*ComplexRule, error) { + // validate regexp if present + var r *regexp.Regexp + if pattern != "" { + var err error + r, err = regexp.Compile(pattern) + if err != nil { + return nil, err + } } - // validate extensions + // validate extensions if present for _, v := range ext { if len(v) < 2 || (len(v) < 3 && v[0] == '!') { // check if no extension is specified @@ -94,16 +99,17 @@ func NewRegexpRule(base, pattern, to string, ext []string) (*RegexpRule, error) } } - return &RegexpRule{ - base, - to, - ext, - r, + return &ComplexRule{ + Base: base, + To: to, + Exts: ext, + Ifs: ifs, + Regexp: r, }, nil } // Rewrite rewrites the internal location of the current request. -func (r *RegexpRule) Rewrite(req *http.Request) bool { +func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { rPath := req.URL.Path // validate base @@ -127,31 +133,13 @@ func (r *RegexpRule) Rewrite(req *http.Request) bool { return false } - // replace variables - to := path.Clean(middleware.NewReplacer(req, nil, "").Replace(r.To)) - - // validate resulting path - url, err := url.Parse(to) - if err != nil { - return false - } - - // take note of this rewrite for internal use by fastcgi - // all we need is the URI, not full URL - req.Header.Set(headerFieldName, req.URL.RequestURI()) - - // perform rewrite - req.URL.Path = url.Path - if url.RawQuery != "" { - // overwrite query string if present - req.URL.RawQuery = url.RawQuery - } - return true + // attempt rewrite + return To(fs, req, r.To) } // matchExt matches rPath against registered file extensions. // Returns true if a match is found and false otherwise. -func (r *RegexpRule) matchExt(rPath string) bool { +func (r *ComplexRule) matchExt(rPath string) bool { f := filepath.Base(rPath) ext := path.Ext(f) if ext == "" { diff --git a/middleware/rewrite/rewrite_test.go b/middleware/rewrite/rewrite_test.go index fb0470262..ca4cc5128 100644 --- a/middleware/rewrite/rewrite_test.go +++ b/middleware/rewrite/rewrite_test.go @@ -4,9 +4,8 @@ import ( "fmt" "net/http" "net/http/httptest" - "testing" - "strings" + "testing" "github.com/mholt/caddy/middleware" ) @@ -38,7 +37,7 @@ func TestRewrite(t *testing.T) { if s := strings.Split(regexpRule[3], "|"); len(s) > 1 { ext = s[:len(s)-1] } - rule, err := NewRegexpRule(regexpRule[0], regexpRule[1], regexpRule[2], ext) + rule, err := NewComplexRule(regexpRule[0], regexpRule[1], regexpRule[2], ext) if err != nil { t.Fatal(err) } diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go new file mode 100644 index 000000000..0ee8a1595 --- /dev/null +++ b/middleware/rewrite/to.go @@ -0,0 +1,68 @@ +package rewrite + +import ( + "log" + "net/http" + "net/url" + "strings" +) + +// To attempts rewrite. It attempts to rewrite to first valid path +// or the last path if none of the paths are valid. +// Returns true if rewrite is successful and false otherwise. +func To(fs http.FileSystem, r *http.Request, to string) bool { + tos := strings.Fields(to) + replacer := newReplacer(r) + + // try each rewrite paths + t := "" + for _, v := range tos { + t = replacer.Replace(v) + if isValidFile(fs, t) { + break + } + } + + // validate resulting path + u, err := url.Parse(t) + if err != nil { + // Let the user know we got here. Rewrite is expected but + // the resulting url is invalid. + log.Printf("[ERROR] rewrite: resulting path '%v' is invalid. error: %v", t, err) + return false + } + + // take note of this rewrite for internal use by fastcgi + // all we need is the URI, not full URL + r.Header.Set(headerFieldName, r.URL.RequestURI()) + + // perform rewrite + r.URL.Path = u.Path + if u.RawQuery != "" { + // overwrite query string if present + r.URL.RawQuery = u.RawQuery + } + if u.Fragment != "" { + // overwrite fragment if present + r.URL.Fragment = u.Fragment + } + + return true +} + +// isValidFile checks if file exists on the filesystem. +// if file ends with `/`, it is validated as a directory. +func isValidFile(fs http.FileSystem, file string) bool { + f, err := fs.Open(file) + if err != nil { + return false + } + defer f.Close() + + stat, err := f.Stat() + if err != nil { + return false + } + + return strings.HasSuffix(file, "/") && stat.IsDir() +} From 4d5bc9fa6c2589a086f7033658c265a66ae7705c Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 23 Dec 2015 09:02:52 +0100 Subject: [PATCH 35/44] Backward compatibility ensured. --- caddy/setup/rewrite.go | 4 ++-- caddy/setup/rewrite_test.go | 5 ----- middleware/rewrite/rewrite.go | 15 ++++++++++++--- middleware/rewrite/rewrite_test.go | 3 ++- middleware/rewrite/to.go | 3 ++- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/caddy/setup/rewrite.go b/caddy/setup/rewrite.go index c70a2f94d..e642d071f 100644 --- a/caddy/setup/rewrite.go +++ b/caddy/setup/rewrite.go @@ -78,8 +78,8 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { return nil, c.ArgErr() } } - // ensure pattern and to are specified - if pattern == "" || to == "" { + // ensure to is specified + if to == "" { return nil, c.ArgErr() } if rule, err = rewrite.NewComplexRule(base, pattern, to, ext, ifs); err != nil { diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index ddef7cd49..13c1372fc 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -119,11 +119,6 @@ func TestRewriteParse(t *testing.T) { &rewrite.ComplexRule{Base: "/path", To: "/dest", Regexp: regexp.MustCompile("rr")}, &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile("[a-z]+")}, }}, - {`rewrite { - to /to - }`, true, []rewrite.Rule{ - &rewrite.ComplexRule{}, - }}, {`rewrite { r .* }`, true, []rewrite.Rule{ diff --git a/middleware/rewrite/rewrite.go b/middleware/rewrite/rewrite.go index cc60e82cd..f173e7547 100644 --- a/middleware/rewrite/rewrite.go +++ b/middleware/rewrite/rewrite.go @@ -128,9 +128,18 @@ func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { start-- } - // validate regexp - if !r.MatchString(rPath[start:]) { - return false + // validate regexp if present + if r.Regexp != nil { + if !r.MatchString(rPath[start:]) { + return false + } + } + + // validate rewrite conditions + for _, i := range r.Ifs { + if !i.True(req) { + return false + } } // attempt rewrite diff --git a/middleware/rewrite/rewrite_test.go b/middleware/rewrite/rewrite_test.go index ca4cc5128..6230f0d9d 100644 --- a/middleware/rewrite/rewrite_test.go +++ b/middleware/rewrite/rewrite_test.go @@ -18,6 +18,7 @@ func TestRewrite(t *testing.T) { NewSimpleRule("/a", "/b"), NewSimpleRule("/b", "/b{uri}"), }, + FileSys: http.Dir("."), } regexpRules := [][]string{ @@ -37,7 +38,7 @@ func TestRewrite(t *testing.T) { if s := strings.Split(regexpRule[3], "|"); len(s) > 1 { ext = s[:len(s)-1] } - rule, err := NewComplexRule(regexpRule[0], regexpRule[1], regexpRule[2], ext) + rule, err := NewComplexRule(regexpRule[0], regexpRule[1], regexpRule[2], ext, nil) if err != nil { t.Fatal(err) } diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go index 0ee8a1595..294b8f045 100644 --- a/middleware/rewrite/to.go +++ b/middleware/rewrite/to.go @@ -4,6 +4,7 @@ import ( "log" "net/http" "net/url" + "path" "strings" ) @@ -17,7 +18,7 @@ func To(fs http.FileSystem, r *http.Request, to string) bool { // try each rewrite paths t := "" for _, v := range tos { - t = replacer.Replace(v) + t = path.Clean(replacer.Replace(v)) if isValidFile(fs, t) { break } From 1ed786f836b59c7893bc944939cc520dcfe00208 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 23 Dec 2015 09:36:00 +0100 Subject: [PATCH 36/44] Cleanups and panic prevention in tests. --- caddy/setup/rewrite.go | 6 ++++-- middleware/rewrite/condition.go | 5 +++-- middleware/rewrite/to.go | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/caddy/setup/rewrite.go b/caddy/setup/rewrite.go index e642d071f..4c84cb5fd 100644 --- a/caddy/setup/rewrite.go +++ b/caddy/setup/rewrite.go @@ -2,6 +2,7 @@ package setup import ( "net/http" + "strings" "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/middleware/rewrite" @@ -54,10 +55,11 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { } pattern = c.Val() case "to": - if !c.NextArg() { + args1 := c.RemainingArgs() + if len(args1) == 0 { return nil, c.ArgErr() } - to = c.Val() + to = strings.Join(args1, " ") case "ext": args1 := c.RemainingArgs() if len(args1) == 0 { diff --git a/middleware/rewrite/condition.go b/middleware/rewrite/condition.go index ab69ef4af..51d4b3a26 100644 --- a/middleware/rewrite/condition.go +++ b/middleware/rewrite/condition.go @@ -2,10 +2,11 @@ package rewrite import ( "fmt" - "github.com/mholt/caddy/middleware" "net/http" "regexp" "strings" + + "github.com/mholt/caddy/middleware" ) const ( @@ -19,7 +20,7 @@ const ( ) func operatorError(operator string) error { - return fmt.Errorf("Invalid operator", operator) + return fmt.Errorf("Invalid operator %v", operator) } func newReplacer(r *http.Request) middleware.Replacer { diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go index 294b8f045..1dc48fdbd 100644 --- a/middleware/rewrite/to.go +++ b/middleware/rewrite/to.go @@ -54,6 +54,10 @@ func To(fs http.FileSystem, r *http.Request, to string) bool { // isValidFile checks if file exists on the filesystem. // if file ends with `/`, it is validated as a directory. func isValidFile(fs http.FileSystem, file string) bool { + if fs == nil { + return false + } + f, err := fs.Open(file) if err != nil { return false From 9110dc4745cf10940b0122ab749ceb992268e80e Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 23 Dec 2015 12:11:11 +0100 Subject: [PATCH 37/44] Refactor. Tests and tests data. --- caddy/setup/rewrite_test.go | 31 +++++--- middleware/rewrite/condition.go | 2 +- middleware/rewrite/condition_test.go | 90 +++++++++++++++++++++++ middleware/rewrite/rewrite_test.go | 4 +- middleware/rewrite/testdata/testdir/empty | 0 middleware/rewrite/testdata/testfile | 1 + middleware/rewrite/to.go | 15 +++- middleware/rewrite/to_test.go | 44 +++++++++++ 8 files changed, 174 insertions(+), 13 deletions(-) create mode 100644 middleware/rewrite/condition_test.go create mode 100644 middleware/rewrite/testdata/testdir/empty create mode 100644 middleware/rewrite/testdata/testfile create mode 100644 middleware/rewrite/to_test.go diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index 13c1372fc..f3d2e9259 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -1,10 +1,9 @@ package setup import ( - "testing" - "fmt" "regexp" + "testing" "github.com/mholt/caddy/middleware/rewrite" ) @@ -96,9 +95,9 @@ func TestRewriteParse(t *testing.T) { }{ {`rewrite { r .* - to /to + to /to /index.php? }`, false, []rewrite.Rule{ - &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile(".*")}, + &rewrite.ComplexRule{Base: "/", To: "/to /index.php?", Regexp: regexp.MustCompile(".*")}, }}, {`rewrite { regexp .* @@ -113,11 +112,11 @@ func TestRewriteParse(t *testing.T) { } rewrite / { regexp [a-z]+ - to /to + to /to /to2 } `, false, []rewrite.Rule{ &rewrite.ComplexRule{Base: "/path", To: "/dest", Regexp: regexp.MustCompile("rr")}, - &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile("[a-z]+")}, + &rewrite.ComplexRule{Base: "/", To: "/to /to2", Regexp: regexp.MustCompile("[a-z]+")}, }}, {`rewrite { r .* @@ -132,6 +131,12 @@ func TestRewriteParse(t *testing.T) { {`rewrite /`, true, []rewrite.Rule{ &rewrite.ComplexRule{}, }}, + {`rewrite { + to /to + if {path} is a + }`, false, []rewrite.Rule{ + &rewrite.ComplexRule{Base: "/", To: "/to", Ifs: []rewrite.If{rewrite.If{"{path}", "is", "a"}}}, + }}, } for i, test := range regexpTests { @@ -170,10 +175,18 @@ func TestRewriteParse(t *testing.T) { i, j, expectedRule.To, actualRule.To) } - if actualRule.String() != expectedRule.String() { - t.Errorf("Test %d, rule %d: Expected Pattern=%s, got %s", - i, j, expectedRule.String(), actualRule.String()) + if actualRule.Regexp != nil { + if actualRule.String() != expectedRule.String() { + t.Errorf("Test %d, rule %d: Expected Pattern=%s, got %s", + i, j, expectedRule.String(), actualRule.String()) + } } + + if fmt.Sprint(actualRule.Ifs) != fmt.Sprint(expectedRule.Ifs) { + t.Errorf("Test %d, rule %d: Expected Pattern=%s, got %s", + i, j, fmt.Sprint(expectedRule.Ifs), fmt.Sprint(actualRule.Ifs)) + } + } } diff --git a/middleware/rewrite/condition.go b/middleware/rewrite/condition.go index 51d4b3a26..c863af4f0 100644 --- a/middleware/rewrite/condition.go +++ b/middleware/rewrite/condition.go @@ -70,7 +70,7 @@ func endsWithFunc(a, b string) bool { } // matchFunc is condition for Match operator. -// It does regexp matching of +// It does regexp matching of a against pattern in b func matchFunc(a, b string) bool { matched, _ := regexp.MatchString(b, a) return matched diff --git a/middleware/rewrite/condition_test.go b/middleware/rewrite/condition_test.go new file mode 100644 index 000000000..7db206471 --- /dev/null +++ b/middleware/rewrite/condition_test.go @@ -0,0 +1,90 @@ +package rewrite + +import ( + "net/http" + "strings" + "testing" +) + +func TestConditions(t *testing.T) { + tests := []struct { + condition string + isTrue bool + }{ + {"a is b", false}, + {"a is a", true}, + {"a not b", true}, + {"a not a", false}, + {"a has a", true}, + {"a has b", false}, + {"ba has b", true}, + {"bab has b", true}, + {"bab has bb", false}, + {"bab starts_with bb", false}, + {"bab starts_with ba", true}, + {"bab starts_with bab", true}, + {"bab ends_with bb", false}, + {"bab ends_with bab", true}, + {"bab ends_with ab", true}, + {"a match *", false}, + {"a match a", true}, + {"a match .*", true}, + {"a match a.*", true}, + {"a match b.*", false}, + {"ba match b.*", true}, + {"ba match b[a-z]", true}, + {"b0 match b[a-z]", false}, + {"b0a match b[a-z]", false}, + {"b0a match b[a-z]+", false}, + {"b0a match b[a-z0-9]+", true}, + } + + for i, test := range tests { + str := strings.Fields(test.condition) + ifCond, err := NewIf(str[0], str[1], str[2]) + if err != nil { + t.Error(err) + } + isTrue := ifCond.True(nil) + if isTrue != test.isTrue { + t.Errorf("Test %v: expected %v found %v", i, test.isTrue, isTrue) + } + } + + invalidOperators := []string{"ss", "and", "if"} + for _, op := range invalidOperators { + _, err := NewIf("a", op, "b") + if err == nil { + t.Error("Invalid operator %v used, expected error.", op) + } + } + + replaceTests := []struct { + url string + condition string + isTrue bool + }{ + {"/home", "{uri} match /home", true}, + {"/hom", "{uri} match /home", false}, + {"/hom", "{uri} starts_with /home", false}, + {"/hom", "{uri} starts_with /h", true}, + {"/home/.hiddenfile", `{uri} match \/\.(.*)`, true}, + {"/home/.hiddendir/afile", `{uri} match \/\.(.*)`, true}, + } + + for i, test := range replaceTests { + r, err := http.NewRequest("GET", test.url, nil) + if err != nil { + t.Error(err) + } + str := strings.Fields(test.condition) + ifCond, err := NewIf(str[0], str[1], str[2]) + if err != nil { + t.Error(err) + } + isTrue := ifCond.True(r) + if isTrue != test.isTrue { + t.Errorf("Test %v: expected %v found %v", i, test.isTrue, isTrue) + } + } +} diff --git a/middleware/rewrite/rewrite_test.go b/middleware/rewrite/rewrite_test.go index 6230f0d9d..a538b79d8 100644 --- a/middleware/rewrite/rewrite_test.go +++ b/middleware/rewrite/rewrite_test.go @@ -21,7 +21,7 @@ func TestRewrite(t *testing.T) { FileSys: http.Dir("."), } - regexpRules := [][]string{ + regexps := [][]string{ {"/reg/", ".*", "/to", ""}, {"/r/", "[a-z]+", "/toaz", "!.html|"}, {"/url/", "a([a-z0-9]*)s([A-Z]{2})", "/to/{path}", ""}, @@ -33,7 +33,7 @@ func TestRewrite(t *testing.T) { {"/ab/", `.*\.jpg`, "/ajpg", ""}, } - for _, regexpRule := range regexpRules { + for _, regexpRule := range regexps { var ext []string if s := strings.Split(regexpRule[3], "|"); len(s) > 1 { ext = s[:len(s)-1] diff --git a/middleware/rewrite/testdata/testdir/empty b/middleware/rewrite/testdata/testdir/empty new file mode 100644 index 000000000..e69de29bb diff --git a/middleware/rewrite/testdata/testfile b/middleware/rewrite/testdata/testfile new file mode 100644 index 000000000..7b4d68d70 --- /dev/null +++ b/middleware/rewrite/testdata/testfile @@ -0,0 +1 @@ +empty \ No newline at end of file diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go index 1dc48fdbd..d6c7f5210 100644 --- a/middleware/rewrite/to.go +++ b/middleware/rewrite/to.go @@ -19,6 +19,13 @@ func To(fs http.FileSystem, r *http.Request, to string) bool { t := "" for _, v := range tos { t = path.Clean(replacer.Replace(v)) + + // add trailing slash for directories, if present + if strings.HasSuffix(v, "/") && !strings.HasSuffix(t, "/") { + t += "/" + } + + // validate file if isValidFile(fs, t) { break } @@ -69,5 +76,11 @@ func isValidFile(fs http.FileSystem, file string) bool { return false } - return strings.HasSuffix(file, "/") && stat.IsDir() + // directory + if strings.HasSuffix(file, "/") { + return stat.IsDir() + } + + // file + return !stat.IsDir() } diff --git a/middleware/rewrite/to_test.go b/middleware/rewrite/to_test.go new file mode 100644 index 000000000..2d8b535ac --- /dev/null +++ b/middleware/rewrite/to_test.go @@ -0,0 +1,44 @@ +package rewrite + +import ( + "net/http" + "net/url" + "testing" +) + +func TestTo(t *testing.T) { + fs := http.Dir("testdata") + tests := []struct { + url string + to string + expected string + }{ + {"/", "/somefiles", "/somefiles"}, + {"/somefiles", "/somefiles /index.php{uri}", "/index.php/somefiles"}, + {"/somefiles", "/testfile /index.php{uri}", "/testfile"}, + {"/somefiles", "/testfile/ /index.php{uri}", "/index.php/somefiles"}, + {"/somefiles", "/somefiles /index.php{uri}", "/index.php/somefiles"}, + {"/?a=b", "/somefiles /index.php?{query}", "/index.php?a=b"}, + {"/?a=b", "/testfile /index.php?{query}", "/testfile?a=b"}, + {"/?a=b", "/testdir /index.php?{query}", "/index.php?a=b"}, + {"/?a=b", "/testdir/ /index.php?{query}", "/testdir/?a=b"}, + } + + uri := func(r *url.URL) string { + uri := r.Path + if r.RawQuery != "" { + uri += "?" + r.RawQuery + } + return uri + } + for i, test := range tests { + r, err := http.NewRequest("GET", test.url, nil) + if err != nil { + t.Error(err) + } + To(fs, r, test.to) + if uri(r.URL) != test.expected { + t.Errorf("Test %v: expected %v found %v", i, test.expected, uri(r.URL)) + } + } +} From 92bd91441893e41c39bb9ca2cf5d2dc509bfdbc8 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 23 Dec 2015 13:23:43 +0100 Subject: [PATCH 38/44] Fix vet errors. --- caddy/setup/rewrite_test.go | 2 +- middleware/rewrite/condition_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index f3d2e9259..c43818b2d 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -135,7 +135,7 @@ func TestRewriteParse(t *testing.T) { to /to if {path} is a }`, false, []rewrite.Rule{ - &rewrite.ComplexRule{Base: "/", To: "/to", Ifs: []rewrite.If{rewrite.If{"{path}", "is", "a"}}}, + &rewrite.ComplexRule{Base: "/", To: "/to", Ifs: []rewrite.If{rewrite.If{A: "{path}", Operator: "is", B: "a"}}}, }}, } diff --git a/middleware/rewrite/condition_test.go b/middleware/rewrite/condition_test.go index 7db206471..c056f8964 100644 --- a/middleware/rewrite/condition_test.go +++ b/middleware/rewrite/condition_test.go @@ -55,7 +55,7 @@ func TestConditions(t *testing.T) { for _, op := range invalidOperators { _, err := NewIf("a", op, "b") if err == nil { - t.Error("Invalid operator %v used, expected error.", op) + t.Errorf("Invalid operator %v used, expected error.", op) } } From 168723a026cdad53e2d5d8434003fa3e57f04e9a Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Thu, 24 Dec 2015 09:00:10 +0100 Subject: [PATCH 39/44] Added escaped versions of uri, query and path. --- middleware/replacer.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/middleware/replacer.go b/middleware/replacer.go index 29c695b77..3fb389a13 100644 --- a/middleware/replacer.go +++ b/middleware/replacer.go @@ -3,6 +3,7 @@ package middleware import ( "net" "net/http" + "net/url" "path" "strconv" "strings" @@ -38,11 +39,13 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla } return "http" }(), - "{host}": r.Host, - "{path}": r.URL.Path, - "{query}": r.URL.RawQuery, - "{fragment}": r.URL.Fragment, - "{proto}": r.Proto, + "{host}": r.Host, + "{path}": r.URL.Path, + "{path_escaped}": url.QueryEscape(r.URL.Path), + "{query}": r.URL.RawQuery, + "{query_escaped}": url.QueryEscape(r.URL.RawQuery), + "{fragment}": r.URL.Fragment, + "{proto}": r.Proto, "{remote}": func() string { if fwdFor := r.Header.Get("X-Forwarded-For"); fwdFor != "" { return fwdFor @@ -60,7 +63,8 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla } return port }(), - "{uri}": r.URL.RequestURI(), + "{uri}": r.URL.RequestURI(), + "{uri_escaped}": url.QueryEscape(r.URL.RequestURI()), "{when}": func() string { return time.Now().Format(timeFormat) }(), From 1e7ec3397b8e67d9e0f0879a2238b556b7d51988 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Tue, 29 Dec 2015 23:32:59 +0100 Subject: [PATCH 40/44] Import allows only one expression --- caddy/parse/parsing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index db4b7e425..6ef908b0b 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -187,7 +187,7 @@ func (p *parser) doImport() error { } importPattern := p.Val() if p.NextArg() { - return p.Err("Import allows only one file to import") + return p.Err("Import allows only one expression, either file or glob pattern") } matches, err := filepath.Glob(importPattern) From 7dadcd58341b8ad63893a4669913099edbcf937c Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 30 Dec 2015 20:42:03 +0100 Subject: [PATCH 41/44] Add ability to set custom values. --- middleware/replacer.go | 6 ++++++ middleware/replacer_test.go | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/middleware/replacer.go b/middleware/replacer.go index 3fb389a13..193fd4fcd 100644 --- a/middleware/replacer.go +++ b/middleware/replacer.go @@ -16,6 +16,7 @@ import ( // NewReplacer to get one of these. type Replacer interface { Replace(string) string + Set(key, value string) } type replacer struct { @@ -117,6 +118,11 @@ func (r replacer) Replace(s string) string { return s } +// Set sets key to value in the replacements map. +func (r replacer) Set(key, value string) { + r.replacements["{"+key+"}"] = value +} + const ( timeFormat = "02/Jan/2006:15:04:05 -0700" headerReplacer = "{>" diff --git a/middleware/replacer_test.go b/middleware/replacer_test.go index 39d91a222..8f1147dea 100644 --- a/middleware/replacer_test.go +++ b/middleware/replacer_test.go @@ -69,3 +69,43 @@ func TestReplace(t *testing.T) { } } + +func TestSet(t *testing.T) { + w := httptest.NewRecorder() + recordRequest := NewResponseRecorder(w) + userJSON := `{"username": "dennis"}` + + reader := strings.NewReader(userJSON) //Convert string to reader + + request, err := http.NewRequest("POST", "http://caddyserver.com", reader) //Create request with JSON body + if err != nil { + t.Fatalf("Request Formation Failed \n") + } + replaceValues := NewReplacer(request, recordRequest, "") + + replaceValues.Set("host", "getcaddy.com") + replaceValues.Set("method", "GET") + replaceValues.Set("status", "201") + replaceValues.Set("variable", "value") + + switch v := replaceValues.(type) { + case replacer: + + if v.Replace("This host is {host}") != "This host is getcaddy.com" { + t.Errorf("Expected host replacement failed") + } + if v.Replace("This request method is {method}") != "This request method is GET" { + t.Errorf("Expected method replacement failed") + } + if v.Replace("The response status is {status}") != "The response status is 201" { + t.Errorf("Expected status replacement failed") + } + if v.Replace("The value of variable is {variable}") != "The value of variable is value" { + t.Errorf("Expected status replacement failed") + } + + default: + t.Fatalf("Return Value from New Replacer expected pass type assertion into a replacer type \n") + } + +} From 3c086fb2e67b885d7c3a6bd417b0e2d6fc66074f Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 30 Dec 2015 21:47:37 +0100 Subject: [PATCH 42/44] Support for rewrite match group. --- middleware/rewrite/rewrite.go | 15 ++++++++++++--- middleware/rewrite/rewrite_test.go | 9 +++++++++ middleware/rewrite/to.go | 5 +++-- middleware/rewrite/to_test.go | 2 +- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/middleware/rewrite/rewrite.go b/middleware/rewrite/rewrite.go index f173e7547..60cc0b9da 100644 --- a/middleware/rewrite/rewrite.go +++ b/middleware/rewrite/rewrite.go @@ -54,7 +54,7 @@ func (s SimpleRule) Rewrite(fs http.FileSystem, r *http.Request) bool { r.Header.Set(headerFieldName, r.URL.RequestURI()) // attempt rewrite - return To(fs, r, s.To) + return To(fs, r, s.To, newReplacer(r)) } return false } @@ -111,6 +111,7 @@ func NewComplexRule(base, pattern, to string, ext []string, ifs []If) (*ComplexR // Rewrite rewrites the internal location of the current request. func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { rPath := req.URL.Path + replacer := newReplacer(req) // validate base if !middleware.Path(rPath).Matches(r.Base) { @@ -130,8 +131,16 @@ func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { // validate regexp if present if r.Regexp != nil { - if !r.MatchString(rPath[start:]) { + matches := r.FindStringSubmatch(rPath[start:]) + switch len(matches) { + case 0: + // no match return false + default: + // set regexp match variables {1}, {2} ... + for i := 1; i < len(matches); i++ { + replacer.Set(fmt.Sprint(i), matches[i]) + } } } @@ -143,7 +152,7 @@ func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { } // attempt rewrite - return To(fs, req, r.To) + return To(fs, req, r.To, replacer) } // matchExt matches rPath against registered file extensions. diff --git a/middleware/rewrite/rewrite_test.go b/middleware/rewrite/rewrite_test.go index a538b79d8..5b8916064 100644 --- a/middleware/rewrite/rewrite_test.go +++ b/middleware/rewrite/rewrite_test.go @@ -31,6 +31,9 @@ func TestRewrite(t *testing.T) { {"/abcd/", "ab", "/a/{dir}/{file}", ".html|"}, {"/abcde/", "ab", "/a#{fragment}", ".html|"}, {"/ab/", `.*\.jpg`, "/ajpg", ""}, + {"/reggrp", `/ad/([0-9]+)([a-z]*)`, "/a{1}/{2}", ""}, + {"/reg2grp", `(.*)`, "/{1}", ""}, + {"/reg3grp", `(.*)/(.*)/(.*)`, "/{1}{2}{3}", ""}, } for _, regexpRule := range regexps { @@ -81,6 +84,12 @@ func TestRewrite(t *testing.T) { {"/abcde/abcde.html", "/a"}, {"/abcde/abcde.html#1234", "/a#1234"}, {"/ab/ab.jpg", "/ajpg"}, + {"/reggrp/ad/12", "/a12"}, + {"/reggrp/ad/124a", "/a124/a"}, + {"/reggrp/ad/124abc", "/a124/abc"}, + {"/reg2grp/ad/124abc", "/ad/124abc"}, + {"/reg3grp/ad/aa/66", "/adaa66"}, + {"/reg3grp/ad612/n1n/ab", "/ad612n1nab"}, } for i, test := range tests { diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go index d6c7f5210..8a577c5e4 100644 --- a/middleware/rewrite/to.go +++ b/middleware/rewrite/to.go @@ -6,14 +6,15 @@ import ( "net/url" "path" "strings" + + "github.com/mholt/caddy/middleware" ) // To attempts rewrite. It attempts to rewrite to first valid path // or the last path if none of the paths are valid. // Returns true if rewrite is successful and false otherwise. -func To(fs http.FileSystem, r *http.Request, to string) bool { +func To(fs http.FileSystem, r *http.Request, to string, replacer middleware.Replacer) bool { tos := strings.Fields(to) - replacer := newReplacer(r) // try each rewrite paths t := "" diff --git a/middleware/rewrite/to_test.go b/middleware/rewrite/to_test.go index 2d8b535ac..6133c0b63 100644 --- a/middleware/rewrite/to_test.go +++ b/middleware/rewrite/to_test.go @@ -36,7 +36,7 @@ func TestTo(t *testing.T) { if err != nil { t.Error(err) } - To(fs, r, test.to) + To(fs, r, test.to, newReplacer(r)) if uri(r.URL) != test.expected { t.Errorf("Test %v: expected %v found %v", i, test.expected, uri(r.URL)) } From e2a3ec4c3dbc537d10ed872b31c2ed693740b3fa Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 31 Dec 2015 12:12:16 -0700 Subject: [PATCH 43/44] Replacer supports case-insensitive header placeholders (fixes #476) --- middleware/replacer.go | 43 ++++++++----- middleware/replacer_test.go | 120 ++++++++++++++++++------------------ 2 files changed, 90 insertions(+), 73 deletions(-) diff --git a/middleware/replacer.go b/middleware/replacer.go index 193fd4fcd..10f0e35af 100644 --- a/middleware/replacer.go +++ b/middleware/replacer.go @@ -86,9 +86,9 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla rep.replacements["{latency}"] = time.Since(rr.start).String() } - // Header placeholders - for header, val := range r.Header { - rep.replacements[headerReplacer+header+"}"] = strings.Join(val, ",") + // Header placeholders (case-insensitive) + for header, values := range r.Header { + rep.replacements[headerReplacer+strings.ToLower(header)+"}"] = strings.Join(values, ",") } return rep @@ -97,6 +97,32 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla // Replace performs a replacement of values on s and returns // the string with the replaced values. func (r replacer) Replace(s string) string { + // Header replacements - these are case-insensitive, so we can't just use strings.Replace() + startPos := strings.Index(s, headerReplacer) + for startPos > -1 { + // carefully find end of placeholder + endOffset := strings.Index(s[startPos+1:], "}") + if endOffset == -1 { + startPos = strings.Index(s[startPos+len(headerReplacer):], headerReplacer) + continue + } + endPos := startPos + len(headerReplacer) + endOffset + + // look for replacement, case-insensitive + placeholder := strings.ToLower(s[startPos:endPos]) + replacement := r.replacements[placeholder] + if replacement == "" { + replacement = r.emptyValue + } + + // do the replacement manually + s = s[:startPos] + replacement + s[endPos:] + + // move to next one + startPos = strings.Index(s[endOffset:], headerReplacer) + } + + // Regular replacements - these are easier because they're case-sensitive for placeholder, replacement := range r.replacements { if replacement == "" { replacement = r.emptyValue @@ -104,17 +130,6 @@ func (r replacer) Replace(s string) string { s = strings.Replace(s, placeholder, replacement, -1) } - // Replace any header placeholders that weren't found - for strings.Contains(s, headerReplacer) { - idxStart := strings.Index(s, headerReplacer) - endOffset := idxStart + len(headerReplacer) - idxEnd := strings.Index(s[endOffset:], "}") - if idxEnd > -1 { - s = s[:idxStart] + r.emptyValue + s[endOffset+idxEnd+1:] - } else { - break - } - } return s } diff --git a/middleware/replacer_test.go b/middleware/replacer_test.go index 8f1147dea..46d7b8f90 100644 --- a/middleware/replacer_test.go +++ b/middleware/replacer_test.go @@ -10,102 +10,104 @@ import ( func TestNewReplacer(t *testing.T) { w := httptest.NewRecorder() recordRequest := NewResponseRecorder(w) - userJSON := `{"username": "dennis"}` + reader := strings.NewReader(`{"username": "dennis"}`) - reader := strings.NewReader(userJSON) //Convert string to reader - - request, err := http.NewRequest("POST", "http://caddyserver.com", reader) //Create request with JSON body + request, err := http.NewRequest("POST", "http://localhost", reader) if err != nil { - t.Fatalf("Request Formation Failed \n") + t.Fatal("Request Formation Failed\n") } replaceValues := NewReplacer(request, recordRequest, "") switch v := replaceValues.(type) { case replacer: - if v.replacements["{host}"] != "caddyserver.com" { - t.Errorf("Expected host to be caddyserver.com") + if v.replacements["{host}"] != "localhost" { + t.Error("Expected host to be localhost") } if v.replacements["{method}"] != "POST" { - t.Errorf("Expected request method to be POST") + t.Error("Expected request method to be POST") } if v.replacements["{status}"] != "200" { - t.Errorf("Expected status to be 200") + t.Error("Expected status to be 200") } default: - t.Fatalf("Return Value from New Replacer expected pass type assertion into a replacer type \n") + t.Fatal("Return Value from New Replacer expected pass type assertion into a replacer type\n") } } func TestReplace(t *testing.T) { w := httptest.NewRecorder() recordRequest := NewResponseRecorder(w) - userJSON := `{"username": "dennis"}` + reader := strings.NewReader(`{"username": "dennis"}`) - reader := strings.NewReader(userJSON) //Convert string to reader - - request, err := http.NewRequest("POST", "http://caddyserver.com", reader) //Create request with JSON body + request, err := http.NewRequest("POST", "http://localhost", reader) if err != nil { - t.Fatalf("Request Formation Failed \n") + t.Fatal("Request Formation Failed\n") } - replaceValues := NewReplacer(request, recordRequest, "") + request.Header.Set("Custom", "fooBar") + repl := NewReplacer(request, recordRequest, "-") - switch v := replaceValues.(type) { - case replacer: - - if v.Replace("This host is {host}") != "This host is caddyserver.com" { - t.Errorf("Expected host replacement failed") - } - if v.Replace("This request method is {method}") != "This request method is POST" { - t.Errorf("Expected method replacement failed") - } - if v.Replace("The response status is {status}") != "The response status is 200" { - t.Errorf("Expected status replacement failed") - } - - default: - t.Fatalf("Return Value from New Replacer expected pass type assertion into a replacer type \n") + if expected, actual := "This host is localhost.", repl.Replace("This host is {host}."); expected != actual { + t.Errorf("{host} replacement: expected '%s', got '%s'", expected, actual) + } + if expected, actual := "This request method is POST.", repl.Replace("This request method is {method}."); expected != actual { + t.Errorf("{method} replacement: expected '%s', got '%s'", expected, actual) + } + if expected, actual := "The response status is 200.", repl.Replace("The response status is {status}."); expected != actual { + t.Errorf("{status} replacement: expected '%s', got '%s'", expected, actual) + } + if expected, actual := "The Custom header is fooBar.", repl.Replace("The Custom header is {>Custom}."); expected != actual { + t.Errorf("{>Custom} replacement: expected '%s', got '%s'", expected, actual) } + // Test header case-insensitivity + if expected, actual := "The cUsToM header is fooBar...", repl.Replace("The cUsToM header is {>cUsToM}..."); expected != actual { + t.Errorf("{>cUsToM} replacement: expected '%s', got '%s'", expected, actual) + } + + // Test non-existent header/value + if expected, actual := "The Non-Existent header is -.", repl.Replace("The Non-Existent header is {>Non-Existent}."); expected != actual { + t.Errorf("{>Non-Existent} replacement: expected '%s', got '%s'", expected, actual) + } + + // Test bad placeholder + if expected, actual := "Bad {host placeholder...", repl.Replace("Bad {host placeholder..."); expected != actual { + t.Errorf("bad placeholder: expected '%s', got '%s'", expected, actual) + } + + // Test bad header placeholder + if expected, actual := "Bad {>Custom placeholder", repl.Replace("Bad {>Custom placeholder"); expected != actual { + t.Errorf("bad header placeholder: expected '%s', got '%s'", expected, actual) + } } func TestSet(t *testing.T) { w := httptest.NewRecorder() recordRequest := NewResponseRecorder(w) - userJSON := `{"username": "dennis"}` + reader := strings.NewReader(`{"username": "dennis"}`) - reader := strings.NewReader(userJSON) //Convert string to reader - - request, err := http.NewRequest("POST", "http://caddyserver.com", reader) //Create request with JSON body + request, err := http.NewRequest("POST", "http://localhost", reader) if err != nil { t.Fatalf("Request Formation Failed \n") } - replaceValues := NewReplacer(request, recordRequest, "") + repl := NewReplacer(request, recordRequest, "") - replaceValues.Set("host", "getcaddy.com") - replaceValues.Set("method", "GET") - replaceValues.Set("status", "201") - replaceValues.Set("variable", "value") + repl.Set("host", "getcaddy.com") + repl.Set("method", "GET") + repl.Set("status", "201") + repl.Set("variable", "value") - switch v := replaceValues.(type) { - case replacer: - - if v.Replace("This host is {host}") != "This host is getcaddy.com" { - t.Errorf("Expected host replacement failed") - } - if v.Replace("This request method is {method}") != "This request method is GET" { - t.Errorf("Expected method replacement failed") - } - if v.Replace("The response status is {status}") != "The response status is 201" { - t.Errorf("Expected status replacement failed") - } - if v.Replace("The value of variable is {variable}") != "The value of variable is value" { - t.Errorf("Expected status replacement failed") - } - - default: - t.Fatalf("Return Value from New Replacer expected pass type assertion into a replacer type \n") + if repl.Replace("This host is {host}") != "This host is getcaddy.com" { + t.Error("Expected host replacement failed") + } + if repl.Replace("This request method is {method}") != "This request method is GET" { + t.Error("Expected method replacement failed") + } + if repl.Replace("The response status is {status}") != "The response status is 201" { + t.Error("Expected status replacement failed") + } + if repl.Replace("The value of variable is {variable}") != "The value of variable is value" { + t.Error("Expected variable replacement failed") } - } From b6326d402d3025084cce6b28b1299506aaaa838c Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 31 Dec 2015 12:31:30 -0700 Subject: [PATCH 44/44] Fix for case-insensitive header replacements (#476) --- middleware/replacer.go | 34 +++++++++++++--------------------- middleware/replacer_test.go | 17 ++++++++++++++--- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/middleware/replacer.go b/middleware/replacer.go index 10f0e35af..8a3d202a2 100644 --- a/middleware/replacer.go +++ b/middleware/replacer.go @@ -98,28 +98,20 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla // the string with the replaced values. func (r replacer) Replace(s string) string { // Header replacements - these are case-insensitive, so we can't just use strings.Replace() - startPos := strings.Index(s, headerReplacer) - for startPos > -1 { - // carefully find end of placeholder - endOffset := strings.Index(s[startPos+1:], "}") - if endOffset == -1 { - startPos = strings.Index(s[startPos+len(headerReplacer):], headerReplacer) - continue + for strings.Contains(s, headerReplacer) { + idxStart := strings.Index(s, headerReplacer) + endOffset := idxStart + len(headerReplacer) + idxEnd := strings.Index(s[endOffset:], "}") + if idxEnd > -1 { + placeholder := strings.ToLower(s[idxStart : endOffset+idxEnd+1]) + replacement := r.replacements[placeholder] + if replacement == "" { + replacement = r.emptyValue + } + s = s[:idxStart] + replacement + s[endOffset+idxEnd+1:] + } else { + break } - endPos := startPos + len(headerReplacer) + endOffset - - // look for replacement, case-insensitive - placeholder := strings.ToLower(s[startPos:endPos]) - replacement := r.replacements[placeholder] - if replacement == "" { - replacement = r.emptyValue - } - - // do the replacement manually - s = s[:startPos] + replacement + s[endPos:] - - // move to next one - startPos = strings.Index(s[endOffset:], headerReplacer) } // Regular replacements - these are easier because they're case-sensitive diff --git a/middleware/replacer_test.go b/middleware/replacer_test.go index 46d7b8f90..d98bd2de1 100644 --- a/middleware/replacer_test.go +++ b/middleware/replacer_test.go @@ -45,7 +45,8 @@ func TestReplace(t *testing.T) { if err != nil { t.Fatal("Request Formation Failed\n") } - request.Header.Set("Custom", "fooBar") + request.Header.Set("Custom", "foobarbaz") + request.Header.Set("ShorterVal", "1") repl := NewReplacer(request, recordRequest, "-") if expected, actual := "This host is localhost.", repl.Replace("This host is {host}."); expected != actual { @@ -57,12 +58,12 @@ func TestReplace(t *testing.T) { if expected, actual := "The response status is 200.", repl.Replace("The response status is {status}."); expected != actual { t.Errorf("{status} replacement: expected '%s', got '%s'", expected, actual) } - if expected, actual := "The Custom header is fooBar.", repl.Replace("The Custom header is {>Custom}."); expected != actual { + if expected, actual := "The Custom header is foobarbaz.", repl.Replace("The Custom header is {>Custom}."); expected != actual { t.Errorf("{>Custom} replacement: expected '%s', got '%s'", expected, actual) } // Test header case-insensitivity - if expected, actual := "The cUsToM header is fooBar...", repl.Replace("The cUsToM header is {>cUsToM}..."); expected != actual { + if expected, actual := "The cUsToM header is foobarbaz...", repl.Replace("The cUsToM header is {>cUsToM}..."); expected != actual { t.Errorf("{>cUsToM} replacement: expected '%s', got '%s'", expected, actual) } @@ -80,6 +81,16 @@ func TestReplace(t *testing.T) { if expected, actual := "Bad {>Custom placeholder", repl.Replace("Bad {>Custom placeholder"); expected != actual { t.Errorf("bad header placeholder: expected '%s', got '%s'", expected, actual) } + + // Test bad header placeholder with valid one later + if expected, actual := "Bad -", repl.Replace("Bad {>Custom placeholder {>ShorterVal}"); expected != actual { + t.Errorf("bad header placeholders: expected '%s', got '%s'", expected, actual) + } + + // Test shorter header value with multiple placeholders + if expected, actual := "Short value 1 then foobarbaz.", repl.Replace("Short value {>ShorterVal} then {>Custom}."); expected != actual { + t.Errorf("short value: expected '%s', got '%s'", expected, actual) + } } func TestSet(t *testing.T) {