Revamp markdown processing.

Nuke pre-generation.  This may come back in the form of a more general
caching layer at some later stage.

Nuke index generation.  This should likely be rethought and re-implemented.

caddy/setup/markdown.go

@@ -2,9 +2,7 @@ package setup
 
 import (
 	"net/http"
-	"path"
 	"path/filepath"
-	"strings"
 
 	"github.com/mholt/caddy/middleware"
 	"github.com/mholt/caddy/middleware/markdown"
@@ -25,25 +23,6 @@ func Markdown(c *Controller) (middleware.Middleware, error) {
 		IndexFiles: []string{"index.md"},
 	}
 
-	// Sweep the whole path at startup to at least generate link index, maybe generate static site
-	c.Startup = append(c.Startup, func() error {
-		for i := range mdconfigs {
-			cfg := mdconfigs[i]
-
-			// Generate link index and static files (if enabled)
-			if err := markdown.GenerateStatic(md, cfg); err != nil {
-				return err
-			}
-
-			// Watch file changes for static site generation if not in development mode.
-			if !cfg.Development {
-				markdown.Watch(md, cfg, markdown.DefaultInterval)
-			}
-		}
-
-		return nil
-	})
-
 	return func(next middleware.Handler) middleware.Handler {
 		md.Next = next
 		return md
@@ -56,8 +35,8 @@ func markdownParse(c *Controller) ([]*markdown.Config, error) {
 	for c.Next() {
 		md := &markdown.Config{
 			Renderer:   blackfriday.HtmlRenderer(0, "", ""),
+			Extensions: make(map[string]struct{}),
 			Templates:  make(map[string]string),
-			StaticFiles: make(map[string]string),
 		}
 
 		// Get the path scope
@@ -80,7 +59,9 @@ func markdownParse(c *Controller) ([]*markdown.Config, error) {
 
 		// If no extensions were specified, assume some defaults
 		if len(md.Extensions) == 0 {
-			md.Extensions = []string{".md", ".markdown", ".mdown"}
+			md.Extensions[".md"] = struct{}{}
+			md.Extensions[".markdown"] = struct{}{}
+			md.Extensions[".mdown"] = struct{}{}
 		}
 
 		mdconfigs = append(mdconfigs, md)
@@ -92,11 +73,9 @@ func markdownParse(c *Controller) ([]*markdown.Config, error) {
 func loadParams(c *Controller, mdc *markdown.Config) error {
 	switch c.Val() {
 	case "ext":
-		exts := c.RemainingArgs()
+		for _, ext := range c.RemainingArgs() {
-		if len(exts) == 0 {
+			mdc.Extensions[ext] = struct{}{}
-			return c.ArgErr()
 		}
-		mdc.Extensions = append(mdc.Extensions, exts...)
 		return nil
 	case "css":
 		if !c.NextArg() {
@@ -113,7 +92,7 @@ func loadParams(c *Controller, mdc *markdown.Config) error {
 	case "template":
 		tArgs := c.RemainingArgs()
 		switch len(tArgs) {
-		case 0:
+		default:
 			return c.ArgErr()
 		case 1:
 			if _, ok := mdc.Templates[markdown.DefaultTemplate]; ok {
@@ -126,31 +105,7 @@ func loadParams(c *Controller, mdc *markdown.Config) error {
 			fpath := filepath.ToSlash(filepath.Clean(c.Root + string(filepath.Separator) + tArgs[1]))
 			mdc.Templates[tArgs[0]] = fpath
 			return nil
-		default:
-			return c.ArgErr()
 		}
-	case "sitegen":
-		if c.NextArg() {
-			mdc.StaticDir = path.Join(c.Root, c.Val())
-		} else {
-			mdc.StaticDir = path.Join(c.Root, markdown.DefaultStaticDir)
-		}
-		if c.NextArg() {
-			// only 1 argument allowed
-			return c.ArgErr()
-		}
-		return nil
-	case "dev":
-		if c.NextArg() {
-			mdc.Development = strings.ToLower(c.Val()) == "true"
-		} else {
-			mdc.Development = true
-		}
-		if c.NextArg() {
-			// only 1 argument allowed
-			return c.ArgErr()
-		}
-		return nil
 	default:
 		return c.Err("Expected valid markdown configuration property")
 	}

caddy/setup/markdown_test.go

@@ -1,15 +1,9 @@
 package setup
 
 import (
-	"bytes"
 	"fmt"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
 	"testing"
 
-	"github.com/mholt/caddy/middleware"
 	"github.com/mholt/caddy/middleware/markdown"
 )
 
@@ -37,83 +31,13 @@ func TestMarkdown(t *testing.T) {
 	if myHandler.Configs[0].PathScope != "/blog" {
 		t.Errorf("Expected /blog as the Path Scope")
 	}
-	if fmt.Sprint(myHandler.Configs[0].Extensions) != fmt.Sprint([]string{".md", ".markdown", ".mdown"}) {
+	if len(myHandler.Configs[0].Extensions) != 3 {
-		t.Errorf("Expected .md, .markdown, and .mdown as default extensions")
+		t.Error("Expected 3 markdown extensions")
 	}
+	for _, key := range []string{".md", ".markdown", ".mdown"} {
+		if ext, ok := myHandler.Configs[0].Extensions[key]; !ok {
+			t.Errorf("Expected extensions to contain %v", ext)
 		}
-
-func TestMarkdownStaticGen(t *testing.T) {
-	c := NewTestController(`markdown /blog {
-	ext .md
-	template tpl_with_include.html
-	sitegen
-}`)
-
-	c.Root = "./testdata"
-	mid, err := Markdown(c)
-
-	if err != nil {
-		t.Errorf("Expected no errors, got: %v", err)
-	}
-
-	if mid == nil {
-		t.Fatal("Expected middleware, was nil instead")
-	}
-
-	for _, start := range c.Startup {
-		err := start()
-		if err != nil {
-			t.Errorf("Startup error: %v", err)
-		}
-	}
-
-	next := middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) {
-		t.Fatalf("Next shouldn't be called")
-		return 0, nil
-	})
-	hndlr := mid(next)
-	mkdwn, ok := hndlr.(markdown.Markdown)
-	if !ok {
-		t.Fatalf("Was expecting a markdown.Markdown but got %T", hndlr)
-	}
-
-	expectedStaticFiles := map[string]string{"/blog/first_post.md": "testdata/generated_site/blog/first_post.md/index.html"}
-	if fmt.Sprint(expectedStaticFiles) != fmt.Sprint(mkdwn.Configs[0].StaticFiles) {
-		t.Fatalf("Test expected StaticFiles to be  %s, but got %s",
-			fmt.Sprint(expectedStaticFiles), fmt.Sprint(mkdwn.Configs[0].StaticFiles))
-	}
-
-	filePath := "testdata/generated_site/blog/first_post.md/index.html"
-	if _, err := os.Stat(filePath); err != nil {
-		t.Fatalf("An error occured when getting the file information: %v", err)
-	}
-
-	html, err := ioutil.ReadFile(filePath)
-	if err != nil {
-		t.Fatalf("An error occured when getting the file content: %v", err)
-	}
-
-	expectedBody := []byte(`<!DOCTYPE html>
-<html>
-<head>
-<title>first_post</title>
-</head>
-<body>
-<h1>Header title</h1>
-
-<h1>Test h1</h1>
-
-</body>
-</html>
-`)
-
-	if !bytes.Equal(html, expectedBody) {
-		t.Fatalf("Expected file content: %s got: %s", string(expectedBody), string(html))
-	}
-
-	fp := filepath.Join(c.Root, markdown.DefaultStaticDir)
-	if err = os.RemoveAll(fp); err != nil {
-		t.Errorf("Error while removing the generated static files: %v", err)
 	}
 }
 
@@ -130,19 +54,22 @@ func TestMarkdownParse(t *testing.T) {
 	js  /resources/js/blog.js
 }`, false, []markdown.Config{{
 			PathScope: "/blog",
-			Extensions: []string{".md", ".txt"},
+			Extensions: map[string]struct{}{
+				".md":  struct{}{},
+				".txt": struct{}{},
+			},
 			Styles:  []string{"/resources/css/blog.css"},
 			Scripts: []string{"/resources/js/blog.js"},
 		}}},
 		{`markdown /blog {
 	ext .md
 	template tpl_with_include.html
-	sitegen
 }`, false, []markdown.Config{{
 			PathScope: "/blog",
-			Extensions: []string{".md"},
+			Extensions: map[string]struct{}{
+				".md": struct{}{},
+			},
 			Templates: map[string]string{markdown.DefaultTemplate: "testdata/tpl_with_include.html"},
-			StaticDir:  markdown.DefaultStaticDir,
 		}}},
 	}
 	for i, test := range tests {

middleware/context.go

@@ -184,7 +184,11 @@ func (c Context) Markdown(filename string) (string, error) {
 		return "", err
 	}
 	renderer := blackfriday.HtmlRenderer(0, "", "")
-	extns := blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_STRIKETHROUGH | blackfriday.EXTENSION_DEFINITION_LISTS
+	extns := 0
+	extns |= blackfriday.EXTENSION_TABLES
+	extns |= blackfriday.EXTENSION_FENCED_CODE
+	extns |= blackfriday.EXTENSION_STRIKETHROUGH
+	extns |= blackfriday.EXTENSION_DEFINITION_LISTS
 	markdown := blackfriday.Markdown([]byte(body), renderer, extns)
 
 	return string(markdown), nil

middleware/markdown/generator.go

@@ -1,146 +0,0 @@
-package markdown
-
-import (
-	"crypto/md5"
-	"encoding/hex"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-
-	"github.com/mholt/caddy/middleware"
-)
-
-// GenerateStatic generate static files and link index from markdowns.
-// It only generates static files if it is enabled (cfg.StaticDir
-// must be set).
-func GenerateStatic(md Markdown, cfg *Config) error {
-	// Generate links since they may be needed, even without sitegen.
-	generated, err := generateLinks(md, cfg)
-	if err != nil {
-		return err
-	}
-
-	// No new file changes, return.
-	if !generated {
-		return nil
-	}
-
-	// If static site generation is enabled, generate the site.
-	if cfg.StaticDir != "" {
-		if err := generateStaticHTML(md, cfg); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-type linkGenerator struct {
-	gens map[*Config]*linkGen
-	sync.Mutex
-}
-
-var generator = linkGenerator{gens: make(map[*Config]*linkGen)}
-
-// generateLinks generates links to all markdown files ordered by newest date.
-// This blocks until link generation is done. When called by multiple goroutines,
-// the first caller starts the generation and others only wait.
-// It returns if generation is done and any error that occurred.
-func generateLinks(md Markdown, cfg *Config) (bool, error) {
-	generator.Lock()
-
-	// if link generator exists for config and running, wait.
-	if g, ok := generator.gens[cfg]; ok {
-		if g.started() {
-			g.addWaiter()
-			generator.Unlock()
-			g.Wait()
-			// another goroutine has done the generation.
-			return false, g.lastErr
-		}
-	}
-
-	g := &linkGen{}
-	generator.gens[cfg] = g
-	generator.Unlock()
-
-	generated := g.generateLinks(md, cfg)
-	g.discardWaiters()
-	return generated, g.lastErr
-}
-
-// generateStaticHTML generates static HTML files from markdowns.
-func generateStaticHTML(md Markdown, cfg *Config) error {
-	// If generated site already exists, clear it out
-	_, err := os.Stat(cfg.StaticDir)
-	if err == nil {
-		err := os.RemoveAll(cfg.StaticDir)
-		if err != nil {
-			return err
-		}
-	}
-
-	fp := filepath.Join(md.Root, cfg.PathScope)
-
-	return filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
-		for _, ext := range cfg.Extensions {
-			if !info.IsDir() && strings.HasSuffix(info.Name(), ext) {
-				// Load the file
-				body, err := ioutil.ReadFile(path)
-				if err != nil {
-					return err
-				}
-
-				// Get the relative path as if it were a HTTP request,
-				// then prepend with "/" (like a real HTTP request)
-				reqPath, err := filepath.Rel(md.Root, path)
-				if err != nil {
-					return err
-				}
-				reqPath = filepath.ToSlash(reqPath)
-				reqPath = "/" + reqPath
-
-				// Create empty requests and url to cater for template values.
-				req, _ := http.NewRequest("", "/", nil)
-				urlVar, _ := url.Parse("/")
-
-				// Generate the static file
-				ctx := middleware.Context{Root: md.FileSys, Req: req, URL: urlVar}
-				_, err = md.Process(cfg, reqPath, body, ctx)
-				if err != nil {
-					return err
-				}
-
-				break // don't try other file extensions
-			}
-		}
-		return nil
-	})
-}
-
-// computeDirHash computes an hash on static directory of c.
-func computeDirHash(md Markdown, c *Config) (string, error) {
-	dir := filepath.Join(md.Root, c.PathScope)
-	if _, err := os.Stat(dir); err != nil {
-		return "", err
-	}
-
-	hashString := ""
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if !info.IsDir() && c.IsValidExt(filepath.Ext(path)) {
-			hashString += fmt.Sprintf("%v%v%v%v", info.ModTime(), info.Name(), info.Size(), path)
-		}
-		return nil
-	})
-	if err != nil {
-		return "", err
-	}
-
-	sum := md5.Sum([]byte(hashString))
-	return hex.EncodeToString(sum[:]), nil
-}

middleware/markdown/markdown.go

@@ -7,8 +7,7 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"strings"
+	"path"
-	"sync"
 
 	"github.com/mholt/caddy/middleware"
 	"github.com/russross/blackfriday"
@@ -52,7 +51,7 @@ type Config struct {
 	PathScope string
 
 	// List of extensions to consider as markdown files
-	Extensions []string
+	Extensions map[string]struct{}
 
 	// List of style sheets to load for each markdown file
 	Styles []string
@@ -62,34 +61,6 @@ type Config struct {
 
 	// Map of registered templates
 	Templates map[string]string
-
-	// Map of request URL to static files generated
-	StaticFiles map[string]string
-
-	// Links to all markdown pages ordered by date.
-	Links []PageLink
-
-	// Stores a directory hash to check for changes.
-	linksHash string
-
-	// Directory to store static files
-	StaticDir string
-
-	// If in development mode. i.e. Actively editing markdown files.
-	Development bool
-
-	sync.RWMutex
-}
-
-// IsValidExt checks to see if an extension is a valid markdown extension
-// for config.
-func (c *Config) IsValidExt(ext string) bool {
-	for _, e := range c.Extensions {
-		if e == ext {
-			return true
-		}
-	}
-	return false
 }
 
 // ServeHTTP implements the http.Handler interface.
@@ -104,8 +75,8 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 			fpath = idx
 		}
 
-		for _, ext := range cfg.Extensions {
+		// If supported extension, process it
-			if strings.HasSuffix(fpath, ext) {
+		if _, ok := cfg.Extensions[path.Ext(fpath)]; ok {
 			f, err := md.FileSys.Open(fpath)
 			if err != nil {
 				if os.IsPermission(err) {
@@ -119,35 +90,6 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 				return http.StatusNotFound, nil
 			}
 
-				// if development is set, scan directory for file changes for links.
-				if cfg.Development {
-					if err := GenerateStatic(md, cfg); err != nil {
-						log.Printf("[ERROR] markdown: on-demand site generation error: %v", err)
-					}
-				}
-
-				cfg.RLock()
-				filepath, ok := cfg.StaticFiles[fpath]
-				cfg.RUnlock()
-				// if static site is generated, attempt to use it
-				if ok {
-					if fs1, err := os.Stat(filepath); err == nil {
-						// if markdown has not been modified since static page
-						// generation, serve the static page
-						if fs.ModTime().Before(fs1.ModTime()) {
-							if html, err := ioutil.ReadFile(filepath); err == nil {
-								middleware.SetLastModifiedHeader(w, fs1.ModTime())
-								w.Write(html)
-								return http.StatusOK, nil
-							}
-							if os.IsPermission(err) {
-								return http.StatusForbidden, err
-							}
-							return http.StatusNotFound, nil
-						}
-					}
-				}
-
 			body, err := ioutil.ReadAll(f)
 			if err != nil {
 				return http.StatusInternalServerError, err
@@ -168,7 +110,6 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 			return http.StatusOK, nil
 		}
 	}
-	}
 
 	// Didn't qualify to serve as markdown; pass-thru
 	return md.Next.ServeHTTP(w, r)

middleware/markdown/markdown_test.go

@@ -2,12 +2,10 @@ package markdown
 
 import (
 	"bufio"
-	"log"
 	"net/http"
 	"net/http/httptest"
 	"os"
 	"strings"
-	"sync"
 	"testing"
 	"time"
 
@@ -25,52 +23,44 @@ func TestMarkdown(t *testing.T) {
 			{
 				Renderer:  blackfriday.HtmlRenderer(0, "", ""),
 				PathScope: "/blog",
-				Extensions:  []string{".md"},
+				Extensions: map[string]struct{}{
+					".md": struct{}{},
+				},
 				Styles:    []string{},
 				Scripts:   []string{},
 				Templates: templates,
-				StaticDir:   DefaultStaticDir,
-				StaticFiles: make(map[string]string),
 			},
 			{
 				Renderer:  blackfriday.HtmlRenderer(0, "", ""),
 				PathScope: "/docflags",
-				Extensions: []string{".md"},
+				Extensions: map[string]struct{}{
+					".md": struct{}{},
+				},
 				Styles:  []string{},
 				Scripts: []string{},
 				Templates: map[string]string{
 					DefaultTemplate: "testdata/docflags/template.txt",
 				},
-				StaticDir:   DefaultStaticDir,
-				StaticFiles: make(map[string]string),
 			},
 			{
 				Renderer:  blackfriday.HtmlRenderer(0, "", ""),
 				PathScope: "/log",
-				Extensions:  []string{".md"},
+				Extensions: map[string]struct{}{
+					".md": struct{}{},
+				},
 				Styles:    []string{"/resources/css/log.css", "/resources/css/default.css"},
 				Scripts:   []string{"/resources/js/log.js", "/resources/js/default.js"},
 				Templates: make(map[string]string),
-				StaticDir:   DefaultStaticDir,
-				StaticFiles: make(map[string]string),
 			},
 			{
 				Renderer:  blackfriday.HtmlRenderer(0, "", ""),
 				PathScope: "/og",
-				Extensions:  []string{".md"},
+				Extensions: map[string]struct{}{
+					".md": struct{}{},
+				},
 				Styles:    []string{},
 				Scripts:   []string{},
 				Templates: templates,
-				StaticDir:   "testdata/og_static",
-				StaticFiles: map[string]string{"/og/first.md": "testdata/og_static/og/first.md/index.html"},
-				Links: []PageLink{
-					{
-						Title:   "first",
-						Summary: "",
-						Date:    time.Now(),
-						URL:     "/og/first.md",
-					},
-				},
 			},
 		},
 		IndexFiles: []string{"index.html"},
@@ -80,14 +70,6 @@ func TestMarkdown(t *testing.T) {
 		}),
 	}
 
-	for i := range md.Configs {
-		c := md.Configs[i]
-		if err := GenerateStatic(md, c); err != nil {
-			t.Fatalf("Error: %v", err)
-		}
-		Watch(md, c, time.Millisecond*100)
-	}
-
 	req, err := http.NewRequest("GET", "/blog/test.md", nil)
 	if err != nil {
 		t.Fatalf("Could not create HTTP request: %v", err)
@@ -219,52 +201,6 @@ Welcome to title!
 	if !equalStrings(respBody, expectedBody) {
 		t.Fatalf("Expected body: %v got: %v", expectedBody, respBody)
 	}
-
-	expectedLinks := []string{
-		"/blog/test.md",
-		"/docflags/test.md",
-		"/log/test.md",
-	}
-
-	for i, c := range md.Configs[:2] {
-		log.Printf("Test number: %d, configuration links: %v, config: %v", i, c.Links, c)
-		if c.Links[0].URL != expectedLinks[i] {
-			t.Fatalf("Expected %v got %v", expectedLinks[i], c.Links[0].URL)
-		}
-	}
-
-	// attempt to trigger race conditions
-	var w sync.WaitGroup
-	f := func() {
-		req, err := http.NewRequest("GET", "/log/test.md", nil)
-		if err != nil {
-			t.Fatalf("Could not create HTTP request: %v", err)
-		}
-		rec := httptest.NewRecorder()
-
-		md.ServeHTTP(rec, req)
-		w.Done()
-	}
-	for i := 0; i < 5; i++ {
-		w.Add(1)
-		go f()
-	}
-	w.Wait()
-
-	f = func() {
-		GenerateStatic(md, md.Configs[0])
-		w.Done()
-	}
-	for i := 0; i < 5; i++ {
-		w.Add(1)
-		go f()
-	}
-	w.Wait()
-
-	if err = os.RemoveAll(DefaultStaticDir); err != nil {
-		t.Errorf("Error while removing the generated static files: %v", err)
-	}
-
 }
 
 func equalStrings(s1, s2 string) bool {

middleware/markdown/metadata.go

@@ -2,12 +2,16 @@ package markdown
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"time"
+)
 
-	"github.com/BurntSushi/toml"
+var (
-	"gopkg.in/yaml.v2"
+	// Date format YYYY-MM-DD HH:MM:SS or YYYY-MM-DD
+	timeLayout = []string{
+		`2006-01-02 15:04:05`,
+		`2006-01-02`,
+	}
 )
 
 // Metadata stores a page's metadata
@@ -30,6 +34,8 @@ type Metadata struct {
 
 // load loads parsed values in parsedMap into Metadata
 func (m *Metadata) load(parsedMap map[string]interface{}) {
+
+	// Pull top level things out
 	if title, ok := parsedMap["title"]; ok {
 		m.Title, _ = title.(string)
 	}
@@ -37,17 +43,21 @@ func (m *Metadata) load(parsedMap map[string]interface{}) {
 		m.Template, _ = template.(string)
 	}
 	if date, ok := parsedMap["date"].(string); ok {
-		if t, err := time.Parse(timeLayout, date); err == nil {
+		for _, layout := range timeLayout {
+			if t, err := time.Parse(layout, date); err == nil {
 				m.Date = t
+				break
 			}
 		}
-	// store everything as a variable
+	}
+
+	// Store everything as a flag or variable
 	for key, val := range parsedMap {
 		switch v := val.(type) {
-		case string:
-			m.Variables[key] = v
 		case bool:
 			m.Flags[key] = v
+		case string:
+			m.Variables[key] = v
 		}
 	}
 }
@@ -70,116 +80,6 @@ type MetadataParser interface {
 	Metadata() Metadata
 }
 
-// JSONMetadataParser is the MetadataParser for JSON
-type JSONMetadataParser struct {
-	metadata Metadata
-}
-
-// Parse the metadata
-func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) {
-	b, markdown, err := extractMetadata(j, b)
-	if err != nil {
-		return markdown, err
-	}
-	m := make(map[string]interface{})
-
-	// Read the preceding JSON object
-	decoder := json.NewDecoder(bytes.NewReader(b))
-	if err := decoder.Decode(&m); err != nil {
-		return markdown, err
-	}
-	j.metadata.load(m)
-
-	return markdown, nil
-}
-
-// Metadata returns parsed metadata.  It should be called
-// only after a call to Parse returns without error.
-func (j *JSONMetadataParser) Metadata() Metadata {
-	return j.metadata
-}
-
-// Opening returns the opening identifier JSON metadata
-func (j *JSONMetadataParser) Opening() []byte {
-	return []byte("{")
-}
-
-// Closing returns the closing identifier JSON metadata
-func (j *JSONMetadataParser) Closing() []byte {
-	return []byte("}")
-}
-
-// TOMLMetadataParser is the MetadataParser for TOML
-type TOMLMetadataParser struct {
-	metadata Metadata
-}
-
-// Parse the metadata
-func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) {
-	b, markdown, err := extractMetadata(t, b)
-	if err != nil {
-		return markdown, err
-	}
-	m := make(map[string]interface{})
-	if err := toml.Unmarshal(b, &m); err != nil {
-		return markdown, err
-	}
-	t.metadata.load(m)
-	return markdown, nil
-}
-
-// Metadata returns parsed metadata.  It should be called
-// only after a call to Parse returns without error.
-func (t *TOMLMetadataParser) Metadata() Metadata {
-	return t.metadata
-}
-
-// Opening returns the opening identifier TOML metadata
-func (t *TOMLMetadataParser) Opening() []byte {
-	return []byte("+++")
-}
-
-// Closing returns the closing identifier TOML metadata
-func (t *TOMLMetadataParser) Closing() []byte {
-	return []byte("+++")
-}
-
-// YAMLMetadataParser is the MetadataParser for YAML
-type YAMLMetadataParser struct {
-	metadata Metadata
-}
-
-// Parse the metadata
-func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) {
-	b, markdown, err := extractMetadata(y, b)
-	if err != nil {
-		return markdown, err
-	}
-
-	m := make(map[string]interface{})
-	if err := yaml.Unmarshal(b, &m); err != nil {
-		return markdown, err
-	}
-	y.metadata.load(m)
-	return markdown, nil
-}
-
-// Metadata returns parsed metadata.  It should be called
-// only after a call to Parse returns without error.
-func (y *YAMLMetadataParser) Metadata() Metadata {
-	return y.metadata
-}
-
-// Opening returns the opening identifier YAML metadata
-func (y *YAMLMetadataParser) Opening() []byte {
-	return []byte("---")
-}
-
-// Closing returns the closing identifier YAML metadata
-func (y *YAMLMetadataParser) Closing() []byte {
-	return []byte("---")
-}
-
 // extractMetadata separates metadata content from from markdown content in b.
 // It returns the metadata, the remaining bytes (markdown), and an error, if any.
 func extractMetadata(parser MetadataParser, b []byte) (metadata []byte, markdown []byte, err error) {

middleware/markdown/metadata_json.go

@@ -0,0 +1,45 @@
+package markdown
+
+import (
+	"bytes"
+	"encoding/json"
+)
+
+// JSONMetadataParser is the MetadataParser for JSON
+type JSONMetadataParser struct {
+	metadata Metadata
+}
+
+// Parse the metadata
+func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) {
+	b, markdown, err := extractMetadata(j, b)
+	if err != nil {
+		return markdown, err
+	}
+	m := make(map[string]interface{})
+
+	// Read the preceding JSON object
+	decoder := json.NewDecoder(bytes.NewReader(b))
+	if err := decoder.Decode(&m); err != nil {
+		return markdown, err
+	}
+	j.metadata.load(m)
+
+	return markdown, nil
+}
+
+// Metadata returns parsed metadata.  It should be called
+// only after a call to Parse returns without error.
+func (j *JSONMetadataParser) Metadata() Metadata {
+	return j.metadata
+}
+
+// Opening returns the opening identifier JSON metadata
+func (j *JSONMetadataParser) Opening() []byte {
+	return []byte("{")
+}
+
+// Closing returns the closing identifier JSON metadata
+func (j *JSONMetadataParser) Closing() []byte {
+	return []byte("}")
+}

middleware/markdown/metadata_toml.go

@@ -0,0 +1,40 @@
+package markdown
+
+import (
+	"github.com/BurntSushi/toml"
+)
+
+// TOMLMetadataParser is the MetadataParser for TOML
+type TOMLMetadataParser struct {
+	metadata Metadata
+}
+
+// Parse the metadata
+func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) {
+	b, markdown, err := extractMetadata(t, b)
+	if err != nil {
+		return markdown, err
+	}
+	m := make(map[string]interface{})
+	if err := toml.Unmarshal(b, &m); err != nil {
+		return markdown, err
+	}
+	t.metadata.load(m)
+	return markdown, nil
+}
+
+// Metadata returns parsed metadata.  It should be called
+// only after a call to Parse returns without error.
+func (t *TOMLMetadataParser) Metadata() Metadata {
+	return t.metadata
+}
+
+// Opening returns the opening identifier TOML metadata
+func (t *TOMLMetadataParser) Opening() []byte {
+	return []byte("+++")
+}
+
+// Closing returns the closing identifier TOML metadata
+func (t *TOMLMetadataParser) Closing() []byte {
+	return []byte("+++")
+}

middleware/markdown/metadata_yaml.go

@@ -0,0 +1,41 @@
+package markdown
+
+import (
+	"gopkg.in/yaml.v2"
+)
+
+// YAMLMetadataParser is the MetadataParser for YAML
+type YAMLMetadataParser struct {
+	metadata Metadata
+}
+
+// Parse the metadata
+func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) {
+	b, markdown, err := extractMetadata(y, b)
+	if err != nil {
+		return markdown, err
+	}
+
+	m := make(map[string]interface{})
+	if err := yaml.Unmarshal(b, &m); err != nil {
+		return markdown, err
+	}
+	y.metadata.load(m)
+	return markdown, nil
+}
+
+// Metadata returns parsed metadata.  It should be called
+// only after a call to Parse returns without error.
+func (y *YAMLMetadataParser) Metadata() Metadata {
+	return y.metadata
+}
+
+// Opening returns the opening identifier YAML metadata
+func (y *YAMLMetadataParser) Opening() []byte {
+	return []byte("---")
+}
+
+// Closing returns the closing identifier YAML metadata
+func (y *YAMLMetadataParser) Closing() []byte {
+	return []byte("---")
+}

middleware/markdown/page.go

@@ -1,169 +0,0 @@
-package markdown
-
-import (
-	"bytes"
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/russross/blackfriday"
-)
-
-const (
-	// Date format YYYY-MM-DD HH:MM:SS
-	timeLayout = `2006-01-02 15:04:05`
-
-	// Maximum length of page summary.
-	summaryLen = 500
-)
-
-// PageLink represents a statically generated markdown page.
-type PageLink struct {
-	Title   string
-	Summary string
-	Date    time.Time
-	URL     string
-}
-
-// byDate sorts PageLink by newest date to oldest.
-type byDate []PageLink
-
-func (p byDate) Len() int           { return len(p) }
-func (p byDate) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-func (p byDate) Less(i, j int) bool { return p[i].Date.After(p[j].Date) }
-
-type linkGen struct {
-	generating bool
-	waiters    int
-	lastErr    error
-	sync.RWMutex
-	sync.WaitGroup
-}
-
-func (l *linkGen) addWaiter() {
-	l.WaitGroup.Add(1)
-	l.waiters++
-}
-
-func (l *linkGen) discardWaiters() {
-	l.Lock()
-	defer l.Unlock()
-	for i := 0; i < l.waiters; i++ {
-		l.Done()
-	}
-}
-
-func (l *linkGen) started() bool {
-	l.RLock()
-	defer l.RUnlock()
-	return l.generating
-}
-
-// generateLinks generate links to markdown files if there are file changes.
-// It returns true when generation is done and false otherwise.
-func (l *linkGen) generateLinks(md Markdown, cfg *Config) bool {
-	l.Lock()
-	l.generating = true
-	l.Unlock()
-
-	fp := filepath.Join(md.Root, cfg.PathScope) // path to scan for .md files
-
-	// If the file path to scan for Markdown files (fp) does
-	// not exist, there are no markdown files to scan for.
-	if _, err := os.Stat(fp); os.IsNotExist(err) {
-		l.Lock()
-		l.lastErr = err
-		l.generating = false
-		l.Unlock()
-		return false
-	}
-
-	hash, err := computeDirHash(md, cfg)
-
-	// same hash, return.
-	if err == nil && hash == cfg.linksHash {
-		l.Lock()
-		l.generating = false
-		l.Unlock()
-		return false
-	} else if err != nil {
-		log.Printf("[ERROR] markdown: Hash error: %v", err)
-	}
-
-	cfg.Links = []PageLink{}
-
-	cfg.Lock()
-	l.lastErr = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
-		for _, ext := range cfg.Extensions {
-			if info.IsDir() || !strings.HasSuffix(info.Name(), ext) {
-				continue
-			}
-
-			// Load the file
-			body, err := ioutil.ReadFile(path)
-			if err != nil {
-				return err
-			}
-
-			// Get the relative path as if it were a HTTP request,
-			// then prepend with "/" (like a real HTTP request)
-			reqPath, err := filepath.Rel(md.Root, path)
-			if err != nil {
-				return err
-			}
-			reqPath = "/" + filepath.ToSlash(reqPath)
-
-			// Make the summary
-			parser := findParser(body)
-			if parser == nil {
-				// no metadata, ignore.
-				continue
-			}
-			summaryRaw, err := parser.Parse(body)
-			if err != nil {
-				return err
-			}
-			summary := blackfriday.Markdown(summaryRaw, SummaryRenderer{}, 0)
-
-			// truncate summary to maximum length
-			if len(summary) > summaryLen {
-				summary = summary[:summaryLen]
-
-				// trim to nearest word
-				lastSpace := bytes.LastIndex(summary, []byte(" "))
-				if lastSpace != -1 {
-					summary = summary[:lastSpace]
-				}
-			}
-
-			metadata := parser.Metadata()
-
-			cfg.Links = append(cfg.Links, PageLink{
-				Title:   metadata.Title,
-				URL:     reqPath,
-				Date:    metadata.Date,
-				Summary: string(summary),
-			})
-
-			break // don't try other file extensions
-		}
-
-		return nil
-	})
-
-	// sort by newest date
-	sort.Sort(byDate(cfg.Links))
-
-	cfg.linksHash = hash
-	cfg.Unlock()
-
-	l.Lock()
-	l.generating = false
-	l.Unlock()
-	return true
-}

middleware/markdown/process.go

@@ -3,10 +3,7 @@ package markdown
 import (
 	"bytes"
 	"io/ioutil"
-	"log"
-	"os"
 	"path/filepath"
-	"strings"
 	"text/template"
 
 	"github.com/mholt/caddy/middleware"
@@ -16,8 +13,6 @@ import (
 const (
 	// DefaultTemplate is the default template.
 	DefaultTemplate = "defaultTemplate"
-	// DefaultStaticDir is the default static directory.
-	DefaultStaticDir = "generated_site"
 )
 
 // Data represents a markdown document.
@@ -25,7 +20,8 @@ type Data struct {
 	middleware.Context
 	Doc      map[string]string
 	DocFlags map[string]bool
-	Links    []PageLink
+	Styles   []string
+	Scripts  []string
 }
 
 // Include "overrides" the embedded middleware.Context's Include()
@@ -75,7 +71,11 @@ func (md Markdown) Process(c *Config, requestPath string, b []byte, ctx middlewa
 	}
 
 	// process markdown
-	extns := blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_STRIKETHROUGH | blackfriday.EXTENSION_DEFINITION_LISTS
+	extns := 0
+	extns |= blackfriday.EXTENSION_TABLES
+	extns |= blackfriday.EXTENSION_FENCED_CODE
+	extns |= blackfriday.EXTENSION_STRIKETHROUGH
+	extns |= blackfriday.EXTENSION_DEFINITION_LISTS
 	markdown = blackfriday.Markdown(markdown, c.Renderer, extns)
 
 	// set it as body for template
@@ -94,123 +94,51 @@ func (md Markdown) Process(c *Config, requestPath string, b []byte, ctx middlewa
 // processTemplate processes a template given a requestPath,
 // template (tmpl) and metadata
 func (md Markdown) processTemplate(c *Config, requestPath string, tmpl []byte, metadata Metadata, ctx middleware.Context) ([]byte, error) {
+	var t *template.Template
+	var err error
+
 	// if template is not specified,
 	// use the default template
 	if tmpl == nil {
-		tmpl = defaultTemplate(c, metadata, requestPath)
+		t = template.Must(template.New("").Parse(htmlTemplate))
-	}
+	} else {
-
+		t, err = template.New("").Parse(string(tmpl))
-	// process the template
-	b := new(bytes.Buffer)
-	t, err := template.New("").Parse(string(tmpl))
 		if err != nil {
 			return nil, err
 		}
+	}
+
+	// process the template
 	mdData := Data{
 		Context:  ctx,
 		Doc:      metadata.Variables,
 		DocFlags: metadata.Flags,
-		Links:    c.Links,
+		Styles:   c.Styles,
+		Scripts:  c.Scripts,
 	}
 
-	c.RLock()
+	b := new(bytes.Buffer)
 	err = t.Execute(b, mdData)
-	c.RUnlock()
-
 	if err != nil {
 		return nil, err
 	}
 
-	// generate static page
-	if err = md.generatePage(c, requestPath, b.Bytes()); err != nil {
-		// if static page generation fails, nothing fatal, only log the error.
-		// TODO: Report (return) this non-fatal error, but don't log it here?
-		log.Println("[ERROR] markdown: Render:", err)
-	}
-
 	return b.Bytes(), nil
-
-}
-
-// generatePage generates a static html page from the markdown in content if c.StaticDir
-// is a non-empty value, meaning that the user enabled static site generation.
-func (md Markdown) generatePage(c *Config, requestPath string, content []byte) error {
-	// Only generate the page if static site generation is enabled
-	if c.StaticDir != "" {
-		// if static directory is not existing, create it
-		if _, err := os.Stat(c.StaticDir); err != nil {
-			err := os.MkdirAll(c.StaticDir, os.FileMode(0755))
-			if err != nil {
-				return err
-			}
-		}
-
-		// the URL will always use "/" as a path separator,
-		// convert that to a native path to support OS that
-		// use different path separators
-		filePath := filepath.Join(c.StaticDir, filepath.FromSlash(requestPath))
-
-		// If it is index file, use the directory instead
-		if md.IsIndexFile(filepath.Base(requestPath)) {
-			filePath, _ = filepath.Split(filePath)
-		}
-
-		// Create the directory in case it is not existing
-		if err := os.MkdirAll(filePath, os.FileMode(0744)); err != nil {
-			return err
-		}
-
-		// generate index.html file in the directory
-		filePath = filepath.Join(filePath, "index.html")
-		err := ioutil.WriteFile(filePath, content, os.FileMode(0664))
-		if err != nil {
-			return err
-		}
-
-		c.Lock()
-		c.StaticFiles[requestPath] = filepath.ToSlash(filePath)
-		c.Unlock()
-	}
-
-	return nil
-}
-
-// defaultTemplate constructs a default template.
-func defaultTemplate(c *Config, metadata Metadata, requestPath string) []byte {
-	var scripts, styles bytes.Buffer
-	for _, style := range c.Styles {
-		styles.WriteString(strings.Replace(cssTemplate, "{{url}}", style, 1))
-		styles.WriteString("\r\n")
-	}
-	for _, script := range c.Scripts {
-		scripts.WriteString(strings.Replace(jsTemplate, "{{url}}", script, 1))
-		scripts.WriteString("\r\n")
-	}
-
-	// Title is first line (length-limited), otherwise filename
-	title, _ := metadata.Variables["title"]
-
-	html := []byte(htmlTemplate)
-	html = bytes.Replace(html, []byte("{{title}}"), []byte(title), 1)
-	html = bytes.Replace(html, []byte("{{css}}"), styles.Bytes(), 1)
-	html = bytes.Replace(html, []byte("{{js}}"), scripts.Bytes(), 1)
-
-	return html
 }
 
 const (
 	htmlTemplate = `<!DOCTYPE html>
 <html>
 	<head>
-		<title>{{title}}</title>
+		<title>{{.Doc.title}}</title>
 		<meta charset="utf-8">
-		{{css}}
+		{{range .Styles}}<link rel="stylesheet" href="{{.}}">
-		{{js}}
+		{{end -}}
+		{{range .Scripts}}<script src="{{.}}"></script>
+		{{end -}}
 	</head>
 	<body>
 		{{.Doc.body}}
 	</body>
 </html>`
-	cssTemplate = `<link rel="stylesheet" href="{{url}}">`
-	jsTemplate  = `<script src="{{url}}"></script>`
 )

middleware/markdown/renderer.go

@@ -1,139 +0,0 @@
-package markdown
-
-import (
-	"bytes"
-)
-
-// SummaryRenderer represents a summary renderer.
-type SummaryRenderer struct{}
-
-// Block-level callbacks
-
-// BlockCode is the code tag callback.
-func (r SummaryRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {}
-
-// BlockQuote is the quote tag callback.
-func (r SummaryRenderer) BlockQuote(out *bytes.Buffer, text []byte) {}
-
-// BlockHtml is the HTML tag callback.
-func (r SummaryRenderer) BlockHtml(out *bytes.Buffer, text []byte) {}
-
-// Header is the header tag callback.
-func (r SummaryRenderer) Header(out *bytes.Buffer, text func() bool, level int, id string) {}
-
-// HRule is the horizontal rule tag callback.
-func (r SummaryRenderer) HRule(out *bytes.Buffer) {}
-
-// List is the list tag callback.
-func (r SummaryRenderer) List(out *bytes.Buffer, text func() bool, flags int) {
-	// TODO: This is not desired (we'd rather not write lists as part of summary),
-	// but see this issue: https://github.com/russross/blackfriday/issues/189
-	marker := out.Len()
-	if !text() {
-		out.Truncate(marker)
-	}
-	out.Write([]byte{' '})
-}
-
-// ListItem is the list item tag callback.
-func (r SummaryRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {}
-
-// Paragraph is the paragraph tag callback.
-func (r SummaryRenderer) Paragraph(out *bytes.Buffer, text func() bool) {
-	marker := out.Len()
-	if !text() {
-		out.Truncate(marker)
-	}
-	out.Write([]byte{' '})
-}
-
-// Table is the table tag callback.
-func (r SummaryRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) {}
-
-// TableRow is the table row tag callback.
-func (r SummaryRenderer) TableRow(out *bytes.Buffer, text []byte) {}
-
-// TableHeaderCell is the table header cell tag callback.
-func (r SummaryRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {}
-
-// TableCell is the table cell tag callback.
-func (r SummaryRenderer) TableCell(out *bytes.Buffer, text []byte, flags int) {}
-
-// Footnotes is the foot notes tag callback.
-func (r SummaryRenderer) Footnotes(out *bytes.Buffer, text func() bool) {}
-
-// FootnoteItem is the footnote item tag callback.
-func (r SummaryRenderer) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {}
-
-// TitleBlock is the title tag callback.
-func (r SummaryRenderer) TitleBlock(out *bytes.Buffer, text []byte) {}
-
-// Span-level callbacks
-
-// AutoLink is the autolink tag callback.
-func (r SummaryRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {}
-
-// CodeSpan is the code span tag callback.
-func (r SummaryRenderer) CodeSpan(out *bytes.Buffer, text []byte) {
-	out.Write([]byte("`"))
-	out.Write(text)
-	out.Write([]byte("`"))
-}
-
-// DoubleEmphasis is the double emphasis tag callback.
-func (r SummaryRenderer) DoubleEmphasis(out *bytes.Buffer, text []byte) {
-	out.Write(text)
-}
-
-// Emphasis is the emphasis tag callback.
-func (r SummaryRenderer) Emphasis(out *bytes.Buffer, text []byte) {
-	out.Write(text)
-}
-
-// Image is the image tag callback.
-func (r SummaryRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {}
-
-// LineBreak is the line break tag callback.
-func (r SummaryRenderer) LineBreak(out *bytes.Buffer) {}
-
-// Link is the link tag callback.
-func (r SummaryRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
-	out.Write(content)
-}
-
-// RawHtmlTag is the raw HTML tag callback.
-func (r SummaryRenderer) RawHtmlTag(out *bytes.Buffer, tag []byte) {}
-
-// TripleEmphasis is the triple emphasis tag callback.
-func (r SummaryRenderer) TripleEmphasis(out *bytes.Buffer, text []byte) {
-	out.Write(text)
-}
-
-// StrikeThrough is the strikethrough tag callback.
-func (r SummaryRenderer) StrikeThrough(out *bytes.Buffer, text []byte) {}
-
-// FootnoteRef is the footnote ref tag callback.
-func (r SummaryRenderer) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {}
-
-// Low-level callbacks
-
-// Entity callback.
-func (r SummaryRenderer) Entity(out *bytes.Buffer, entity []byte) {
-	out.Write(entity)
-}
-
-// NormalText callback.
-func (r SummaryRenderer) NormalText(out *bytes.Buffer, text []byte) {
-	out.Write(text)
-}
-
-// Header and footer
-
-// DocumentHeader callback.
-func (r SummaryRenderer) DocumentHeader(out *bytes.Buffer) {}
-
-// DocumentFooter callback.
-func (r SummaryRenderer) DocumentFooter(out *bytes.Buffer) {}
-
-// GetFlags returns zero.
-func (r SummaryRenderer) GetFlags() int { return 0 }

middleware/markdown/watcher.go

@@ -1,42 +0,0 @@
-package markdown
-
-import (
-	"log"
-	"time"
-)
-
-// DefaultInterval is the default interval at which the markdown watcher
-// checks for changes.
-const DefaultInterval = time.Second * 60
-
-// Watch monitors the configured markdown directory for changes. It calls GenerateLinks
-// when there are changes.
-func Watch(md Markdown, c *Config, interval time.Duration) (stopChan chan struct{}) {
-	return TickerFunc(interval, func() {
-		if err := GenerateStatic(md, c); err != nil {
-			log.Printf("[ERROR] markdown: Re-generating static site: %v", err)
-		}
-	})
-}
-
-// TickerFunc runs f at interval. A message to the returned channel will stop the
-// executing goroutine.
-func TickerFunc(interval time.Duration, f func()) chan struct{} {
-	stopChan := make(chan struct{})
-
-	ticker := time.NewTicker(interval)
-	go func() {
-	loop:
-		for {
-			select {
-			case <-ticker.C:
-				f()
-			case <-stopChan:
-				ticker.Stop()
-				break loop
-			}
-		}
-	}()
-
-	return stopChan
-}

middleware/markdown/watcher_test.go

@@ -1,50 +0,0 @@
-package markdown
-
-import (
-	"fmt"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-)
-
-func TestWatcher(t *testing.T) {
-	expected := "12345678"
-	interval := time.Millisecond * 100
-	i := 0
-	out := ""
-	syncChan := make(chan struct{})
-	stopChan := TickerFunc(interval, func() {
-		i++
-		out += fmt.Sprint(i)
-		syncChan <- struct{}{}
-	})
-	sleepInSync(8, syncChan, stopChan)
-	if out != expected {
-		t.Fatalf("Expected to have prefix %v, found %v", expected, out)
-	}
-	out = ""
-	i = 0
-	var mu sync.Mutex
-	stopChan = TickerFunc(interval, func() {
-		i++
-		mu.Lock()
-		out += fmt.Sprint(i)
-		mu.Unlock()
-		syncChan <- struct{}{}
-	})
-	sleepInSync(9, syncChan, stopChan)
-	mu.Lock()
-	res := out
-	mu.Unlock()
-	if !strings.HasPrefix(res, expected) || res == expected {
-		t.Fatalf("expected (%v) must be a proper prefix of out(%v).", expected, out)
-	}
-}
-
-func sleepInSync(times int, syncChan chan struct{}, stopChan chan struct{}) {
-	for i := 0; i < times; i++ {
-		<-syncChan
-	}
-	stopChan <- struct{}{}
-}