diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go index caca1d597..6a9f71d40 100644 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -1,7 +1,6 @@ -// FastCGI is middleware that acts as a FastCGI client. Requests -// that get forwarded to FastCGI stop the middleware execution -// chain. The most common use for this layer is to serve PHP -// websites with php-fpm. +// Package fastcgi has middleware that acts as a FastCGI client. Requests +// that get forwarded to FastCGI stop the middleware execution chain. +// The most common use for this layer is to serve PHP websites via php-fpm. package fastcgi import ( @@ -13,8 +12,6 @@ import ( "strings" "github.com/mholt/caddy/middleware" - - "bitbucket.org/PinIdea/fcgi_client" // TODO: Inline this dependency. It'll need some work. ) // New generates a new FastCGI middleware. @@ -62,6 +59,7 @@ func New(c middleware.Controller) (middleware.Middleware, error) { } // TODO: Do we really have to make this map from scratch for each request? + // TODO: We have quite a few more to map, too. env := make(map[string]string) env["SERVER_SOFTWARE"] = "caddy" // TODO: Obtain version info... env["SERVER_PROTOCOL"] = r.Proto @@ -73,7 +71,7 @@ func New(c middleware.Controller) (middleware.Middleware, error) { env["DOCUMENT_URI"] = r.URL.Path env["DOCUMENT_ROOT"] = absRootPath - fcgi, err := fcgiclient.Dial("tcp", rule.address) + fcgi, err := Dial("tcp", rule.address) if err != nil { // TODO! } diff --git a/middleware/fastcgi/fcgi_test.php b/middleware/fastcgi/fcgi_test.php new file mode 100644 index 000000000..3f5e5f2db --- /dev/null +++ b/middleware/fastcgi/fcgi_test.php @@ -0,0 +1,79 @@ + $val) { + $md5 = md5($val); + + if ($key != $md5) { + $stat = "FAILED"; + echo "server:err ".$md5." != ".$key."\n"; + } + + $length += strlen($key) + strlen($val); + + $ret .= $key."(".strlen($key).") "; + } + $ret .= "] ["; + foreach ($_FILES as $k0 => $val) { + + $error = $val["error"]; + if ($error == UPLOAD_ERR_OK) { + $tmp_name = $val["tmp_name"]; + $name = $val["name"]; + $datafile = "/tmp/test.go"; + move_uploaded_file($tmp_name, $datafile); + $md5 = md5_file($datafile); + + if ($k0 != $md5) { + $stat = "FAILED"; + echo "server:err ".$md5." != ".$key."\n"; + } + + $length += strlen($k0) + filesize($datafile); + + unlink($datafile); + $ret .= $k0."(".strlen($k0).") "; + } + else{ + $stat = "FAILED"; + echo "server:file err ".file_upload_error_message($error)."\n"; + } + } + $ret .= "]"; + echo "server:got data length " .$length."\n"; +} + + +echo "-{$stat}-POST(".count($_POST).") FILE(".count($_FILES).")\n"; + +function file_upload_error_message($error_code) { + switch ($error_code) { + case UPLOAD_ERR_INI_SIZE: + return 'The uploaded file exceeds the upload_max_filesize directive in php.ini'; + case UPLOAD_ERR_FORM_SIZE: + return 'The uploaded file exceeds the MAX_FILE_SIZE directive that was specified in the HTML form'; + case UPLOAD_ERR_PARTIAL: + return 'The uploaded file was only partially uploaded'; + case UPLOAD_ERR_NO_FILE: + return 'No file was uploaded'; + case UPLOAD_ERR_NO_TMP_DIR: + return 'Missing a temporary folder'; + case UPLOAD_ERR_CANT_WRITE: + return 'Failed to write file to disk'; + case UPLOAD_ERR_EXTENSION: + return 'File upload stopped by extension'; + default: + return 'Unknown upload error'; + } +} \ No newline at end of file diff --git a/middleware/fastcgi/fcgiclient.go b/middleware/fastcgi/fcgiclient.go new file mode 100644 index 000000000..baf2bbbfb --- /dev/null +++ b/middleware/fastcgi/fcgiclient.go @@ -0,0 +1,462 @@ +// Forked Jan. 2015 from http://bitbucket.org/PinIdea/fcgi_client +// (which is forked from https://code.google.com/p/go-fastcgi-client/) + +// This fork contains several fixes and improvements by Matt Holt and +// other contributors to this project. + +// Copyright 2012 Junqing Tan and The Go Authors +// Use of this source code is governed by a BSD-style +// Part of source code is from Go fcgi package + +package fastcgi + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "io" + "io/ioutil" + "mime/multipart" + "net" + "net/http" + "net/http/httputil" + "net/textproto" + "net/url" + "os" + "path/filepath" + "strconv" + "sync" +) + +const FCGI_LISTENSOCK_FILENO uint8 = 0 +const FCGI_HEADER_LEN uint8 = 8 +const VERSION_1 uint8 = 1 +const FCGI_NULL_REQUEST_ID uint8 = 0 +const FCGI_KEEP_CONN uint8 = 1 +const doubleCRLF = "\r\n\r\n" + +const ( + FCGI_BEGIN_REQUEST uint8 = iota + 1 + FCGI_ABORT_REQUEST + FCGI_END_REQUEST + FCGI_PARAMS + FCGI_STDIN + FCGI_STDOUT + FCGI_STDERR + FCGI_DATA + FCGI_GET_VALUES + FCGI_GET_VALUES_RESULT + FCGI_UNKNOWN_TYPE + FCGI_MAXTYPE = FCGI_UNKNOWN_TYPE +) + +const ( + FCGI_RESPONDER uint8 = iota + 1 + FCGI_AUTHORIZER + FCGI_FILTER +) + +const ( + FCGI_REQUEST_COMPLETE uint8 = iota + FCGI_CANT_MPX_CONN + FCGI_OVERLOADED + FCGI_UNKNOWN_ROLE +) + +const ( + FCGI_MAX_CONNS string = "MAX_CONNS" + FCGI_MAX_REQS string = "MAX_REQS" + FCGI_MPXS_CONNS string = "MPXS_CONNS" +) + +const ( + maxWrite = 65500 // 65530 may work, but for compatibility + maxPad = 255 +) + +type header struct { + Version uint8 + Type uint8 + Id uint16 + ContentLength uint16 + PaddingLength uint8 + Reserved uint8 +} + +// for padding so we don't have to allocate all the time +// not synchronized because we don't care what the contents are +var pad [maxPad]byte + +func (h *header) init(recType uint8, reqId uint16, contentLength int) { + h.Version = 1 + h.Type = recType + h.Id = reqId + h.ContentLength = uint16(contentLength) + h.PaddingLength = uint8(-contentLength & 7) +} + +type record struct { + h header + rbuf []byte +} + +func (rec *record) read(r io.Reader) (buf []byte, err error) { + if err = binary.Read(r, binary.BigEndian, &rec.h); err != nil { + return + } + if rec.h.Version != 1 { + err = errors.New("fcgi: invalid header version") + return + } + if rec.h.Type == FCGI_END_REQUEST { + err = io.EOF + return + } + n := int(rec.h.ContentLength) + int(rec.h.PaddingLength) + if len(rec.rbuf) < n { + rec.rbuf = make([]byte, n) + } + if n, err = io.ReadFull(r, rec.rbuf[:n]); err != nil { + return + } + buf = rec.rbuf[:int(rec.h.ContentLength)] + + return +} + +type FCGIClient struct { + mutex sync.Mutex + rwc io.ReadWriteCloser + h header + buf bytes.Buffer + keepAlive bool + reqId uint16 +} + +// Connects to the fcgi responder at the specified network address. +// See func net.Dial for a description of the network and address parameters. +func Dial(network, address string) (fcgi *FCGIClient, err error) { + var conn net.Conn + + conn, err = net.Dial(network, address) + if err != nil { + return + } + + fcgi = &FCGIClient{ + rwc: conn, + keepAlive: false, + reqId: 1, + } + + return +} + +// Close fcgi connnection +func (this *FCGIClient) Close() { + this.rwc.Close() +} + +func (this *FCGIClient) writeRecord(recType uint8, content []byte) (err error) { + this.mutex.Lock() + defer this.mutex.Unlock() + this.buf.Reset() + this.h.init(recType, this.reqId, len(content)) + if err := binary.Write(&this.buf, binary.BigEndian, this.h); err != nil { + return err + } + if _, err := this.buf.Write(content); err != nil { + return err + } + if _, err := this.buf.Write(pad[:this.h.PaddingLength]); err != nil { + return err + } + _, err = this.rwc.Write(this.buf.Bytes()) + return err +} + +func (this *FCGIClient) writeBeginRequest(role uint16, flags uint8) error { + b := [8]byte{byte(role >> 8), byte(role), flags} + return this.writeRecord(FCGI_BEGIN_REQUEST, b[:]) +} + +func (this *FCGIClient) writeEndRequest(appStatus int, protocolStatus uint8) error { + b := make([]byte, 8) + binary.BigEndian.PutUint32(b, uint32(appStatus)) + b[4] = protocolStatus + return this.writeRecord(FCGI_END_REQUEST, b) +} + +func (this *FCGIClient) writePairs(recType uint8, pairs map[string]string) error { + w := newWriter(this, recType) + b := make([]byte, 8) + nn := 0 + for k, v := range pairs { + m := 8 + len(k) + len(v) + if m > maxWrite { + // param data size exceed 65535 bytes" + vl := maxWrite - 8 - len(k) + v = v[:vl] + } + n := encodeSize(b, uint32(len(k))) + n += encodeSize(b[n:], uint32(len(v))) + m = n + len(k) + len(v) + if (nn + m) > maxWrite { + w.Flush() + nn = 0 + } + nn += m + if _, err := w.Write(b[:n]); err != nil { + return err + } + if _, err := w.WriteString(k); err != nil { + return err + } + if _, err := w.WriteString(v); err != nil { + return err + } + } + w.Close() + return nil +} + +func readSize(s []byte) (uint32, int) { + if len(s) == 0 { + return 0, 0 + } + size, n := uint32(s[0]), 1 + if size&(1<<7) != 0 { + if len(s) < 4 { + return 0, 0 + } + n = 4 + size = binary.BigEndian.Uint32(s) + size &^= 1 << 31 + } + return size, n +} + +func readString(s []byte, size uint32) string { + if size > uint32(len(s)) { + return "" + } + return string(s[:size]) +} + +func encodeSize(b []byte, size uint32) int { + if size > 127 { + size |= 1 << 31 + binary.BigEndian.PutUint32(b, size) + return 4 + } + b[0] = byte(size) + return 1 +} + +// bufWriter encapsulates bufio.Writer but also closes the underlying stream when +// Closed. +type bufWriter struct { + closer io.Closer + *bufio.Writer +} + +func (w *bufWriter) Close() error { + if err := w.Writer.Flush(); err != nil { + w.closer.Close() + return err + } + return w.closer.Close() +} + +func newWriter(c *FCGIClient, recType uint8) *bufWriter { + s := &streamWriter{c: c, recType: recType} + w := bufio.NewWriterSize(s, maxWrite) + return &bufWriter{s, w} +} + +// streamWriter abstracts out the separation of a stream into discrete records. +// It only writes maxWrite bytes at a time. +type streamWriter struct { + c *FCGIClient + recType uint8 +} + +func (w *streamWriter) Write(p []byte) (int, error) { + nn := 0 + for len(p) > 0 { + n := len(p) + if n > maxWrite { + n = maxWrite + } + if err := w.c.writeRecord(w.recType, p[:n]); err != nil { + return nn, err + } + nn += n + p = p[n:] + } + return nn, nil +} + +func (w *streamWriter) Close() error { + // send empty record to close the stream + return w.c.writeRecord(w.recType, nil) +} + +type streamReader struct { + c *FCGIClient + buf []byte +} + +func (w *streamReader) Read(p []byte) (n int, err error) { + + if len(p) > 0 { + if len(w.buf) == 0 { + rec := &record{} + w.buf, err = rec.read(w.c.rwc) + if err != nil { + return + } + } + + n = len(p) + if n > len(w.buf) { + n = len(w.buf) + } + copy(p, w.buf[:n]) + w.buf = w.buf[n:] + } + + return +} + +// Do made the request and returns a io.Reader that translates the data read +// from fcgi responder out of fcgi packet before returning it. +func (this *FCGIClient) Do(p map[string]string, req io.Reader) (r io.Reader, err error) { + err = this.writeBeginRequest(uint16(FCGI_RESPONDER), 0) + if err != nil { + return + } + + err = this.writePairs(FCGI_PARAMS, p) + if err != nil { + return + } + + body := newWriter(this, FCGI_STDIN) + if req != nil { + io.Copy(body, req) + } + body.Close() + + r = &streamReader{c: this} + return +} + +// Request returns a HTTP Response with Header and Body +// from fcgi responder +func (this *FCGIClient) Request(p map[string]string, req io.Reader) (resp *http.Response, err error) { + + r, err := this.Do(p, req) + if err != nil { + return + } + + rb := bufio.NewReader(r) + tp := textproto.NewReader(rb) + resp = new(http.Response) + + // Parse the response headers. + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil && err != io.EOF { + return + } + resp.Header = http.Header(mimeHeader) + + // TODO: fixTransferEncoding ? + resp.TransferEncoding = resp.Header["Transfer-Encoding"] + resp.ContentLength, _ = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + + if chunked(resp.TransferEncoding) { + resp.Body = ioutil.NopCloser(httputil.NewChunkedReader(rb)) + } else { + resp.Body = ioutil.NopCloser(rb) + } + + return +} + +// Get issues a GET request to the fcgi responder. +func (this *FCGIClient) Get(p map[string]string) (resp *http.Response, err error) { + + p["REQUEST_METHOD"] = "GET" + p["CONTENT_LENGTH"] = "0" + + return this.Request(p, nil) +} + +// Get issues a Post request to the fcgi responder. with request body +// in the format that bodyType specified +func (this *FCGIClient) Post(p map[string]string, bodyType string, body io.Reader, l int) (resp *http.Response, err error) { + + if len(p["REQUEST_METHOD"]) == 0 || p["REQUEST_METHOD"] == "GET" { + p["REQUEST_METHOD"] = "POST" + } + p["CONTENT_LENGTH"] = strconv.Itoa(l) + if len(bodyType) > 0 { + p["CONTENT_TYPE"] = bodyType + } else { + p["CONTENT_TYPE"] = "application/x-www-form-urlencoded" + } + + return this.Request(p, body) +} + +// PostForm issues a POST to the fcgi responder, with form +// as a string key to a list values (url.Values) +func (this *FCGIClient) PostForm(p map[string]string, data url.Values) (resp *http.Response, err error) { + body := bytes.NewReader([]byte(data.Encode())) + return this.Post(p, "application/x-www-form-urlencoded", body, body.Len()) +} + +// PostFile issues a POST to the fcgi responder in multipart(RFC 2046) standard, +// with form as a string key to a list values (url.Values), +// and/or with file as a string key to a list file path. +func (this *FCGIClient) PostFile(p map[string]string, data url.Values, file map[string]string) (resp *http.Response, err error) { + buf := &bytes.Buffer{} + writer := multipart.NewWriter(buf) + bodyType := writer.FormDataContentType() + + for key, val := range data { + for _, v0 := range val { + err = writer.WriteField(key, v0) + if err != nil { + return + } + } + } + + for key, val := range file { + fd, e := os.Open(val) + if e != nil { + return nil, e + } + defer fd.Close() + + part, e := writer.CreateFormFile(key, filepath.Base(val)) + if e != nil { + return nil, e + } + _, err = io.Copy(part, fd) + } + + err = writer.Close() + if err != nil { + return + } + + return this.Post(p, bodyType, buf, buf.Len()) +} + +// Checks whether chunked is part of the encodings stack +func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } diff --git a/middleware/fastcgi/fcgiclient_test.go b/middleware/fastcgi/fcgiclient_test.go new file mode 100644 index 000000000..c18917bbd --- /dev/null +++ b/middleware/fastcgi/fcgiclient_test.go @@ -0,0 +1,276 @@ +package fastcgi + +import ( + "bytes" + "crypto/md5" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" + "net" + "net/http" + "net/http/fcgi" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// test fcgi protocol includes: +// Get, Post, Post in multipart/form-data, and Post with files +// each key should be the md5 of the value or the file uploaded +// sepicify remote fcgi responer ip:port to test with php +// test failed if the remote fcgi(script) failed md5 verification +// and output "FAILED" in response +const ( + script_file = "/tank/www/fcgic_test.php" + //ip_port = "remote-php-serv:59000" + ip_port = "127.0.0.1:59000" +) + +var ( + t_ *testing.T = nil +) + +type FastCGIServer struct{} + +func (s FastCGIServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + + req.ParseMultipartForm(100000000) + + stat := "PASSED" + fmt.Fprintln(resp, "-") + file_num := 0 + { + length := 0 + for k0, v0 := range req.Form { + h := md5.New() + io.WriteString(h, v0[0]) + md5 := fmt.Sprintf("%x", h.Sum(nil)) + + length += len(k0) + length += len(v0[0]) + + // echo error when key != md5(val) + if md5 != k0 { + fmt.Fprintln(resp, "server:err ", md5, k0) + stat = "FAILED" + } + } + if req.MultipartForm != nil { + file_num = len(req.MultipartForm.File) + for kn, fns := range req.MultipartForm.File { + //fmt.Fprintln(resp, "server:filekey ", kn ) + length += len(kn) + for _, f := range fns { + fd, err := f.Open() + if err != nil { + log.Println("server:", err) + return + } + h := md5.New() + l0, err := io.Copy(h, fd) + if err != nil { + log.Println(err) + return + } + length += int(l0) + defer fd.Close() + md5 := fmt.Sprintf("%x", h.Sum(nil)) + //fmt.Fprintln(resp, "server:filemd5 ", md5 ) + + if kn != md5 { + fmt.Fprintln(resp, "server:err ", md5, kn) + stat = "FAILED" + } + //fmt.Fprintln(resp, "server:filename ", f.Filename ) + } + } + } + + fmt.Fprintln(resp, "server:got data length", length) + } + fmt.Fprintln(resp, "-"+stat+"-POST(", len(req.Form), ")-FILE(", file_num, ")--") +} + +func sendFcgi(reqType int, fcgi_params map[string]string, data []byte, posts map[string]string, files map[string]string) (content []byte) { + fcgi, err := Dial("tcp", ip_port) + if err != nil { + log.Println("err:", err) + return + } + + length := 0 + + var resp *http.Response + switch reqType { + case 0: + if len(data) > 0 { + length = len(data) + rd := bytes.NewReader(data) + resp, err = fcgi.Post(fcgi_params, "", rd, rd.Len()) + } else if len(posts) > 0 { + values := url.Values{} + for k, v := range posts { + values.Set(k, v) + length += len(k) + 2 + len(v) + } + resp, err = fcgi.PostForm(fcgi_params, values) + } else { + resp, err = fcgi.Get(fcgi_params) + } + + default: + values := url.Values{} + for k, v := range posts { + values.Set(k, v) + length += len(k) + 2 + len(v) + } + + for k, v := range files { + fi, _ := os.Lstat(v) + length += len(k) + int(fi.Size()) + } + resp, err = fcgi.PostFile(fcgi_params, values, files) + } + + if err != nil { + log.Println("err:", err) + return + } + + defer resp.Body.Close() + content, err = ioutil.ReadAll(resp.Body) + + log.Println("c: send data length ≈", length, string(content)) + fcgi.Close() + time.Sleep(1 * time.Second) + + if bytes.Index(content, []byte("FAILED")) >= 0 { + t_.Error("Server return failed message") + } + + return +} + +func generateRandFile(size int) (p string, m string) { + + p = filepath.Join(os.TempDir(), "fcgict"+strconv.Itoa(rand.Int())) + + // open output file + fo, err := os.Create(p) + if err != nil { + panic(err) + } + // close fo on exit and check for its returned error + defer func() { + if err := fo.Close(); err != nil { + panic(err) + } + }() + + h := md5.New() + for i := 0; i < size/16; i++ { + buf := make([]byte, 16) + binary.PutVarint(buf, rand.Int63()) + fo.Write(buf) + h.Write(buf) + } + m = fmt.Sprintf("%x", h.Sum(nil)) + return +} + +func Test(t *testing.T) { + // TODO: test chunked reader + + t_ = t + rand.Seed(time.Now().UTC().UnixNano()) + + // server + go func() { + listener, err := net.Listen("tcp", ip_port) + if err != nil { + // handle error + log.Println("listener creatation failed: ", err) + } + + srv := new(FastCGIServer) + fcgi.Serve(listener, srv) + }() + + time.Sleep(1 * time.Second) + + // init + fcgi_params := make(map[string]string) + fcgi_params["REQUEST_METHOD"] = "GET" + fcgi_params["SERVER_PROTOCOL"] = "HTTP/1.1" + //fcgi_params["GATEWAY_INTERFACE"] = "CGI/1.1" + fcgi_params["SCRIPT_FILENAME"] = script_file + + // simple GET + log.Println("test:", "get") + sendFcgi(0, fcgi_params, nil, nil, nil) + + // simple post data + log.Println("test:", "post") + sendFcgi(0, fcgi_params, []byte("c4ca4238a0b923820dcc509a6f75849b=1&7b8b965ad4bca0e41ab51de7b31363a1=n"), nil, nil) + + log.Println("test:", "post data (more than 60KB)") + data := "" + length := 0 + for i := 0x00; i < 0xff; i++ { + v0 := strings.Repeat(string(i), 256) + h := md5.New() + io.WriteString(h, v0) + k0 := fmt.Sprintf("%x", h.Sum(nil)) + + length += len(k0) + length += len(v0) + + data += k0 + "=" + url.QueryEscape(v0) + "&" + } + sendFcgi(0, fcgi_params, []byte(data), nil, nil) + + log.Println("test:", "post form (use url.Values)") + p0 := make(map[string]string, 1) + p0["c4ca4238a0b923820dcc509a6f75849b"] = "1" + p0["7b8b965ad4bca0e41ab51de7b31363a1"] = "n" + sendFcgi(1, fcgi_params, nil, p0, nil) + + log.Println("test:", "post forms (256 keys, more than 1MB)") + p1 := make(map[string]string, 1) + for i := 0x00; i < 0xff; i++ { + v0 := strings.Repeat(string(i), 4096) + h := md5.New() + io.WriteString(h, v0) + k0 := fmt.Sprintf("%x", h.Sum(nil)) + p1[k0] = v0 + } + sendFcgi(1, fcgi_params, nil, p1, nil) + + log.Println("test:", "post file (1 file, 500KB)) ") + f0 := make(map[string]string, 1) + path0, m0 := generateRandFile(500000) + f0[m0] = path0 + sendFcgi(1, fcgi_params, nil, p1, f0) + + log.Println("test:", "post multiple files (2 files, 5M each) and forms (256 keys, more than 1MB data") + path1, m1 := generateRandFile(5000000) + f0[m1] = path1 + sendFcgi(1, fcgi_params, nil, p1, f0) + + log.Println("test:", "post only files (2 files, 5M each)") + sendFcgi(1, fcgi_params, nil, nil, f0) + + log.Println("test:", "post only 1 file") + delete(f0, "m0") + sendFcgi(1, fcgi_params, nil, nil, f0) + + os.Remove(path0) + os.Remove(path1) +}