move mime parser to its own package

api_test.go

@@ -397,7 +397,7 @@ func TestSetAddProcessor(t *testing.T) {
 		t.Error(err)
 	}
 	// lets have a talk with the server
-	if err := talkToServer("127.0.0.1:2525"); err != nil {
+	if err := talkToServer("127.0.0.1:2525", ""); err != nil {
 		t.Error(err)
 	}
 
@@ -426,7 +426,7 @@ func TestSetAddProcessor(t *testing.T) {
 
 }
 
-func talkToServer(address string) (err error) {
+func talkToServer(address string, body string) (err error) {
 
 	conn, err := net.Dial("tcp", address)
 	if err != nil {
@@ -472,80 +472,32 @@ func talkToServer(address string) (err error) {
 	if err != nil {
 		return err
 	}
-	_, err = fmt.Fprint(conn, "Subject: Test subject\r\n")
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprint(conn, "\r\n")
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprint(conn, "A an email body\r\n")
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprint(conn, ".\r\n")
-	if err != nil {
-		return err
-	}
-	str, err = in.ReadString('\n')
-	if err != nil {
-		return err
-	}
-	_ = str
-	return nil
-}
-
-func talkToServer2(address string, body string) (err error) {
-
-	conn, err := net.Dial("tcp", address)
-	if err != nil {
-		return
-	}
-	in := bufio.NewReader(conn)
-	str, err := in.ReadString('\n')
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprint(conn, "HELO maildiranasaurustester\r\n")
-	if err != nil {
-		return err
-	}
-	str, err = in.ReadString('\n')
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprint(conn, "MAIL FROM:<[email protected]>r\r\n")
-	if err != nil {
-		return err
-	}
-	str, err = in.ReadString('\n')
-	if err != nil {
-		return err
-	}
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprint(conn, "RCPT TO:<[email protected]>\r\n")
-	if err != nil {
-		return err
-	}
-	str, err = in.ReadString('\n')
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprint(conn, "DATA\r\n")
-	if err != nil {
-		return err
-	}
-	str, err = in.ReadString('\n')
-	if err != nil {
-		return err
-	}
-
-	_, err = fmt.Fprint(conn, body)
-	if err != nil {
-		return err
+	if body == "" {
+		_, err = fmt.Fprint(conn, "Subject: Test subject\r\n")
+		if err != nil {
+			return err
+		}
+		_, err = fmt.Fprint(conn, "\r\n")
+		if err != nil {
+			return err
+		}
+		_, err = fmt.Fprint(conn, "A an email body\r\n")
+		if err != nil {
+			return err
+		}
+		_, err = fmt.Fprint(conn, ".\r\n")
+		if err != nil {
+			return err
+		}
+	} else {
+		_, err = fmt.Fprint(conn, body)
+		if err != nil {
+			return err
+		}
+		_, err = fmt.Fprint(conn, ".\r\n")
+		if err != nil {
+			return err
+		}
 	}
 
 	str, err = in.ReadString('\n')
@@ -748,7 +700,7 @@ func TestCustomBackendResult(t *testing.T) {
 		t.Error(err)
 	}
 	// lets have a talk with the server
-	if err := talkToServer("127.0.0.1:2525"); err != nil {
+	if err := talkToServer("127.0.0.1:2525", ""); err != nil {
 		t.Error(err)
 	}
 
@@ -801,11 +753,172 @@ func TestStreamProcessor(t *testing.T) {
 		"Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" +
 		"Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" +
 		"Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" +
-		"Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n" +
+		"Header|headersparser|compress|Decompress|debug Header|headersparser|compress|Decompress|debug.\r\n"
+
+	// lets have a talk with the server
+	if err := talkToServer("127.0.0.1:2525", body); err != nil {
+		t.Error(err)
+	}
+
+	d.Shutdown()
+
+	b, err := ioutil.ReadFile("tests/testlog")
+	if err != nil {
+		t.Error("could not read logfile")
+		return
+	}
+
+	// lets check for fingerprints
+	if strings.Index(string(b), "Debug stream") < 0 {
+		t.Error("did not log: Debug stream")
+	}
+
+	if strings.Index(string(b), "Error") != -1 {
+		t.Error("There was an error", string(b))
+	}
+
+}
+
+var mime = `MIME-Version: 1.0
+X-Mailer: MailBee.NET 8.0.4.428
+Subject: test subject
+To: [email protected]
+Content-Type: multipart/mixed;
+       boundary="XXXXboundary text"
+
+--XXXXboundary text
+Content-Type: multipart/alternative;
+       boundary="XXXXboundary text"
+
+--XXXXboundary text
+Content-Type: text/plain;
+       charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+
+This is the body text of a sample message.
+--XXXXboundary text
+Content-Type: text/html;
+       charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+
+<pre>This is the body text of a sample message.</pre>
+
+--XXXXboundary text
+Content-Type: text/plain;
+ name="log_attachment.txt"
+Content-Disposition: attachment;
+ filename="log_attachment.txt"
+Content-Transfer-Encoding: base64
+
+TUlNRS1WZXJzaW9uOiAxLjANClgtTWFpbGVyOiBNYWlsQmVlLk5FVCA4LjAuNC40MjgNClN1Ympl
+Y3Q6IHRlc3Qgc3ViamVjdA0KVG86IGtldmlubUBkYXRhbW90aW9uLmNvbQ0KQ29udGVudC1UeXBl
+OiBtdWx0aXBhcnQvYWx0ZXJuYXRpdmU7DQoJYm91bmRhcnk9Ii0tLS09X05leHRQYXJ0XzAwMF9B
+RTZCXzcyNUUwOUFGLjg4QjdGOTM0Ig0KDQoNCi0tLS0tLT1fTmV4dFBhcnRfMDAwX0FFNkJfNzI1
+RTA5QUYuODhCN0Y5MzQNCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsNCgljaGFyc2V0PSJ1dGYt
+OCINCkNvbnRlbnQtVHJhbnNmZXItRW5jb2Rpbmc6IHF1b3RlZC1wcmludGFibGUNCg0KdGVzdCBi
+b2R5DQotLS0tLS09X05leHRQYXJ0XzAwMF9BRTZCXzcyNUUwOUFGLjg4QjdGOTM0DQpDb250ZW50
+LVR5cGU6IHRleHQvaHRtbDsNCgljaGFyc2V0PSJ1dGYtOCINCkNvbnRlbnQtVHJhbnNmZXItRW5j
+b2Rpbmc6IHF1b3RlZC1wcmludGFibGUNCg0KPHByZT50ZXN0IGJvZHk8L3ByZT4NCi0tLS0tLT1f
+TmV4dFBhcnRfMDAwX0FFNkJfNzI1RTA5QUYuODhCN0Y5MzQtLQ0K
+
+--XXXXboundary text--
+`
+
+var mime2 = `From: [email protected]
+Content-Type: multipart/mixed;
+        boundary="----_=_NextPart_001_01CBE273.65A0E7AA"
+To: [email protected]
+
+This is a multi-part message in MIME format.
+
+------_=_NextPart_001_01CBE273.65A0E7AA
+Content-Type: multipart/alternative;
+        boundary="----_=_NextPart_002_01CBE273.65A0E7AA"
+
+
+------_=_NextPart_002_01CBE273.65A0E7AA
+Content-Type: text/plain;
+        charset="UTF-8"
+Content-Transfer-Encoding: base64
+
+[base64-content]
+------_=_NextPart_002_01CBE273.65A0E7AA
+Content-Type: text/html;
+        charset="UTF-8"
+Content-Transfer-Encoding: base64
+
+[base64-content]
+------_=_NextPart_002_01CBE273.65A0E7AA--
+------_=_NextPart_001_01CBE273.65A0E7AA
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+
+X-MimeOLE: Produced By Microsoft Exchange V6.5
+Content-class: urn:content-classes:message
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+        boundary="----_=_NextPart_003_01CBE272.13692C80"
+From: [email protected]
+To: [email protected]
+
+This is a multi-part message in MIME format.
+
+------_=_NextPart_003_01CBE272.13692C80
+Content-Type: multipart/alternative;
+        boundary="----_=_NextPart_004_01CBE272.13692C80"
+
+
+------_=_NextPart_004_01CBE272.13692C80
+Content-Type: text/plain;
+        charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+=20
+
+Viele Gr=FC=DFe
+
+------_=_NextPart_004_01CBE272.13692C80
+Content-Type: text/html;
+        charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+<html>...</html>
+------_=_NextPart_004_01CBE272.13692C80--
+------_=_NextPart_003_01CBE272.13692C80
+Content-Type: application/x-zip-compressed;
+        name="abc.zip"
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment;
+        filename="abc.zip"
+
+[base64-content]
+
+------_=_NextPart_003_01CBE272.13692C80--
+------_=_NextPart_001_01CBE273.65A0E7AA--
+`
+
+func TestStreamMimeProcessor(t *testing.T) {
+	if err := os.Truncate("tests/testlog", 0); err != nil {
+		t.Error(err)
+	}
+	cfg := &AppConfig{
+		LogFile:      "tests/testlog",
+		AllowedHosts: []string{"grr.la"},
+		BackendConfig: backends.BackendConfig{
+			"save_process":        "HeadersParser|Debugger",
+			"stream_save_process": "Header|mimeanalyzer|headersparser|compress|Decompress|debug",
+		},
+	}
+	d := Daemon{Config: cfg}
+
+	if err := d.Start(); err != nil {
+		t.Error(err)
+	}
 
-		".\r\n"
+	// change \n to \r\n
+	mime = strings.Replace(mime, "\n", "\r\n", -1)
 	// lets have a talk with the server
-	if err := talkToServer2("127.0.0.1:2525", body); err != nil {
+	if err := talkToServer("127.0.0.1:2525", mime); err != nil {
 		t.Error(err)
 	}
 

backends/s_mime.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/flashmob/go-guerrilla/mail"
+	"github.com/flashmob/go-guerrilla/mail/mime"
 	"io"
 	"net/textproto"
 	"strconv"
@@ -28,867 +29,6 @@ func init() {
 	}
 }
 
-type mimepart struct {
-	/*
-			[starting-pos] => 0
-		    [starting-pos-body] => 270
-		    [ending-pos] => 2651
-		    [ending-pos-body] => 2651
-		    [line-count] => 72
-		    [body-line-count] => 65
-		    [charset] => us-ascii
-		    [transfer-encoding] => 8bit
-		    [content-boundary] => D7F------------D7FD5A0B8AB9C65CCDBFA872
-		    [content-type] => multipart/mixed
-		    [content-base] => /
-
-			[starting-pos] => 2023
-		    [starting-pos-body] => 2172
-		    [ending-pos] => 2561
-		    [ending-pos-body] => 2561
-		    [line-count] => 9
-		    [body-line-count] => 5
-		    [charset] => us-ascii
-		    [transfer-encoding] => base64
-		    [content-name] => map_of_Argentina.gif
-		    [content-type] => image/gif
-		    [disposition-fi1ename] => map_of_Argentina.gif
-		    [content-disposition] => in1ine
-		    [content-base] => /
-	*/
-}
-
-const (
-	maxBoundaryLen = 70 + 10
-	doubleDash     = "--"
-)
-
-type parser struct {
-	state int
-
-	accept bytes.Buffer
-
-	once            bool
-	boundaryMatched int
-
-	// related to the buffer
-	buf                         []byte
-	pos                         int
-	ch                          byte
-	gotNewSlice, consumed, halt chan bool
-	result                      chan parserMsg
-	isHalting                   bool
-
-	// mime variables
-	parts           []*mimeHeader
-	msgPos          uint
-	msgLine         uint
-	lastBoundaryPos uint
-}
-
-type mimeHeader struct {
-	headers textproto.MIMEHeader
-
-	part string
-
-	startingPos      uint // including header (after boundary, 0 at the top)
-	startingPosBody  uint // after header \n\n
-	endingPos        uint // redundant (same as endingPos)
-	endingPosBody    uint // the char before the boundary marker
-	lineCount        uint
-	bodyLineCount    uint
-	charset          string
-	transferEncoding string
-	contentBoundary  string
-	contentType      *contentType
-	contentBase      string
-
-	dispositionFi1eName string
-	contentDisposition  string
-	contentName         string
-}
-
-type contentType struct {
-	superType  string
-	subType    string
-	parameters map[string]string
-}
-
-func (c *contentType) String() string {
-	return fmt.Sprintf("%s/%s", c.superType, c.subType)
-}
-
-func NewMimeHeader() *mimeHeader {
-	mh := new(mimeHeader)
-	mh.headers = make(textproto.MIMEHeader, 1)
-	return mh
-}
-
-func (p *parser) addPart(mh *mimeHeader, id string) {
-	mh.part = id
-	p.parts = append(p.parts, mh)
-}
-
-func (p *parser) endBody(mh *mimeHeader) {
-
-}
-
-//
-func (p *parser) more() bool {
-	p.consumed <- true // signal that we've reached the end of available input
-	select {
-	// wait for a new new slice
-	case gotMore := <-p.gotNewSlice:
-		if !gotMore {
-			// no more data, closing
-			return false
-		}
-	case <-p.halt:
-		p.isHalting = true
-		return false
-	}
-	return true
-}
-
-func (p *parser) next() byte {
-	// wait for a new new slice if reached the end
-	if p.pos+1 >= len(p.buf) {
-		if !p.more() {
-			p.ch = 0
-			return 0
-		}
-	}
-
-	// go to the next byte
-	p.pos++
-	p.ch = p.buf[p.pos]
-	p.msgPos++
-	if p.ch == '\n' {
-		p.msgLine++
-	}
-	return p.ch
-}
-
-func (p *parser) peek() byte {
-
-	// reached the end?
-	if p.pos+1 >= len(p.buf) {
-		if !p.more() {
-			p.ch = 0
-			return 0
-		}
-	}
-
-	// peek the next byte
-	if p.pos+1 < len(p.buf) {
-		return p.buf[p.pos+1]
-	}
-	return 0
-}
-
-// simulate a byte stream
-func (p *parser) inject(input ...[]byte) {
-	p.msgPos = 0
-	p.set(input[0])
-	p.pos = 0
-	p.ch = p.buf[0]
-	go func() {
-		for i := 1; i < len(input); i++ {
-			<-p.consumed
-			p.set(input[i])
-			p.gotNewSlice <- true
-		}
-		<-p.consumed
-		p.gotNewSlice <- false // no more data
-	}()
-}
-
-func (p *parser) set(input []byte) {
-	if p.pos != -1 {
-		// rewind
-		p.pos = -1
-	}
-	p.buf = input
-
-}
-
-func (p *parser) skip(nBytes int) {
-
-	for i := 0; i < nBytes; i++ {
-		p.next()
-		if p.ch == 0 {
-			return
-		}
-	}
-
-}
-
-// boundary scans until next boundary string, returns error if not found
-// syntax specified https://tools.ietf.org/html/rfc2046 p21
-func (p *parser) boundary(contentBoundary string) (end bool, err error) {
-	defer func() {
-		if err == nil {
-			if p.ch == '\n' {
-				p.next()
-			}
-		}
-	}()
-
-	if len(contentBoundary) < 1 {
-		err = errors.New("content boundary too short")
-	}
-	boundary := doubleDash + contentBoundary
-	p.boundaryMatched = 0
-	for {
-		if i := bytes.Index(p.buf[p.pos:], []byte(boundary)); i > -1 {
-			// advance the pointer to 1 char before the end of the boundary
-			// then let next() to advance the last char.
-			// in case the boundary is the tail part of buffer, calling next()
-			// will wait until we get a new buffer
-
-			p.skip(i)
-			p.lastBoundaryPos = p.msgPos - 1 // - uint(len(boundary))
-			p.skip(len(boundary))
-			end, err = p.boundaryEnd()
-			if err != nil {
-				return
-			}
-			p.transportPadding()
-			if p.ch != '\n' && p.ch != 0 {
-				err = errors.New("boundary new line expected")
-			}
-
-			return
-
-		} else {
-			// search the tail for partial match
-			// if one is found, load more data and continue the match
-			// if matched, advance buffer in same way as above
-			start := len(p.buf) - len(boundary) + 1
-			if start < 0 {
-				start = 0
-			}
-			subject := p.buf[start:]
-
-			for i := 0; i < len(subject); i++ {
-				if subject[i] == boundary[p.boundaryMatched] {
-					p.boundaryMatched++
-				} else {
-					p.boundaryMatched = 0
-				}
-			}
-			p.skip(len(p.buf))
-			if p.ch == 0 {
-				return false, io.EOF
-			} else if p.boundaryMatched > 0 {
-				// check for a match by joining the match from the end of the last buf
-				// & the beginning of this buf
-				if bytes.Compare(
-					p.buf[0:len(boundary)-p.boundaryMatched],
-					[]byte(boundary[p.boundaryMatched:])) == 0 {
-
-					// advance the pointer
-					p.skip(len(boundary) - p.boundaryMatched)
-
-					p.lastBoundaryPos = p.msgPos - uint(len(boundary)) - 1
-					end, err = p.boundaryEnd()
-					if err != nil {
-						return
-					}
-					p.transportPadding()
-					if p.ch != '\n' && p.ch != 0 {
-						err = errors.New("boundary new line expected")
-					}
-					return
-				}
-				p.boundaryMatched = 0
-			}
-			//_ = subject
-		}
-	}
-}
-
-// is it the end of a boundary?
-func (p *parser) boundaryEnd() (result bool, err error) {
-	if p.ch == '-' && p.peek() == '-' {
-		p.next()
-		p.next()
-		result = true
-	}
-	if p.ch == 0 {
-		err = io.EOF
-	}
-	return
-}
-
-// *LWSP-char
-// = *(WSP / CRLF WSP)
-func (p *parser) transportPadding() {
-	for {
-		if p.ch == ' ' || p.ch == '\t' {
-			p.next()
-		} else if c := p.peek(); p.ch == '\n' && (c == ' ' || c == '\t') {
-			p.next()
-			p.next()
-		} else {
-			return
-		}
-	}
-}
-
-type parserMsg struct {
-	err error
-}
-
-func (p *parser) engine() {
-	var err error
-
-	p.next() // load in some bytes
-	for {
-		err = p.message()
-		p.result <- parserMsg{err}
-		//p.next()
-		p.next()
-		if p.isHalting {
-			return
-		}
-	}
-
-}
-
-func (p *parser) message() error {
-	var err error
-
-	if p.isWSP(p.ch) {
-		err = errors.New("headers cannot start with w-space")
-		return err
-	}
-	mh := NewMimeHeader()
-	if err = p.header(mh); err != nil {
-		return err
-	}
-
-	if p.ch == '\n' && p.next() == '\n' {
-		err = p.body(mh)
-	} else {
-		err = errors.New("body not found")
-	}
-	return err
-}
-
-func (p *parser) header(mh *mimeHeader) (err error) {
-	var state int
-	var name string
-
-	defer func() {
-		fmt.Println(mh.headers)
-		p.accept.Reset()
-		if val := mh.headers.Get("Content-Transfer-Encoding"); val != "" {
-			mh.transferEncoding = val
-		}
-		if val := mh.headers.Get("Content-Disposition"); val != "" {
-			mh.contentDisposition = val
-		}
-
-	}()
-
-	for {
-
-		switch state {
-		case 0:
-			if (p.ch >= 33 && p.ch <= 126) && p.ch != ':' {
-				// capture
-				p.accept.WriteByte(p.ch)
-			} else if p.ch == ':' {
-				state = 1
-			} else if p.ch == ' ' && p.peek() == ':' { // tolerate a SP before the :
-				p.next()
-				state = 1
-			} else {
-				pc := p.peek()
-				err = errors.New("unexpected char:" + string(p.ch) + string(pc))
-				return
-			}
-			if state == 1 {
-				if p.accept.Len() < 2 {
-					err = errors.New("header field too short")
-					return
-				}
-				name = p.accept.String()
-				p.accept.Reset()
-				if c := p.peek(); c == ' ' {
-					// skip the space
-					p.next()
-				}
-				p.next()
-				continue
-			}
-
-		case 1:
-
-			if name == "Content-Type" {
-				var err error
-				contentType, err := p.contentType()
-				if err != nil {
-					return err
-				}
-				mh.contentType = &contentType
-				if val, ok := contentType.parameters["boundary"]; ok {
-					mh.contentBoundary = val
-				}
-				if val, ok := contentType.parameters["charset"]; ok {
-					mh.charset = val
-				}
-				if val, ok := contentType.parameters["name"]; ok {
-					mh.contentName = val
-				}
-				mh.headers.Add("Content-Type", contentType.String())
-				state = 0
-			} else {
-				if (p.ch >= 33 && p.ch <= 126) || p.isWSP(p.ch) {
-					p.accept.WriteByte(p.ch)
-				} else if p.ch == '\n' {
-					c := p.peek()
-
-					if p.isWSP(c) {
-						break // skip \n
-					} else {
-						mh.headers.Add(name, p.accept.String())
-						p.accept.Reset()
-
-						state = 0
-					}
-				} else {
-					err = errors.New("parse error")
-					return
-				}
-			}
-
-		}
-		if p.ch == '\n' && p.peek() == '\n' {
-			return nil
-		}
-		p.next()
-
-		if p.ch == 0 {
-			return io.EOF
-		}
-
-	}
-
-}
-
-func (p *parser) isWSP(b byte) bool {
-	return b == ' ' || b == '\t'
-}
-
-// type "/" subtype
-// *(";" parameter)
-
-// content disposition
-// The Content-Disposition Header Field (rfc2183)
-// https://stackoverflow.com/questions/48347574/do-rfc-standards-require-the-filename-value-for-mime-attachment-to-be-encapsulat
-func (p *parser) contentDisposition() (result contentType, err error) {
-	result = contentType{}
-	return
-}
-
-func (p *parser) contentType() (result contentType, err error) {
-	result = contentType{}
-
-	if result.superType, err = p.mimeType(); err != nil {
-		return
-	}
-	if p.ch != '/' {
-		return result, errors.New("missing subtype")
-	}
-	p.next()
-
-	if result.subType, err = p.mimeSubType(); err != nil {
-		return
-	}
-	if p.ch == ';' {
-		p.next()
-		for {
-			if p.ch == '\n' {
-				c := p.peek()
-				if p.isWSP(c) {
-					p.next() // skip \n (FWS)
-					continue
-				}
-				if c == '\n' { // end of header
-					return
-				}
-			}
-			if p.isWSP(p.ch) { // skip WSP
-				p.next()
-				continue
-			}
-			if p.ch == '(' {
-				if err = p.comment(); err != nil {
-					return
-				}
-				continue
-			}
-
-			if p.ch > 32 && p.ch < 128 && !isTokenSpecial[p.ch] {
-				if key, val, err := p.parameter(); err != nil {
-					return result, err
-				} else {
-					if result.parameters == nil {
-						result.parameters = make(map[string]string, 1)
-					}
-					result.parameters[key] = val
-				}
-			} else {
-				break
-			}
-
-		}
-	}
-
-	return
-}
-
-var isTokenSpecial = [128]bool{
-	'(':  true,
-	')':  true,
-	'<':  true,
-	'>':  true,
-	'@':  true,
-	',':  true,
-	';':  true,
-	':':  true,
-	'\\': true,
-	'"':  true,
-	'/':  true,
-	'[':  true,
-	']':  true,
-	'?':  true,
-	'=':  true,
-}
-
-func (p *parser) mimeType() (str string, err error) {
-
-	defer func() {
-		if p.accept.Len() > 0 {
-			str = p.accept.String()
-			p.accept.Reset()
-		}
-	}()
-	if p.ch < 128 && p.ch > 32 && !isTokenSpecial[p.ch] {
-		for {
-			p.accept.WriteByte(p.ch)
-			p.next()
-			if !(p.ch < 128 && p.ch > 32 && !isTokenSpecial[p.ch]) {
-				return
-			}
-
-		}
-	} else {
-		err = errors.New("unexpected tok")
-		return
-	}
-}
-
-func (p *parser) mimeSubType() (str string, err error) {
-	return p.mimeType()
-}
-
-// comment     =  "(" *(ctext / quoted-pair / comment) ")"
-//
-// ctext       =  <any CHAR excluding "(",     ; => may be folded
-//                     ")", "\" & CR, & including
-//                     linear-white-space>
-//
-// quoted-pair =  "\" CHAR                     ; may quote any char
-func (p *parser) comment() (err error) {
-	// all header fields except for Content-Disposition
-	// can include RFC 822 comments
-	if p.ch != '(' {
-		err = errors.New("unexpected token")
-	}
-
-	for {
-		p.next()
-		if p.ch == ')' {
-			p.next()
-			return
-		}
-	}
-
-}
-
-func (p *parser) token() (str string, err error) {
-	defer func() {
-		if err == nil {
-			str = p.accept.String()
-		}
-		if p.accept.Len() > 0 {
-			p.accept.Reset()
-		}
-	}()
-	var once bool // must match at least 1 good char
-	for {
-		if p.ch > 32 && p.ch < 128 && !isTokenSpecial[p.ch] {
-			p.accept.WriteByte(p.ch)
-			once = true
-		} else if !once {
-			err = errors.New("invalid token")
-			return
-		} else {
-			return
-		}
-		p.next()
-	}
-}
-
-// quoted-string  = ( <"> *(qdtext | quoted-pair ) <"> )
-// quoted-pair    = "\" CHAR
-// CHAR           = <any US-ASCII character (octets 0 - 127)>
-// qdtext         = <any TEXT except <">>
-// TEXT           = <any OCTET except CTLs, but including LWS>
-func (p *parser) quotedString() (str string, err error) {
-	defer func() {
-		if err == nil {
-			str = p.accept.String()
-		}
-		if p.accept.Len() > 0 {
-			p.accept.Reset()
-		}
-	}()
-
-	if p.ch != '"' {
-		err = errors.New("unexpected token")
-		return
-	}
-	p.next()
-	state := 0
-	for {
-		switch state {
-		case 0: // inside quotes
-
-			if p.ch == '"' {
-				p.next()
-				return
-			}
-			if p.ch == '\\' {
-				state = 1
-				break
-			}
-			if (p.ch < 127 && p.ch > 32) || p.isWSP(p.ch) {
-				p.accept.WriteByte(p.ch)
-			} else {
-				err = errors.New("unexpected token")
-				return
-			}
-		case 1:
-			// escaped (<any US-ASCII character (octets 0 - 127)>)
-			if p.ch != 0 && p.ch <= 127 {
-				p.accept.WriteByte(p.ch)
-				state = 0
-			} else {
-				err = errors.New("unexpected token")
-				return
-			}
-		}
-		p.next()
-	}
-}
-
-// parameter := attribute "=" value
-// attribute := token
-// token := 1*<any (US-ASCII) CHAR except SPACE, CTLs, or tspecials>
-// value := token / quoted-string
-// CTL := %x00-1F / %x7F
-// quoted-string : <"> <">
-func (p *parser) parameter() (attribute, value string, err error) {
-	defer func() {
-		p.accept.Reset()
-	}()
-
-	if attribute, err = p.token(); err != nil {
-		return "", "", err
-	}
-	if p.ch != '=' {
-		return "", "", errors.New("expecting =")
-	}
-	p.next()
-	if p.ch == '"' {
-		if value, err = p.quotedString(); err != nil {
-			return
-		}
-		return
-	} else {
-		if value, err = p.token(); err != nil {
-			return
-		}
-		return
-	}
-}
-
-func (p *parser) body(mh *mimeHeader) (err error) {
-	var body bytes.Buffer
-
-	if mh.contentBoundary != "" {
-		if end, err := p.boundary(mh.contentBoundary); err != nil {
-			return err
-		} else {
-			fmt.Println("boundary end:", end)
-		}
-		return
-	} else {
-		for {
-
-			p.next()
-			if p.ch == 0 {
-				return io.EOF
-			}
-			if p.ch == '\n' && p.peek() == '\n' {
-				p.next()
-				return
-			}
-
-			body.WriteByte(p.ch)
-
-		}
-	}
-
-}
-
-func (p *parser) isMulti(part *mimeHeader) bool {
-	if part.contentType != nil {
-		if part.contentType.superType == "multipart" ||
-			part.contentType.superType == "message" {
-			return true
-		}
-	}
-	return false
-}
-
-func (p *parser) multi(part *mimeHeader, depth string) (err error) {
-	if part.contentType != nil {
-		if part.contentType.superType == "multipart" {
-			if end, bErr := p.boundary(part.contentBoundary); bErr != nil {
-				return bErr
-			} else if end {
-
-				part.endingPosBody = p.lastBoundaryPos
-				return
-			}
-		}
-		if part.contentType.superType == "message" ||
-			part.contentType.superType == "multipart" {
-			err = p.mime(part, depth)
-			if err != nil {
-
-				return err
-			}
-
-		}
-
-	}
-	return
-}
-
-func (p *parser) mime(parent *mimeHeader, depth string) (err error) {
-
-	count := 1
-	for {
-
-		var part mimeHeader
-
-		part = *NewMimeHeader()
-		part.startingPos = p.msgPos
-		if p.ch >= 33 && p.ch <= 126 {
-			err = p.header(&part)
-			if err != nil {
-				return err
-			}
-		}
-		if p.ch == '\n' && p.peek() == '\n' {
-			p.next()
-			p.next()
-		}
-		if part.contentBoundary == "" {
-			part.contentBoundary = parent.contentBoundary
-		}
-
-		part.startingPosBody = p.msgPos
-		partID := strconv.Itoa(count)
-		if depth != "" {
-			partID = depth + "." + strconv.Itoa(count)
-		}
-		p.addPart(&part, partID)
-
-		if p.isMulti(&part) {
-
-			err = p.multi(&part, partID)
-			part.endingPosBody = p.lastBoundaryPos
-			if err != nil {
-				break
-			}
-
-		}
-		if end, bErr := p.boundary(parent.contentBoundary); bErr != nil {
-			part.endingPosBody = p.lastBoundaryPos
-			return bErr
-		} else if end {
-			part.endingPosBody = p.lastBoundaryPos
-			return
-		}
-		part.endingPosBody = p.lastBoundaryPos
-		count++
-	}
-	return
-}
-
-func (p *parser) close() error {
-
-	p.msgPos = 0
-	p.msgLine = 0
-	p.gotNewSlice <- false // signal to engine() that there's no more data
-
-	r := <-p.result
-	return r.err
-	//return nil
-}
-
-func (p *parser) parse(buf []byte) error {
-
-	if !p.once {
-		<-p.consumed
-		p.once = true
-	}
-
-	p.set(buf)
-	p.gotNewSlice <- true // unblock
-
-	// make sure that engine() is blocked or stopped before we return
-	select {
-	case <-p.consumed: // wait for it to block on p.gotNewSlice
-		return nil
-	case r := <-p.result:
-
-		return r.err
-	}
-
-}
-
-func newMimeParser() *parser {
-	p := new(parser)
-	p.consumed = make(chan bool)
-	p.gotNewSlice = make(chan bool)
-	p.halt = make(chan bool)
-	p.result = make(chan parserMsg, 1)
-
-	return p
-}
-
-func (p *parser) start() {
-	go p.engine()
-}
-
 func StreamMimeAnalyzer() *StreamDecorator {
 
 	sd := &StreamDecorator{}
@@ -899,45 +39,51 @@ func StreamMimeAnalyzer() *StreamDecorator {
 			var (
 				envelope *mail.Envelope
 				parseErr error
-				parser   *parser
+				parser   *mime.Parser
 			)
 			Svc.AddInitializer(InitializeWith(func(backendConfig BackendConfig) error {
-				parser = newMimeParser()
-				parser.start()
+				parser = mime.NewMimeParser()
 				return nil
 			}))
 
 			Svc.AddShutdowner(ShutdownWith(func() error {
-				//<-parser.end
-				//parser.halt <- true
+				fmt.Println("shutdownewr")
+				_ = parser.Close()
 				return nil
 			}))
 
 			sd.Open = func(e *mail.Envelope) error {
 				envelope = e
-				return nil
+				return parser.Open()
 			}
 
 			sd.Close = func() error {
-				if parseErr != nil {
-					return nil
+				if parts, ok := envelope.Values["MimeParts"].(*[]*mime.MimeHeader); ok {
+					for _, v := range *parts {
+						fmt.Println(v.part + " " + strconv.Itoa(int(v.startingPos)) + " " + strconv.Itoa(int(v.startingPosBody)) + " " + strconv.Itoa(int(v.endingPosBody)))
+					}
 				}
-				err := parser.close()
-				if err != nil {
-					fmt.Println("parse err", err)
+
+				if parseErr == nil {
+					err := parser.Close()
+					return err
+				} else {
+					return parseErr
 				}
-				return nil
 			}
 
 			return StreamProcessWith(func(p []byte) (int, error) {
 				_ = envelope
 				if len(envelope.Header) > 0 {
 
+				}
+				if _, ok := envelope.Values["MimeParts"]; !ok {
+					envelope.Values["MimeParts"] = &parser.Parts
 				}
 				if parseErr == nil {
-					parseErr = parser.parse(p)
+					_, parseErr = parser.Read(p)
 					if parseErr != nil {
-						fmt.Println("parse err", parseErr)
+						Log().WithError(parseErr).Error("mime parse error")
 					}
 				}
 

mail/mime/reader.go

@@ -0,0 +1,865 @@
+package mime
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/textproto"
+	"strconv"
+)
+
+type mimepart struct {
+	/*
+			[starting-pos] => 0
+		    [starting-pos-body] => 270
+		    [ending-pos] => 2651
+		    [ending-pos-body] => 2651
+		    [line-count] => 72
+		    [body-line-count] => 65
+		    [charset] => us-ascii
+		    [transfer-encoding] => 8bit
+		    [content-boundary] => D7F------------D7FD5A0B8AB9C65CCDBFA872
+		    [content-type] => multipart/mixed
+		    [content-base] => /
+
+			[starting-pos] => 2023
+		    [starting-pos-body] => 2172
+		    [ending-pos] => 2561
+		    [ending-pos-body] => 2561
+		    [line-count] => 9
+		    [body-line-count] => 5
+		    [charset] => us-ascii
+		    [transfer-encoding] => base64
+		    [content-name] => map_of_Argentina.gif
+		    [content-type] => image/gif
+		    [disposition-fi1ename] => map_of_Argentina.gif
+		    [content-disposition] => in1ine
+		    [content-base] => /
+	*/
+}
+
+const (
+	maxBoundaryLen = 70 + 10
+	doubleDash     = "--"
+	startPos       = -1
+)
+
+type Parser struct {
+
+	// related to the state of the parser
+
+	buf                   []byte
+	pos                   int
+	ch                    byte
+	gotNewSlice, consumed chan bool
+	accept                bytes.Buffer
+	boundaryMatched       int
+	count                 uint
+	result                chan parserMsg
+
+	// mime variables
+
+	// Parts is the mime parts tree. The parser builds the parts as it consumes the input
+	// In order to represent the tree in an array, we use Parts.part to store the name of
+	// each node. The name of the node is the *path* of the node. The root node is always
+	// "1". The child would be "1.1", the next sibling would be "1.2", while the child of
+	// "1.2" would be "1.2.1"
+	Parts           []*MimeHeader
+	msgPos          uint
+	msgLine         uint
+	lastBoundaryPos uint
+}
+
+type MimeHeader struct {
+	Headers textproto.MIMEHeader
+
+	Part string
+
+	StartingPos      uint // including header (after boundary, 0 at the top)
+	StartingPosBody  uint // after header \n\n
+	EndingPos        uint // redundant (same as endingPos)
+	EndingPosBody    uint // the char before the boundary marker
+	LineCount        uint
+	BodyLineCount    uint
+	Charset          string
+	TransferEncoding string
+	ContentBoundary  string
+	ContentType      *contentType
+	ContentBase      string
+
+	DispositionFi1eName string
+	ContentDisposition  string
+	ContentName         string
+}
+
+type contentType struct {
+	superType  string
+	subType    string
+	parameters map[string]string
+}
+
+type parserMsg struct {
+	err error
+}
+
+var isTokenSpecial = [128]bool{
+	'(':  true,
+	')':  true,
+	'<':  true,
+	'>':  true,
+	'@':  true,
+	',':  true,
+	';':  true,
+	':':  true,
+	'\\': true,
+	'"':  true,
+	'/':  true,
+	'[':  true,
+	']':  true,
+	'?':  true,
+	'=':  true,
+}
+
+func (c *contentType) String() string {
+	return fmt.Sprintf("%s/%s", c.superType, c.subType)
+}
+
+func newMimeHeader() *MimeHeader {
+	mh := new(MimeHeader)
+	mh.Headers = make(textproto.MIMEHeader, 1)
+	return mh
+}
+
+func (p *Parser) addPart(mh *MimeHeader, id string) {
+	mh.Part = id
+	p.Parts = append(p.Parts, mh)
+}
+
+//
+func (p *Parser) more() bool {
+	p.consumed <- true // signal that we've reached the end of available input
+	gotMore := <-p.gotNewSlice
+	return gotMore
+}
+
+func (p *Parser) next() byte {
+	// wait for a new new slice if reached the end
+	if p.pos+1 >= len(p.buf) {
+		if !p.more() {
+			p.ch = 0
+			return 0
+		}
+	}
+
+	// go to the next byte
+	p.pos++
+	p.ch = p.buf[p.pos]
+	p.msgPos++
+	if p.ch == '\n' {
+		p.msgLine++
+	}
+	return p.ch
+}
+
+func (p *Parser) peek() byte {
+
+	// reached the end?
+	if p.pos+1 >= len(p.buf) {
+		if !p.more() {
+			p.ch = 0
+			return 0
+		}
+	}
+
+	// peek the next byte
+	if p.pos+1 < len(p.buf) {
+		return p.buf[p.pos+1]
+	}
+	return 0
+}
+
+// inject is used for testing, to simulate a byte stream
+func (p *Parser) inject(input ...[]byte) {
+	p.msgPos = 0
+	p.set(input[0])
+	p.pos = 0
+	p.ch = p.buf[0]
+	go func() {
+		for i := 1; i < len(input); i++ {
+			<-p.consumed
+			p.set(input[i])
+			p.gotNewSlice <- true
+		}
+		<-p.consumed
+		p.gotNewSlice <- false // no more data
+	}()
+}
+
+func (p *Parser) set(input []byte) {
+	if p.pos != startPos {
+		// rewind
+		p.pos = startPos
+	}
+	p.buf = input
+
+}
+
+func (p *Parser) skip(nBytes int) {
+
+	for i := 0; i < nBytes; i++ {
+		p.next()
+		if p.ch == 0 {
+			return
+		}
+	}
+
+}
+
+// boundary scans until next boundary string, returns error if not found
+// syntax specified https://tools.ietf.org/html/rfc2046 p21
+func (p *Parser) boundary(contentBoundary string) (end bool, err error) {
+	defer func() {
+		if err == nil {
+			if p.ch == '\n' {
+				p.next()
+			}
+		}
+	}()
+
+	if len(contentBoundary) < 1 {
+		err = errors.New("content boundary too short")
+	}
+	boundary := doubleDash + contentBoundary
+	p.boundaryMatched = 0
+	for {
+		if i := bytes.Index(p.buf[p.pos:], []byte(boundary)); i > -1 {
+			// advance the pointer to 1 char before the end of the boundary
+			// then let next() to advance the last char.
+			// in case the boundary is the tail part of buffer, calling next()
+			// will wait until we get a new buffer
+
+			p.skip(i)
+			p.lastBoundaryPos = p.msgPos - 1 // - uint(len(boundary))
+			p.skip(len(boundary))
+			if end, err = p.boundaryEnd(); err != nil {
+				return
+			}
+			if err = p.transportPadding(); err != nil {
+				return
+			}
+			if p.ch != '\n' {
+				err = errors.New("boundary new line expected")
+			}
+			return
+
+		} else {
+			// search the tail for partial match
+			// if one is found, load more data and continue the match
+			// if matched, advance buffer in same way as above
+			start := len(p.buf) - len(boundary) + 1
+			if start < 0 {
+				start = 0
+			}
+			subject := p.buf[start:]
+
+			for i := 0; i < len(subject); i++ {
+				if subject[i] == boundary[p.boundaryMatched] {
+					p.boundaryMatched++
+				} else {
+					p.boundaryMatched = 0
+				}
+			}
+			p.skip(len(p.buf))
+			if p.ch == 0 {
+				return false, io.EOF
+			} else if p.boundaryMatched > 0 {
+				// check for a match by joining the match from the end of the last buf
+				// & the beginning of this buf
+				if bytes.Compare(
+					p.buf[0:len(boundary)-p.boundaryMatched],
+					[]byte(boundary[p.boundaryMatched:])) == 0 {
+
+					// advance the pointer
+					p.skip(len(boundary) - p.boundaryMatched)
+
+					p.lastBoundaryPos = p.msgPos - uint(len(boundary)) - 1
+					end, err = p.boundaryEnd()
+					if err != nil {
+						return
+					}
+					if err = p.transportPadding(); err != nil {
+						return
+					}
+					if p.ch != '\n' {
+						err = errors.New("boundary new line expected")
+					}
+					return
+				}
+				p.boundaryMatched = 0
+			}
+			//_ = subject
+		}
+	}
+}
+
+// is it the end of a boundary?
+func (p *Parser) boundaryEnd() (result bool, err error) {
+	if p.ch == '-' && p.peek() == '-' {
+		p.next()
+		p.next()
+		result = true
+	}
+	if p.ch == 0 {
+		err = io.EOF
+	}
+	return
+}
+
+// *LWSP-char
+// = *(WSP / CRLF WSP)
+func (p *Parser) transportPadding() (err error) {
+	for {
+		if p.ch == ' ' || p.ch == '\t' {
+			p.next()
+		} else if c := p.peek(); p.ch == '\n' && (c == ' ' || c == '\t') {
+			p.next()
+			p.next()
+		} else {
+			if p.ch == 0 {
+				err = io.EOF
+			}
+			return
+		}
+	}
+}
+
+func (p *Parser) header(mh *MimeHeader) (err error) {
+	var state int
+	var name string
+
+	defer func() {
+		fmt.Println(mh.Headers)
+		p.accept.Reset()
+		if val := mh.Headers.Get("Content-Transfer-Encoding"); val != "" {
+			mh.TransferEncoding = val
+		}
+		if val := mh.Headers.Get("Content-Disposition"); val != "" {
+			mh.ContentDisposition = val
+		}
+
+	}()
+
+	for {
+
+		switch state {
+		case 0:
+			if (p.ch >= 33 && p.ch <= 126) && p.ch != ':' {
+				// capture
+				p.accept.WriteByte(p.ch)
+			} else if p.ch == ':' {
+				state = 1
+			} else if p.ch == ' ' && p.peek() == ':' { // tolerate a SP before the :
+				p.next()
+				state = 1
+			} else {
+				pc := p.peek()
+				err = errors.New("unexpected char:" + string(p.ch) + ", peek:" + string(pc))
+				return
+			}
+			if state == 1 {
+				if p.accept.Len() < 2 {
+					err = errors.New("header field too short")
+					return
+				}
+				name = p.accept.String()
+				p.accept.Reset()
+				if c := p.peek(); c == ' ' {
+					// skip the space
+					p.next()
+				}
+				p.next()
+				continue
+			}
+
+		case 1:
+
+			if name == "Content-Type" {
+				var err error
+				contentType, err := p.contentType()
+				if err != nil {
+					return err
+				}
+				mh.ContentType = &contentType
+				if val, ok := contentType.parameters["boundary"]; ok {
+					mh.ContentBoundary = val
+				}
+				if val, ok := contentType.parameters["charset"]; ok {
+					mh.Charset = val
+				}
+				if val, ok := contentType.parameters["name"]; ok {
+					mh.ContentName = val
+				}
+				mh.Headers.Add("Content-Type", contentType.String())
+				state = 0
+			} else {
+				if (p.ch >= 33 && p.ch <= 126) || p.isWSP(p.ch) {
+					p.accept.WriteByte(p.ch)
+				} else if p.ch == '\n' {
+					c := p.peek()
+
+					if p.isWSP(c) {
+						break // skip \n
+					} else {
+						mh.Headers.Add(name, p.accept.String())
+						p.accept.Reset()
+
+						state = 0
+					}
+				} else {
+					err = errors.New("parse error")
+					return
+				}
+			}
+
+		}
+		if p.ch == '\n' && p.peek() == '\n' {
+			return nil
+		}
+		p.next()
+
+		if p.ch == 0 {
+			return io.EOF
+		}
+
+	}
+
+}
+
+func (p *Parser) isWSP(b byte) bool {
+	return b == ' ' || b == '\t'
+}
+
+// type "/" subtype
+// *(";" parameter)
+
+// content disposition
+// The Content-Disposition Header Field (rfc2183)
+// https://stackoverflow.com/questions/48347574/do-rfc-standards-require-the-filename-value-for-mime-attachment-to-be-encapsulat
+func (p *Parser) contentDisposition() (result contentType, err error) {
+	result = contentType{}
+	return
+}
+
+func (p *Parser) contentType() (result contentType, err error) {
+	result = contentType{}
+
+	if result.superType, err = p.mimeType(); err != nil {
+		return
+	}
+	if p.ch != '/' {
+		return result, errors.New("missing subtype")
+	}
+	p.next()
+
+	if result.subType, err = p.mimeSubType(); err != nil {
+		return
+	}
+	if p.ch == ';' {
+		p.next()
+		for {
+			if p.ch == '\n' {
+				c := p.peek()
+				if p.isWSP(c) {
+					p.next() // skip \n (FWS)
+					continue
+				}
+				if c == '\n' { // end of header
+					return
+				}
+			}
+			if p.isWSP(p.ch) { // skip WSP
+				p.next()
+				continue
+			}
+			if p.ch == '(' {
+				if err = p.comment(); err != nil {
+					return
+				}
+				continue
+			}
+
+			if p.ch > 32 && p.ch < 128 && !isTokenSpecial[p.ch] {
+				if key, val, err := p.parameter(); err != nil {
+					return result, err
+				} else {
+					if result.parameters == nil {
+						result.parameters = make(map[string]string, 1)
+					}
+					result.parameters[key] = val
+				}
+			} else {
+				break
+			}
+
+		}
+	}
+
+	return
+}
+
+func (p *Parser) mimeType() (str string, err error) {
+
+	defer func() {
+		if p.accept.Len() > 0 {
+			str = p.accept.String()
+			p.accept.Reset()
+		}
+	}()
+	if p.ch < 128 && p.ch > 32 && !isTokenSpecial[p.ch] {
+		for {
+			p.accept.WriteByte(p.ch)
+			p.next()
+			if !(p.ch < 128 && p.ch > 32 && !isTokenSpecial[p.ch]) {
+				return
+			}
+
+		}
+	} else {
+		err = errors.New("unexpected tok")
+		return
+	}
+}
+
+func (p *Parser) mimeSubType() (str string, err error) {
+	return p.mimeType()
+}
+
+// comment     =  "(" *(ctext / quoted-pair / comment) ")"
+//
+// ctext       =  <any CHAR excluding "(",     ; => may be folded
+//                     ")", "\" & CR, & including
+//                     linear-white-space>
+//
+// quoted-pair =  "\" CHAR                     ; may quote any char
+func (p *Parser) comment() (err error) {
+	// all header fields except for Content-Disposition
+	// can include RFC 822 comments
+	if p.ch != '(' {
+		err = errors.New("unexpected token")
+	}
+
+	for {
+		p.next()
+		if p.ch == ')' {
+			p.next()
+			return
+		}
+	}
+
+}
+
+func (p *Parser) token() (str string, err error) {
+	defer func() {
+		if err == nil {
+			str = p.accept.String()
+		}
+		if p.accept.Len() > 0 {
+			p.accept.Reset()
+		}
+	}()
+	var once bool // must match at least 1 good char
+	for {
+		if p.ch > 32 && p.ch < 128 && !isTokenSpecial[p.ch] {
+			p.accept.WriteByte(p.ch)
+			once = true
+		} else if !once {
+			err = errors.New("invalid token")
+			return
+		} else {
+			return
+		}
+		p.next()
+	}
+}
+
+// quoted-string  = ( <"> *(qdtext | quoted-pair ) <"> )
+// quoted-pair    = "\" CHAR
+// CHAR           = <any US-ASCII character (octets 0 - 127)>
+// qdtext         = <any TEXT except <">>
+// TEXT           = <any OCTET except CTLs, but including LWS>
+func (p *Parser) quotedString() (str string, err error) {
+	defer func() {
+		if err == nil {
+			str = p.accept.String()
+		}
+		if p.accept.Len() > 0 {
+			p.accept.Reset()
+		}
+	}()
+
+	if p.ch != '"' {
+		err = errors.New("unexpected token")
+		return
+	}
+	p.next()
+	state := 0
+	for {
+		switch state {
+		case 0: // inside quotes
+
+			if p.ch == '"' {
+				p.next()
+				return
+			}
+			if p.ch == '\\' {
+				state = 1
+				break
+			}
+			if (p.ch < 127 && p.ch > 32) || p.isWSP(p.ch) {
+				p.accept.WriteByte(p.ch)
+			} else {
+				err = errors.New("unexpected token")
+				return
+			}
+		case 1:
+			// escaped (<any US-ASCII character (octets 0 - 127)>)
+			if p.ch != 0 && p.ch <= 127 {
+				p.accept.WriteByte(p.ch)
+				state = 0
+			} else {
+				err = errors.New("unexpected token")
+				return
+			}
+		}
+		p.next()
+	}
+}
+
+// parameter := attribute "=" value
+// attribute := token
+// token := 1*<any (US-ASCII) CHAR except SPACE, CTLs, or tspecials>
+// value := token / quoted-string
+// CTL := %x00-1F / %x7F
+// quoted-string : <"> <">
+func (p *Parser) parameter() (attribute, value string, err error) {
+	defer func() {
+		p.accept.Reset()
+	}()
+
+	if attribute, err = p.token(); err != nil {
+		return "", "", err
+	}
+	if p.ch != '=' {
+		return "", "", errors.New("expecting =")
+	}
+	p.next()
+	if p.ch == '"' {
+		if value, err = p.quotedString(); err != nil {
+			return
+		}
+		return
+	} else {
+		if value, err = p.token(); err != nil {
+			return
+		}
+		return
+	}
+}
+
+func (p *Parser) body(mh *MimeHeader) (err error) {
+	var body bytes.Buffer
+
+	if mh.ContentBoundary != "" {
+		if end, err := p.boundary(mh.ContentBoundary); err != nil {
+			return err
+		} else {
+			fmt.Println("boundary end:", end)
+		}
+		return
+	} else {
+		for {
+
+			p.next()
+			if p.ch == 0 {
+				return io.EOF
+			}
+			if p.ch == '\n' && p.peek() == '\n' {
+				p.next()
+				return
+			}
+
+			body.WriteByte(p.ch)
+
+		}
+	}
+
+}
+
+// isBranch determines if we should branch this part, when building
+// the mime tree
+func (p *Parser) isBranch(part *MimeHeader, parent *MimeHeader) bool {
+	ct := part.ContentType
+	if parent != nil && ct.superType != "message" {
+		if parent.ContentBoundary == part.ContentBoundary {
+			return false
+		}
+	}
+	if ct != nil {
+		if ct.superType == "multipart" ||
+			ct.superType == "message" {
+			return true
+		}
+	}
+	return false
+}
+
+func (p *Parser) multi(part *MimeHeader, depth string) (err error) {
+	if part.ContentType != nil {
+		if part.ContentType.superType == "multipart" {
+			if end, bErr := p.boundary(part.ContentBoundary); bErr != nil {
+				return bErr
+			} else if end {
+
+				part.EndingPosBody = p.lastBoundaryPos
+				return
+			}
+		}
+		if part.ContentType.superType == "message" ||
+			part.ContentType.superType == "multipart" {
+			err = p.mime(part, depth)
+			if err != nil {
+
+				return err
+			}
+
+		}
+
+	}
+	return
+}
+
+func (p *Parser) mime(parent *MimeHeader, depth string) (err error) {
+
+	count := 1
+	for {
+
+		var part MimeHeader
+
+		part = *newMimeHeader()
+		part.StartingPos = p.msgPos
+		if p.ch >= 33 && p.ch <= 126 {
+			err = p.header(&part)
+			if err != nil {
+				return err
+			}
+		} else {
+			return errors.New("parse error")
+		}
+		if p.ch == '\n' && p.peek() == '\n' {
+			p.next()
+			p.next()
+		}
+		if part.ContentBoundary == "" {
+			part.ContentBoundary = parent.ContentBoundary
+		}
+
+		part.StartingPosBody = p.msgPos
+		partID := strconv.Itoa(count)
+		if depth != "" {
+			partID = depth + "." + strconv.Itoa(count)
+		}
+		p.addPart(&part, partID)
+
+		if p.isBranch(&part, parent) {
+
+			err = p.multi(&part, partID)
+			part.EndingPosBody = p.lastBoundaryPos
+			if err != nil {
+				break
+			}
+
+		}
+		if end, bErr := p.boundary(parent.ContentBoundary); bErr != nil {
+			part.EndingPosBody = p.lastBoundaryPos
+			return bErr
+		} else if end {
+			part.EndingPosBody = p.lastBoundaryPos
+			return
+		}
+		part.EndingPosBody = p.lastBoundaryPos
+		count++
+	}
+	return
+}
+
+// Close tells the MIME Parser there's no more data & waits for it to return a result
+// it will return an io.EOF error if no error with parsing MIME was detected
+// Close is not concurrency safe, must be called synchronously after all calls to
+// Read have completed
+func (p *Parser) Close() error {
+	if p.count > 0 {
+		for {
+			// dont't exit unless we get the result.
+			select {
+			case <-p.consumed: // wait for p.buf to be consumed
+				p.gotNewSlice <- false // tell next() that there's no more data
+			case r := <-p.result:
+				// mime() has returned with a result
+				p.count = 0
+				return r.err
+			}
+		}
+	}
+	return nil
+
+}
+
+func (p *Parser) Open() error {
+	p.lastBoundaryPos = 0
+	p.pos = startPos
+	p.msgPos = 0
+	p.msgLine = 0
+	return nil
+
+}
+
+// Read takes a byte stream, and feeds it to the MIME Parser
+// waits for the Parser to consume all input before returning.
+// The Parser will build
+// returns error if there's a parse error
+
+func (p *Parser) Read(buf []byte) (int, error) {
+	defer func() {
+		p.count++
+	}()
+	if p.count == 0 {
+		p.set(buf)
+		p.next()
+		go func() {
+			err := p.mime(nil, "")
+			fmt.Println("mine() ret", err)
+			p.result <- parserMsg{err}
+		}()
+		return p.pos + 1, nil
+
+	}
+	select {
+	case <-p.consumed: // wait for prev buf to be consumed
+		p.set(buf)
+		p.gotNewSlice <- true
+		return p.pos + 1, nil
+	case r := <-p.result:
+		// mime() has returned with a result
+		return p.pos + 1, r.err
+	}
+
+}
+
+func NewMimeParser() *Parser {
+	p := new(Parser)
+	p.consumed = make(chan bool)
+	p.gotNewSlice = make(chan bool)
+	p.result = make(chan parserMsg, 1)
+	return p
+}

backends/s_mime_test.go → mail/mime/reader_test.go

@@ -1,4 +1,4 @@
-package backends
+package mime
 
 import (
 	"bytes"
@@ -6,12 +6,13 @@ import (
 	"io"
 	"strconv"
 	"testing"
+	"time"
 )
 
-var p *parser
+var p *Parser
 
 func init() {
-	p = newMimeParser()
+	p = NewMimeParser()
 }
 func TestInject(t *testing.T) {
 	var b bytes.Buffer
@@ -95,18 +96,18 @@ Al
 This
 `
 	p.inject([]byte(in))
-	h := NewMimeHeader()
+	h := newMimeHeader()
 	err := p.header(h)
 	if err != nil {
 		t.Error(err)
 	}
-	if _, err := p.boundary(h.contentBoundary); err != nil {
+	if _, err := p.boundary(h.ContentBoundary); err != nil {
 		t.Error(err)
 	} else {
 		//_ = part
 		//p.addPart(part)
 
-		//nextPart := NewMimeHeader()
+		//nextPart := newMimeHeader()
 		//err = p.body(part)
 		//if err != nil {
 		//	t.Error(err)
@@ -116,14 +117,14 @@ This
 
 func TestBoundary(t *testing.T) {
 	var err error
-	part := NewMimeHeader()
-	part.contentBoundary = "-wololo-"
+	part := newMimeHeader()
+	part.ContentBoundary = "-wololo-"
 
 	// in the middle of the string
 	test := "The quick brown fo---wololo-\nx jumped over the lazy dog"
 	p.inject([]byte(test))
 
-	_, err = p.boundary(part.contentBoundary)
+	_, err = p.boundary(part.ContentBoundary)
 	if err != nil {
 		t.Error(err)
 	}
@@ -131,7 +132,7 @@ func TestBoundary(t *testing.T) {
 
 	// at the end (with the -- postfix)
 	p.inject([]byte("The quick brown fox jumped over the lazy dog---wololo---\n"))
-	_, err = p.boundary(part.contentBoundary)
+	_, err = p.boundary(part.ContentBoundary)
 	if err != nil {
 		t.Error(err)
 	}
@@ -143,7 +144,7 @@ func TestBoundary(t *testing.T) {
 	p.inject(
 		[]byte("The quick brown fox jumped ov---wolo"),
 		[]byte("lo---\ner the lazy dog"))
-	_, err = p.boundary(part.contentBoundary)
+	_, err = p.boundary(part.ContentBoundary)
 	if err != nil {
 		t.Error(err)
 	}
@@ -157,7 +158,7 @@ func TestBoundary(t *testing.T) {
 		[]byte("this is the middle"),
 		[]byte("and thats the end---wololo---\n"))
 
-	_, err = p.boundary(part.contentBoundary)
+	_, err = p.boundary(part.ContentBoundary)
 	if err != nil {
 		t.Error(err)
 	}
@@ -362,25 +363,78 @@ Content-Disposition: attachment;
 ------_=_NextPart_003_01CBE272.13692C80--
 ------_=_NextPart_001_01CBE273.65A0E7AA--`
 
+var email3 = `MIME-Version: 1.0
+X-Mailer: MailBee.NET 8.0.4.428
+Subject: test subject
+To: [email protected]
+Content-Type: multipart/mixed;
+       boundary="XXXXboundary text"
+
+--XXXXboundary text
+Content-Type: multipart/alternative;
+       boundary="XXXXboundary text"
+
+--XXXXboundary text
+Content-Type: text/plain;
+       charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+
+This is the body text of a sample message.
+--XXXXboundary text
+Content-Type: text/html;
+       charset="utf-8"
+Content-Transfer-Encoding: quoted-printable
+
+<pre>This is the body text of a sample message.</pre>
+
+--XXXXboundary text
+Content-Type: text/plain;
+       name="log_attachment.txt"
+Content-Disposition: attachment;
+       filename="log_attachment.txt"
+Content-Transfer-Encoding: base64
+
+TUlNRS1WZXJzaW9uOiAxLjANClgtTWFpbGVyOiBNYWlsQmVlLk5FVCA4LjAuNC40MjgNClN1Ympl
+Y3Q6IHRlc3Qgc3ViamVjdA0KVG86IGtldmlubUBkYXRhbW90aW9uLmNvbQ0KQ29udGVudC1UeXBl
+OiBtdWx0aXBhcnQvYWx0ZXJuYXRpdmU7DQoJYm91bmRhcnk9Ii0tLS09X05leHRQYXJ0XzAwMF9B
+RTZCXzcyNUUwOUFGLjg4QjdGOTM0Ig0KDQoNCi0tLS0tLT1fTmV4dFBhcnRfMDAwX0FFNkJfNzI1
+RTA5QUYuODhCN0Y5MzQNCkNvbnRlbnQtVHlwZTogdGV4dC9wbGFpbjsNCgljaGFyc2V0PSJ1dGYt
+OCINCkNvbnRlbnQtVHJhbnNmZXItRW5jb2Rpbmc6IHF1b3RlZC1wcmludGFibGUNCg0KdGVzdCBi
+b2R5DQotLS0tLS09X05leHRQYXJ0XzAwMF9BRTZCXzcyNUUwOUFGLjg4QjdGOTM0DQpDb250ZW50
+LVR5cGU6IHRleHQvaHRtbDsNCgljaGFyc2V0PSJ1dGYtOCINCkNvbnRlbnQtVHJhbnNmZXItRW5j
+b2Rpbmc6IHF1b3RlZC1wcmludGFibGUNCg0KPHByZT50ZXN0IGJvZHk8L3ByZT4NCi0tLS0tLT1f
+TmV4dFBhcnRfMDAwX0FFNkJfNzI1RTA5QUYuODhCN0Y5MzQtLQ0K
+--XXXXboundary text--
+`
+
 func TestNestedEmail(t *testing.T) {
-	email = email2
+	email = email
 	p.inject([]byte(email))
 
+	go func() {
+		time.Sleep(time.Second * 15)
+		//panic("here")
+		//		*moo = *moo + 6
+
+		//pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+		//os.Exit(1)
+	}()
+
 	if err := p.mime(nil, ""); err != nil && err != io.EOF {
 		t.Error(err)
 	}
-	for part := range p.parts {
-		email = replaceAtIndex(email, '#', p.parts[part].startingPos)
-		email = replaceAtIndex(email, '&', p.parts[part].startingPosBody)
-		email = replaceAtIndex(email, '*', p.parts[part].endingPosBody)
-		fmt.Println(p.parts[part].part + " " + strconv.Itoa(int(p.parts[part].startingPos)) + " " + strconv.Itoa(int(p.parts[part].startingPosBody)) + " " + strconv.Itoa(int(p.parts[part].endingPosBody)))
+	for part := range p.Parts {
+		email = replaceAtIndex(email, '#', p.Parts[part].StartingPos)
+		email = replaceAtIndex(email, '&', p.Parts[part].StartingPosBody)
+		email = replaceAtIndex(email, '*', p.Parts[part].EndingPosBody)
+		fmt.Println(p.Parts[part].Part + " " + strconv.Itoa(int(p.Parts[part].StartingPos)) + " " + strconv.Itoa(int(p.Parts[part].StartingPosBody)) + " " + strconv.Itoa(int(p.Parts[part].EndingPosBody)))
 	}
 	fmt.Print(email)
 	//fmt.Println(strings.Index(email, "--D7F------------D7FD5A0B8AB9C65CCDBFA872--"))
 
 	//fmt.Println(email[p.parts[1].startingPosBody:p.parts[1].endingPosBody])
-	i := 2
-	fmt.Println("**********{" + email[p.parts[i].startingPosBody:p.parts[i].endingPosBody] + "}**********")
+	//i := 2
+	//fmt.Println("**********{" + email[p.parts[i].startingPosBody:p.parts[i].endingPosBody] + "}**********")
 }
 
 func replaceAtIndex(str string, replacement rune, index uint) string {