New dotreader (#43)

* fixes #42  
* Uses textproto package to read dot-stuffed data in DATA state
* envelope.Data is now a buffer
* backend to use data (compress, etc)
* Moved subject parsing to envelope package & using textproto for reading headers


Other changes:

* guerrilla redis & db backend: user a sync.Pool for buffers used for compressing
* adds shutdown state to backend + Reinitialize to backend proxy

backends/abstract.go

@@ -57,8 +57,12 @@ func (b *AbstractBackend) Process(mail *envelope.Envelope) BackendResult {
 	if b.extend != nil {
 		return b.extend.Process(mail)
 	}
+	mail.ParseHeaders()
+
 	if b.config.LogReceivedMails {
 		log.Infof("Mail from: %s / to: %v", mail.MailFrom.String(), mail.RcptTo)
+		log.Info("Headers are: %s", mail.Header)
+
 	}
 	return NewBackendResult("250 OK")
 }

backends/backend.go

@@ -52,10 +52,6 @@ type savePayload struct {
 	recipient   *envelope.EmailAddress
 	savedNotify chan *saveStatus
 }
-type helper struct {
-	saveMailChan chan *savePayload
-	wg           sync.WaitGroup
-}
 
 // BackendResult represents a response to an SMTP client after receiving DATA.
 // The String method should return an SMTP message ready to send back to the
@@ -103,13 +99,15 @@ type BackendGateway struct {
 	b  Backend
 	// controls access to state
 	stateGuard sync.Mutex
-	state      int
+	State      int
+	config     BackendConfig
 }
 
 // possible values for state
 const (
-	BackendStateProcessing = iota
-	BackendStateShutdown
+	BackendStateRunning = iota
+	BackendStateShuttered
+	BackendStateError
 )
 
 // New retrieve a backend specified by the backendName, and initialize it using
@@ -119,17 +117,21 @@ func New(backendName string, backendConfig BackendConfig) (Backend, error) {
 	if !found {
 		return nil, fmt.Errorf("backend %q not found", backendName)
 	}
-	p := &BackendGateway{b: backend}
-	err := p.Initialize(backendConfig)
+	gateway := &BackendGateway{b: backend, config: backendConfig}
+	err := gateway.Initialize(backendConfig)
 	if err != nil {
 		return nil, fmt.Errorf("error while initializing the backend: %s", err)
 	}
-	p.state = BackendStateProcessing
-	return p, nil
+	gateway.State = BackendStateRunning
+	return gateway, nil
 }
 
 // Distributes an envelope to one of the backend workers
 func (gw *BackendGateway) Process(e *envelope.Envelope) BackendResult {
+	if gw.State != BackendStateRunning {
+		return NewBackendResult("554 Transaction failed - backend not running" + strconv.Itoa(gw.State))
+	}
+
 	to := e.RcptTo
 	from := e.MailFrom
 
@@ -153,26 +155,42 @@ func (gw *BackendGateway) Process(e *envelope.Envelope) BackendResult {
 func (gw *BackendGateway) Shutdown() error {
 	gw.stateGuard.Lock()
 	defer gw.stateGuard.Unlock()
-	if gw.state != BackendStateShutdown {
+	if gw.State != BackendStateShuttered {
 		err := gw.b.Shutdown()
 		if err == nil {
 			close(gw.saveMailChan) // workers will stop
 			gw.wg.Wait()
-			gw.state = BackendStateShutdown
+			gw.State = BackendStateShuttered
 		}
 		return err
 	}
 	return nil
 }
 
+// Reinitialize starts up a backend gateway that was shutdown before
+func (gw *BackendGateway) Reinitialize() error {
+	if gw.State != BackendStateShuttered {
+		return errors.New("backend must be in BackendStateshuttered state to Reinitialize")
+	}
+	err := gw.Initialize(gw.config)
+	if err != nil {
+		return fmt.Errorf("error while initializing the backend: %s", err)
+	} else {
+		gw.State = BackendStateRunning
+	}
+	return err
+}
+
 func (gw *BackendGateway) Initialize(cfg BackendConfig) error {
 	err := gw.b.Initialize(cfg)
 	if err == nil {
 		workersSize := gw.b.getNumberOfWorkers()
 		if workersSize < 1 {
+			gw.State = BackendStateError
 			return errors.New("Must have at least 1 worker")
 		}
 		if err := gw.b.testSettings(); err != nil {
+			gw.State = BackendStateError
 			return err
 		}
 		gw.saveMailChan = make(chan *savePayload, workersSize)
@@ -184,6 +202,8 @@ func (gw *BackendGateway) Initialize(cfg BackendConfig) error {
 				gw.wg.Done()
 			}()
 		}
+	} else {
+		gw.State = BackendStateError
 	}
 	return err
 }

backends/dummy.go

@@ -18,8 +18,6 @@ type DummyBackend struct {
 	config dummyConfig
 	// embed functions form AbstractBackend so that DummyBackend satisfies the Backend interface
 	AbstractBackend
-	// helps with worker management
-	helper
 }
 
 // Backends should implement this method and set b.config field with a custom config struct

backends/guerrilla_db_redis.go

@@ -9,9 +9,13 @@ import (
 	log "github.com/Sirupsen/logrus"
 	"github.com/garyburd/redigo/redis"
 
+	"bytes"
+	"compress/zlib"
 	"github.com/flashmob/go-guerrilla/envelope"
 	"github.com/ziutek/mymysql/autorc"
 	_ "github.com/ziutek/mymysql/godrv"
+	"io"
+	"sync"
 )
 
 func init() {
@@ -73,6 +77,60 @@ type redisClient struct {
 	time        int
 }
 
+// compressedData struct will be compressed using zlib when printed via fmt
+type compressedData struct {
+	extraHeaders []byte
+	data         *bytes.Buffer
+	pool         sync.Pool
+}
+
+// newCompressedData returns a new CompressedData
+func newCompressedData() *compressedData {
+	var p = sync.Pool{
+		New: func() interface{} {
+			var b bytes.Buffer
+			return &b
+		},
+	}
+	return &compressedData{
+		pool: p,
+	}
+}
+
+// Set the extraheaders and buffer of data to compress
+func (c *compressedData) set(b []byte, d *bytes.Buffer) {
+	c.extraHeaders = b
+	c.data = d
+}
+
+// implement Stringer interface
+func (c *compressedData) String() string {
+	if c.data == nil {
+		return ""
+	}
+	//borrow a buffer form the pool
+	b := c.pool.Get().(*bytes.Buffer)
+	// put back in the pool
+	defer func() {
+		b.Reset()
+		c.pool.Put(b)
+	}()
+
+	var r *bytes.Reader
+	w, _ := zlib.NewWriterLevel(b, zlib.BestSpeed)
+	r = bytes.NewReader(c.extraHeaders)
+	io.Copy(w, r)
+	io.Copy(w, c.data)
+	w.Close()
+	return b.String()
+}
+
+// clear it, without clearing the pool
+func (c *compressedData) clear() {
+	c.extraHeaders = []byte{}
+	c.data = nil
+}
+
 func (g *GuerrillaDBAndRedisBackend) saveMailWorker(saveMailChan chan *savePayload) {
 	var to, body string
 	var err error
@@ -114,6 +172,7 @@ func (g *GuerrillaDBAndRedisBackend) saveMailWorker(saveMailChan chan *savePaylo
 		}
 	}()
 
+	data := newCompressedData()
 	//  receives values from the channel repeatedly until it is closed.
 	for {
 		payload := <-saveMailChan
@@ -122,9 +181,10 @@ func (g *GuerrillaDBAndRedisBackend) saveMailWorker(saveMailChan chan *savePaylo
 			return
 		}
 		to = payload.recipient.User + "@" + g.config.PrimaryHost
-		length = len(payload.mail.Data)
+		length = payload.mail.Data.Len()
+
 		ts := fmt.Sprintf("%d", time.Now().UnixNano())
-		payload.mail.Subject = MimeHeaderDecode(payload.mail.Subject)
+		payload.mail.ParseHeaders()
 		hash := MD5Hex(
 			to,
 			payload.mail.MailFrom.String(),
@@ -136,26 +196,35 @@ func (g *GuerrillaDBAndRedisBackend) saveMailWorker(saveMailChan chan *savePaylo
 		addHead += "Received: from " + payload.mail.Helo + " (" + payload.mail.Helo + "  [" + payload.mail.RemoteAddress + "])\r\n"
 		addHead += "	by " + payload.recipient.Host + " with SMTP id " + hash + "@" + payload.recipient.Host + ";\r\n"
 		addHead += "	" + time.Now().Format(time.RFC1123Z) + "\r\n"
-		// compress to save space
-		payload.mail.Data = Compress(addHead, payload.mail.Data)
+
+		// data will be compressed when printed, with addHead added to beginning
+
+		data.set([]byte(addHead), &payload.mail.Data)
 		body = "gzencode"
+
+		// data will be written to redis - it implements the Stringer interface, redigo uses fmt to
+		// print the data to redis.
+
 		redisErr = redisClient.redisConnection(g.config.RedisInterface)
 		if redisErr == nil {
-			_, doErr := redisClient.conn.Do("SETEX", hash, g.config.RedisExpireSeconds, payload.mail.Data)
+			_, doErr := redisClient.conn.Do("SETEX", hash, g.config.RedisExpireSeconds, data)
 			if doErr == nil {
-				payload.mail.Data = ""
-				body = "redis"
+				//payload.mail.Data = ""
+				//payload.mail.Data.Reset()
+				body = "redis" // the backend system will know to look in redis for the message data
+				data.clear()   // blank
 			}
 		} else {
 			log.WithError(redisErr).Warn("Error while SETEX on redis")
 		}
+
 		// bind data to cursor
 		ins.Bind(
 			to,
 			payload.mail.MailFrom.String(),
 			payload.mail.Subject,
 			body,
-			payload.mail.Data,
+			data.String(),
 			hash,
 			to,
 			payload.mail.RemoteAddress,

backends/guerrilla_db_redis_test.go

@@ -0,0 +1,33 @@
+package backends
+
+import (
+	"bytes"
+	"compress/zlib"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+)
+
+func TestCompressedData(t *testing.T) {
+	var b bytes.Buffer
+	var out bytes.Buffer
+	str := "Hello Hello Hello Hello Hello Hello Hello!"
+	sbj := "Subject:hello\r\n"
+	b.WriteString(str)
+	cd := newCompressedData()
+	cd.set([]byte(sbj), &b)
+
+	// compress
+	fmt.Fprint(&out, cd)
+
+	// decompress
+	var result bytes.Buffer
+	zReader, _ := zlib.NewReader(bytes.NewReader(out.Bytes()))
+	io.Copy(&result, zReader)
+	expect := sbj + str
+	if delta := strings.Compare(expect, result.String()); delta != 0 {
+		t.Error(delta, "compression did match, expected", expect, "but got", result.String())
+	}
+
+}

backends/util.go

@@ -4,16 +4,11 @@ import (
 	"bytes"
 	"compress/zlib"
 	"crypto/md5"
-	"encoding/base64"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/textproto"
 	"regexp"
 	"strings"
-
-	"github.com/sloonz/go-qprintable"
-	"gopkg.in/iconv.v1"
 )
 
 // First capturing group is header name, second is header value.
@@ -40,110 +35,6 @@ func ParseHeaders(mailData string) map[string]string {
 	return headers
 }
 
-var mimeRegex, _ = regexp.Compile(`=\?(.+?)\?([QBqp])\?(.+?)\?=`)
-
-// Decode strings in Mime header format
-// eg. =?ISO-2022-JP?B?GyRCIVo9dztSOWJAOCVBJWMbKEI=?=
-func MimeHeaderDecode(str string) string {
-
-	matched := mimeRegex.FindAllStringSubmatch(str, -1)
-	var charset, encoding, payload string
-	if matched != nil {
-		for i := 0; i < len(matched); i++ {
-			if len(matched[i]) > 2 {
-				charset = matched[i][1]
-				encoding = strings.ToUpper(matched[i][2])
-				payload = matched[i][3]
-				switch encoding {
-				case "B":
-					str = strings.Replace(
-						str,
-						matched[i][0],
-						MailTransportDecode(payload, "base64", charset),
-						1)
-				case "Q":
-					str = strings.Replace(
-						str,
-						matched[i][0],
-						MailTransportDecode(payload, "quoted-printable", charset),
-						1)
-				}
-			}
-		}
-	}
-	return str
-}
-
-// decode from 7bit to 8bit UTF-8
-// encodingType can be "base64" or "quoted-printable"
-func MailTransportDecode(str string, encodingType string, charset string) string {
-	if charset == "" {
-		charset = "UTF-8"
-	} else {
-		charset = strings.ToUpper(charset)
-	}
-	if encodingType == "base64" {
-		str = fromBase64(str)
-	} else if encodingType == "quoted-printable" {
-		str = fromQuotedP(str)
-	}
-
-	if charset != "UTF-8" {
-		charset = fixCharset(charset)
-		// TODO: remove dependency to os-dependent iconv library
-		if cd, err := iconv.Open("UTF-8", charset); err == nil {
-			defer func() {
-				cd.Close()
-				if r := recover(); r != nil {
-					//logln(1, fmt.Sprintf("Recovered in %v", r))
-				}
-			}()
-			// eg. charset can be "ISO-2022-JP"
-			return cd.ConvString(str)
-		}
-
-	}
-	return str
-}
-
-func fromBase64(data string) string {
-	buf := bytes.NewBufferString(data)
-	decoder := base64.NewDecoder(base64.StdEncoding, buf)
-	res, _ := ioutil.ReadAll(decoder)
-	return string(res)
-}
-
-func fromQuotedP(data string) string {
-	buf := bytes.NewBufferString(data)
-	decoder := qprintable.NewDecoder(qprintable.BinaryEncoding, buf)
-	res, _ := ioutil.ReadAll(decoder)
-	return string(res)
-}
-
-var charsetRegex, _ = regexp.Compile(`[_:.\/\\]`)
-
-func fixCharset(charset string) string {
-	fixed_charset := charsetRegex.ReplaceAllString(charset, "-")
-	// Fix charset
-	// borrowed from http://squirrelmail.svn.sourceforge.net/viewvc/squirrelmail/trunk/squirrelmail/include/languages.php?revision=13765&view=markup
-	// OE ks_c_5601_1987 > cp949
-	fixed_charset = strings.Replace(fixed_charset, "ks-c-5601-1987", "cp949", -1)
-	// Moz x-euc-tw > euc-tw
-	fixed_charset = strings.Replace(fixed_charset, "x-euc", "euc", -1)
-	// Moz x-windows-949 > cp949
-	fixed_charset = strings.Replace(fixed_charset, "x-windows_", "cp", -1)
-	// windows-125x and cp125x charsets
-	fixed_charset = strings.Replace(fixed_charset, "windows-", "cp", -1)
-	// ibm > cp
-	fixed_charset = strings.Replace(fixed_charset, "ibm", "cp", -1)
-	// iso-8859-8-i -> iso-8859-8
-	fixed_charset = strings.Replace(fixed_charset, "iso-8859-8-i", "iso-8859-8", -1)
-	if charset != fixed_charset {
-		return fixed_charset
-	}
-	return charset
-}
-
 // returns an md5 hash as string of hex characters
 func MD5Hex(stringArguments ...string) string {
 	h := md5.New()

client.go

@@ -6,7 +6,7 @@ import (
 	log "github.com/Sirupsen/logrus"
 	"github.com/flashmob/go-guerrilla/envelope"
 	"net"
-	"strings"
+	"net/textproto"
 	"sync"
 	"time"
 )
@@ -37,16 +37,19 @@ type client struct {
 	state        ClientState
 	messagesSent int
 	// Response to be written to the client
-	response string
-	conn     net.Conn
-	bufin    *smtpBufferedReader
-	bufout   *bufio.Writer
+	response   string
+	conn       net.Conn
+	bufin      *smtpBufferedReader
+	bufout     *bufio.Writer
+	smtpReader *textproto.Reader
+	ar         *adjustableLimitedReader
 	// guards access to conn
 	connGuard sync.Mutex
 }
 
+// Allocate a new client
 func NewClient(conn net.Conn, clientID uint64) *client {
-	return &client{
+	c := &client{
 		conn: conn,
 		Envelope: &envelope.Envelope{
 			RemoteAddress: conn.RemoteAddr().String(),
@@ -56,19 +59,32 @@ func NewClient(conn net.Conn, clientID uint64) *client {
 		bufout:      bufio.NewWriter(conn),
 		ID:          clientID,
 	}
+	// used for reading the DATA state
+	c.smtpReader = textproto.NewReader(c.bufin.Reader)
+	return c
 }
 
+// Add a response to be written on the next turn
 func (c *client) responseAdd(r string) {
 	c.response = c.response + r + "\r\n"
 }
 
+// resetTransaction resets the SMTP transaction, ready for the next email (doesn't disconnect)
+// Transaction ends on:
+// -HELO/EHLO/REST command
+// -End of DATA command
+// TLS handhsake
 func (c *client) resetTransaction() {
 	c.MailFrom = &envelope.EmailAddress{}
 	c.RcptTo = []envelope.EmailAddress{}
-	c.Data = ""
+	c.Data.Reset()
 	c.Subject = ""
+	c.Header = nil
 }
 
+// isInTransaction returns true if the connection is inside a transaction.
+// A transaction starts after a MAIL command gets issued by the client.
+// Call resetTransaction to end the transaction
 func (c *client) isInTransaction() bool {
 	isMailFromEmpty := (c.MailFrom == nil || *c.MailFrom == (envelope.EmailAddress{}))
 	if isMailFromEmpty {
@@ -77,31 +93,16 @@ func (c *client) isInTransaction() bool {
 	return true
 }
 
+// kill flags the connection to close on the next turn
 func (c *client) kill() {
 	c.KilledAt = time.Now()
 }
 
+// isAlive returns true if the client is to close on the next turn
 func (c *client) isAlive() bool {
 	return c.KilledAt.IsZero()
 }
 
-func (c *client) scanSubject(reply string) {
-	if c.Subject == "" && (len(reply) > 8) {
-		test := strings.ToUpper(reply[0:9])
-		if i := strings.Index(test, "SUBJECT: "); i == 0 {
-			// first line with \r\n
-			c.Subject = reply[9:]
-		}
-	} else if strings.HasSuffix(c.Subject, "\r\n") {
-		// chop off the \r\n
-		c.Subject = c.Subject[0 : len(c.Subject)-2]
-		if (strings.HasPrefix(reply, " ")) || (strings.HasPrefix(reply, "\t")) {
-			// subject is multi-line
-			c.Subject = c.Subject + reply[1:]
-		}
-	}
-}
-
 // setTimeout adjust the timeout on the connection, goroutine safe
 func (c *client) setTimeout(t time.Duration) {
 	defer c.connGuard.Unlock()
@@ -119,11 +120,14 @@ func (c *client) closeConn() {
 	c.conn = nil
 }
 
+// init is called after the client is borrowed from the pool, to get it ready for the connection
 func (c *client) init(conn net.Conn, clientID uint64) {
 	c.conn = conn
 	// reset our reader & writer
 	c.bufout.Reset(conn)
 	c.bufin.Reset(conn)
+	// reset the data buffer, keep it allocated
+	c.Data.Reset()
 	// reset session data
 	c.state = 0
 	c.KilledAt = time.Time{}
@@ -133,8 +137,10 @@ func (c *client) init(conn net.Conn, clientID uint64) {
 	c.errors = 0
 	c.response = ""
 	c.Helo = ""
+	c.Header = nil
 }
 
+// getId returns the client's unique ID
 func (c *client) getID() uint64 {
 	return c.ID
 }

envelope/envelope.go

@@ -1,6 +1,18 @@
 package envelope
 
-import "fmt"
+import (
+	"bufio"
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"github.com/sloonz/go-qprintable"
+	"gopkg.in/iconv.v1"
+	"io/ioutil"
+	"net/textproto"
+	"regexp"
+	"strings"
+)
 
 // EmailAddress encodes an email address of the form `<user@host>`
 type EmailAddress struct {
@@ -25,8 +37,150 @@ type Envelope struct {
 	// Sender
 	MailFrom *EmailAddress
 	// Recipients
-	RcptTo  []EmailAddress
-	Data    string
+	RcptTo []EmailAddress
+	// Data stores the header and message body
+	Data bytes.Buffer
+	// Subject stores the subject of the email, extracted and decoded after calling ParseHeaders()
 	Subject string
-	TLS     bool
+	// TLS is true if the email was received using a TLS connection
+	TLS bool
+	// Header stores the results from ParseHeaders()
+	Header textproto.MIMEHeader
+}
+
+// ParseHeaders parses the headers into Header field of the Envelope struct.
+// Data buffer must be full before calling.
+// It assumes that at most 30kb of email data can be a header
+// Decoding of encoding to UTF is only done on the Subject, where the result is assigned to the Subject field
+func (e *Envelope) ParseHeaders() error {
+	var err error
+	if e.Header != nil {
+		return errors.New("Headers already parsed")
+	}
+	all := e.Data.Bytes()
+
+	// find where the header ends, assuming that over 30 kb would be max
+	max := 1024 * 30
+	if len(all) < max {
+		max = len(all) - 1
+	}
+	headerEnd := bytes.Index(all[:max], []byte("\n\n"))
+
+	if headerEnd > -1 {
+		headerReader := textproto.NewReader(bufio.NewReader(bytes.NewBuffer(all[0:headerEnd])))
+		e.Header, err = headerReader.ReadMIMEHeader()
+		if err != nil {
+			// decode the subject
+			if subject, ok := e.Header["Subject"]; ok {
+				e.Subject = MimeHeaderDecode(subject[0])
+			}
+		}
+	} else {
+		err = errors.New("header not found")
+	}
+	return err
+}
+
+var mimeRegex, _ = regexp.Compile(`=\?(.+?)\?([QBqp])\?(.+?)\?=`)
+
+// Decode strings in Mime header format
+// eg. =?ISO-2022-JP?B?GyRCIVo9dztSOWJAOCVBJWMbKEI=?=
+func MimeHeaderDecode(str string) string {
+
+	matched := mimeRegex.FindAllStringSubmatch(str, -1)
+	var charset, encoding, payload string
+	if matched != nil {
+		for i := 0; i < len(matched); i++ {
+			if len(matched[i]) > 2 {
+				charset = matched[i][1]
+				encoding = strings.ToUpper(matched[i][2])
+				payload = matched[i][3]
+				switch encoding {
+				case "B":
+					str = strings.Replace(
+						str,
+						matched[i][0],
+						MailTransportDecode(payload, "base64", charset),
+						1)
+				case "Q":
+					str = strings.Replace(
+						str,
+						matched[i][0],
+						MailTransportDecode(payload, "quoted-printable", charset),
+						1)
+				}
+			}
+		}
+	}
+	return str
+}
+
+// decode from 7bit to 8bit UTF-8
+// encodingType can be "base64" or "quoted-printable"
+func MailTransportDecode(str string, encodingType string, charset string) string {
+	if charset == "" {
+		charset = "UTF-8"
+	} else {
+		charset = strings.ToUpper(charset)
+	}
+	if encodingType == "base64" {
+		str = fromBase64(str)
+	} else if encodingType == "quoted-printable" {
+		str = fromQuotedP(str)
+	}
+
+	if charset != "UTF-8" {
+		charset = fixCharset(charset)
+		// iconv is pretty good at what it does
+		if cd, err := iconv.Open("UTF-8", charset); err == nil {
+			defer func() {
+				cd.Close()
+				if r := recover(); r != nil {
+					//logln(1, fmt.Sprintf("Recovered in %v", r))
+				}
+			}()
+			// eg. charset can be "ISO-2022-JP"
+			return cd.ConvString(str)
+		}
+
+	}
+	return str
+}
+
+func fromBase64(data string) string {
+	buf := bytes.NewBufferString(data)
+	decoder := base64.NewDecoder(base64.StdEncoding, buf)
+	res, _ := ioutil.ReadAll(decoder)
+	return string(res)
+}
+
+func fromQuotedP(data string) string {
+	buf := bytes.NewBufferString(data)
+	decoder := qprintable.NewDecoder(qprintable.BinaryEncoding, buf)
+	res, _ := ioutil.ReadAll(decoder)
+	return string(res)
+}
+
+var charsetRegex, _ = regexp.Compile(`[_:.\/\\]`)
+
+func fixCharset(charset string) string {
+	fixed_charset := charsetRegex.ReplaceAllString(charset, "-")
+	// Fix charset
+	// borrowed from http://squirrelmail.svn.sourceforge.net/viewvc/squirrelmail/trunk/squirrelmail/include/languages.php?revision=13765&view=markup
+	// OE ks_c_5601_1987 > cp949
+	fixed_charset = strings.Replace(fixed_charset, "ks-c-5601-1987", "cp949", -1)
+	// Moz x-euc-tw > euc-tw
+	fixed_charset = strings.Replace(fixed_charset, "x-euc", "euc", -1)
+	// Moz x-windows-949 > cp949
+	fixed_charset = strings.Replace(fixed_charset, "x-windows_", "cp", -1)
+	// windows-125x and cp125x charsets
+	fixed_charset = strings.Replace(fixed_charset, "windows-", "cp", -1)
+	// ibm > cp
+	fixed_charset = strings.Replace(fixed_charset, "ibm", "cp", -1)
+	// iso-8859-8-i -> iso-8859-8
+	fixed_charset = strings.Replace(fixed_charset, "iso-8859-8-i", "iso-8859-8", -1)
+	if charset != fixed_charset {
+		return fixed_charset
+	}
+	return charset
 }

guerrilla.go

@@ -273,6 +273,12 @@ func (g *guerrilla) Start() error {
 	}
 	if len(startErrors) > 0 {
 		return startErrors
+	} else {
+		if gw, ok := g.backend.(*backends.BackendGateway); ok {
+			if gw.State == backends.BackendStateShuttered {
+				_ = gw.Reinitialize()
+			}
+		}
 	}
 	return nil
 }

server.go

@@ -215,30 +215,15 @@ func (server *server) allowsHost(host string) bool {
 
 // Reads from the client until a terminating sequence is encountered,
 // or until a timeout occurs.
-func (server *server) read(client *client, maxSize int64) (string, error) {
+func (server *server) readCommand(client *client, maxSize int64) (string, error) {
 	var input, reply string
 	var err error
-
 	// In command state, stop reading at line breaks
 	suffix := "\r\n"
-	if client.state == ClientData {
-		// In data state, stop reading at solo periods
-		suffix = "\r\n.\r\n"
-	}
-
 	for {
 		client.setTimeout(server.timeout.Load().(time.Duration))
 		reply, err = client.bufin.ReadString('\n')
 		input = input + reply
-		if err == nil && client.state == ClientData {
-			if reply != "" {
-				// Extract the subject while we're at it
-				client.scanSubject(reply)
-			}
-			if int64(len(input)) > maxSize {
-				return input, fmt.Errorf("Maximum DATA size exceeded (%d)", maxSize)
-			}
-		}
 		if err != nil {
 			break
 		}
@@ -313,7 +298,7 @@ func (server *server) handleClient(client *client) {
 			client.state = ClientCmd
 		case ClientCmd:
 			client.bufin.setLimit(CommandLineMaxLength)
-			input, err := server.read(client, sc.MaxSize)
+			input, err := server.readCommand(client, sc.MaxSize)
 			log.Debugf("Client sent: %s", input)
 			if err == io.EOF {
 				log.WithError(err).Warnf("Client closed the connection: %s", client.RemoteAddress)
@@ -426,11 +411,15 @@ func (server *server) handleClient(client *client) {
 			}
 
 		case ClientData:
-			var err error
+
 			// intentionally placed the limit 1MB above so that reading does not return with an error
 			// if the client goes a little over. Anything above will err
 			client.bufin.setLimit(int64(sc.MaxSize) + 1024000) // This a hard limit.
-			client.Data, err = server.read(client, sc.MaxSize)
+
+			n, err := client.Data.ReadFrom(client.smtpReader.DotReader())
+			if n > sc.MaxSize {
+				err = fmt.Errorf("Maximum DATA size exceeded (%d)", sc.MaxSize)
+			}
 			if err != nil {
 				if err == LineLimitExceeded {
 					client.responseAdd("550 Error: " + LineLimitExceeded.Error())

tests/guerrilla_test.go

@@ -692,7 +692,7 @@ func TestDataMaxLength(t *testing.T) {
 				conn,
 				bufin,
 				fmt.Sprintf("Subject:test\r\n\r\nHello %s\r\n.\r\n",
-					strings.Repeat("n", int(config.Servers[0].MaxSize-26))))
+					strings.Repeat("n", int(config.Servers[0].MaxSize-20))))
 
 			//expected := "500 Line too long"
 			expected := "451 Error: Maximum DATA size exceeded"
@@ -715,3 +715,101 @@ func TestDataMaxLength(t *testing.T) {
 	logBuffer.Reset()
 	logIn.Reset(&logBuffer)
 }
+
+func TestDataCommand(t *testing.T) {
+	if initErr != nil {
+		t.Error(initErr)
+		t.FailNow()
+	}
+
+	testHeader :=
+		"Subject: =?Shift_JIS?B?W4NYg06DRYNGg0GBRYNHg2qDYoNOg1ggg0GDSoNFg5ODZ12DQYNKg0WDk4Nn?=\r\n" +
+			"\t=?Shift_JIS?B?k2+YXoqul7mCzIKokm2C54K5?=\r\n"
+
+	email :=
+		"Delivered-To: [email protected]\r\n" +
+			"\tReceived: from mail.guerrillamail.com (mail.guerrillamail.com  [104.218.55.28:44246])\r\n" +
+			"\tby grr.la with SMTP id [email protected];\r\n" +
+			"\tWed, 18 Jan 2017 15:43:29 +0000\r\n" +
+			"Received: by 192.99.19.220 with HTTP; Wed, 18 Jan 2017 15:43:29 +0000\r\n" +
+			"MIME-Version: 1.0\r\n" +
+			"Message-ID: <[email protected]>\r\n" +
+			"Date: Wed, 18 Jan 2017 15:43:29 +0000\r\n" +
+			"To: \"[email protected]\" <[email protected]>\r\n" +
+			"From: <[email protected]>\r\n" +
+			"Subject: test\r\n" +
+			"X-Originating-IP: [60.241.160.150]\r\n" +
+			"Content-Type: text/plain; charset=\"utf-8\"\r\n" +
+			"Content-Transfer-Encoding: quoted-printable\r\n" +
+			"X-Domain-Signer: PHP mailDomainSigner 0.2-20110415 <http://code.google.com/p/php-mail-domain-signer/>\r\n" +
+			"DKIM-Signature: v=1; a=rsa-sha256; s=highgrade; d=guerrillamail.com; l=182;\r\n" +
+			"\tt=1484754209; c=relaxed/relaxed; h=to:from:subject;\r\n" +
+			"\tbh=GHSgjHpBp5QjNn9tzfug681+RcWMOUgpwAuTzppM5wY=;\r\n" +
+			"\tb=R7FxWgACnT+pKXqEg15qgzH4ywMFRx5pDlIFCnSt1BfwmLvZPZK7oOLrbiRoGGR2OJnSfyCxeASH\r\n" +
+			"\t019LNeLB/B8o+fMRX87m/tBpqIZ2vgXdT9rUCIbSDJnYoCHXakGcF+zGtTE3SEksMbeJQ76aGj6M\r\n" +
+			"\tG80p76IT2Xu3iDJLYYWxcAeX+7z4M/bbYNeqxMQcXYZp1wNYlSlHahL6RDUYdcqikDqKoXmzMNVd\r\n" +
+			"\tDr0EbH9iiu1DQtfUDzVE5LLus1yn36WU/2KJvEak45gJvm9s9J+Xrcb882CaYkxlAbgQDz1KeQLf\r\n" +
+			"\teUyNspyAabkh2yTg7kOvNZSOJtbMSQS6/GMxsg==\r\n" +
+			"\r\n" +
+			"test=0A.mooo=0A..mooo=0Atest=0A.=0A=0A=0A=0A=0A=0A----=0ASent using Guerril=\r\n" +
+			"lamail.com=0ABlock or report abuse: https://www.guerrillamail.com//abuse/?a=\r\n" +
+			"=3DVURnES0HUaZbhA8%3D=0A\r\n.\r\n"
+
+	if startErrors := app.Start(); startErrors == nil {
+		conn, bufin, err := Connect(config.Servers[0], 20)
+		if err != nil {
+			// handle error
+			t.Error(err.Error(), config.Servers[0].ListenInterface)
+			t.FailNow()
+		} else {
+			// client goes into command state
+			if _, err := Command(conn, bufin, "HELO localtester"); err != nil {
+				t.Error("Hello command failed", err.Error())
+			}
+
+			response, err := Command(conn, bufin, "MAIL FROM:[email protected]")
+			if err != nil {
+				t.Error("command failed", err.Error())
+			}
+			//fmt.Println(response)
+			response, err = Command(conn, bufin, "RCPT TO:[email protected]")
+			if err != nil {
+				t.Error("command failed", err.Error())
+			}
+			//fmt.Println(response)
+			response, err = Command(conn, bufin, "DATA")
+			if err != nil {
+				t.Error("command failed", err.Error())
+			}
+			/*
+				response, err = Command(
+					conn,
+					bufin,
+					testHeader+"\r\nHello World\r\n.\r\n")
+			*/
+			_ = testHeader
+			response, err = Command(
+				conn,
+				bufin,
+				email+"\r\n.\r\n")
+			//expected := "500 Line too long"
+			expected := "250 OK : queued as s0m3l337Ha5hva1u3LOL"
+			if strings.Index(response, expected) != 0 {
+				t.Error("Server did not respond with", expected, ", it said:"+response, err)
+			}
+
+		}
+		conn.Close()
+		app.Shutdown()
+	} else {
+		if startErrors := app.Start(); startErrors != nil {
+			t.Error(startErrors)
+			app.Shutdown()
+			t.FailNow()
+		}
+	}
+	logOut.Flush()
+	// don't forget to reset
+	logBuffer.Reset()
+	logIn.Reset(&logBuffer)
+}