Refactor log and type helpers; move healthcheck code to health/ (wip)

This disables the health check feature, but I wanted a work in progress
snapshot before starting the work to enable it again.

log.go → applog/log.go

@@ -1,4 +1,4 @@
-package main
+package applog
 
 import (
 	"log"
@@ -6,6 +6,8 @@ import (
 	"time"
 )
 
+var Enabled bool
+
 type logToFile struct {
 	fn      string
 	file    *os.File
@@ -22,14 +24,14 @@ func newlogToFile(fn string) *logToFile {
 	}
 }
 
-func logPrintf(format string, a ...interface{}) {
-	if *flaglog {
+func Printf(format string, a ...interface{}) {
+	if Enabled {
 		log.Printf(format, a...)
 	}
 }
 
-func logPrintln(a ...interface{}) {
-	if *flaglog {
+func Println(a ...interface{}) {
+	if Enabled {
 		log.Println(a...)
 	}
 }
@@ -62,8 +64,7 @@ func logToFileMonitor() {
 	}
 }
 
-func logToFileOpen(fn string) {
-
+func FileOpen(fn string) {
 	ltf = newlogToFile(fn)
 
 	var err error
@@ -79,7 +80,7 @@ func logToFileOpen(fn string) {
 	go logToFileMonitor()
 }
 
-func logToFileClose() {
+func FileClose() {
 	if ltf != nil {
 		log.Printf("Closing log file")
 		errc := make(chan error) // pass a 'chan error' through the closing channel

geodns.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/abh/geodns/applog"
 	"github.com/abh/geodns/querylog"
 	"github.com/pborman/uuid"
 )
@@ -98,8 +99,12 @@ func main() {
 
 	srv := Server{}
 
+	if *flaglog {
+		applog.Enabled = true
+	}
+
 	if len(*flagLogFile) > 0 {
-		logToFileOpen(*flagLogFile)
+		applog.FileOpen(*flagLogFile)
 	}
 
 	if len(*flagidentifier) > 0 {
@@ -228,5 +233,5 @@ func main() {
 		pprof.WriteHeapProfile(f)
 		f.Close()
 	}
-	logToFileClose()
+	applog.FileClose()
 }

healthtest.go → health/healthtest.go

@@ -1,18 +1,20 @@
-package main
+package health
 
 import (
 	"fmt"
-	"log"
 	"math/rand"
 	"net"
 	"sync"
 	"time"
 
+	"github.com/abh/geodns/applog"
+	"github.com/abh/geodns/typeutil"
+
 	"github.com/miekg/dns"
 )
 
 var (
-	healthQtypes = []uint16{dns.TypeA, dns.TypeAAAA}
+	Qtypes = []uint16{dns.TypeA, dns.TypeAAAA}
 )
 
 type HealthTester interface {
@@ -51,7 +53,7 @@ type HealthTestRunner struct {
 	entryMutex sync.RWMutex
 }
 
-var healthTestRunner = &HealthTestRunner{
+var TestRunner = &HealthTestRunner{
 	entries: make(map[string]*HealthTestRunnerEntry),
 }
 
@@ -65,7 +67,7 @@ func defaultHealthTestParameters() HealthTestParameters {
 	}
 }
 
-func newHealthTest(ipAddress net.IP, htp HealthTestParameters, tester *HealthTester) *HealthTest {
+func NewTest(ipAddress net.IP, htp HealthTestParameters, tester *HealthTester) *HealthTest {
 	ht := HealthTest{
 		ipAddress:            ipAddress,
 		HealthTestParameters: htp,
@@ -98,7 +100,7 @@ func (ht *HealthTest) String() string {
 // safe copy function that copies the parameters but not (e.g.) the
 // mutex
 func (ht *HealthTest) copy(ipAddress net.IP) *HealthTest {
-	return newHealthTest(ipAddress, ht.HealthTestParameters, ht.tester)
+	return NewTest(ipAddress, ht.HealthTestParameters, ht.tester)
 }
 
 func (ht *HealthTest) setGlobal(g map[string]bool) {
@@ -147,14 +149,14 @@ func (ht *HealthTest) run() {
 				failCount = 0
 			} else {
 				failCount++
-				logPrintf("Failure for %s, retry count=%d, healthy=%v", ht.ipAddress, failCount, ht.isHealthy())
+				applog.Printf("Failure for %s, retry count=%d, healthy=%v", ht.ipAddress, failCount, ht.isHealthy())
 				if failCount >= ht.retries {
 					ht.setHealthy(false)
 					nextPoll = pollStart.Add(ht.retryTime)
 				}
 			}
 			pollStart = time.Time{}
-			logPrintf("Check result for %s health=%v, next poll at %s", ht.ipAddress, h, nextPoll)
+			applog.Printf("Check result for %s health=%v, next poll at %s", ht.ipAddress, h, nextPoll)
 			//randomDelay := rand.Int63n(time.Second.Nanoseconds())
 			//nextPoll = nextPoll.Add(time.Duration(randomDelay))
 		}
@@ -162,25 +164,26 @@ func (ht *HealthTest) run() {
 }
 
 func (ht *HealthTest) poll() {
-	logPrintf("Checking health of %s", ht.ipAddress)
+	applog.Printf("Checking health of %s", ht.ipAddress)
 	result := (*ht.tester).Test(ht)
-	logPrintf("Checked health of %s, healthy=%v", ht.ipAddress, result)
+	applog.Printf("Checked health of %s, healthy=%v", ht.ipAddress, result)
 	ht.health <- result
 }
 
 func (ht *HealthTest) start() {
 	ht.closing = make(chan chan error)
 	ht.health = make(chan bool)
-	logPrintf("Starting health test on %s, frequency=%s, retry_time=%s, timeout=%s, retries=%d", ht.ipAddress, ht.frequency, ht.retryTime, ht.timeout, ht.retries)
+	applog.Printf("Starting health test on %s, frequency=%s, retry_time=%s, timeout=%s, retries=%d", ht.ipAddress, ht.frequency, ht.retryTime, ht.timeout, ht.retries)
 	go ht.run()
 }
 
-func (ht *HealthTest) stop() (err error) {
+// Stop the health check from running
+func (ht *HealthTest) Stop() (err error) {
 	// Check it's been started by existing of the closing channel
 	if ht.closing == nil {
 		return nil
 	}
-	logPrintf("Stopping health test on %s", ht.ipAddress)
+	applog.Printf("Stopping health test on %s", ht.ipAddress)
 	errc := make(chan error)
 	ht.closing <- errc
 	err = <-errc
@@ -191,6 +194,13 @@ func (ht *HealthTest) stop() (err error) {
 	return err
 }
 
+func (ht *HealthTest) IP() net.IP {
+	return ht.ipAddress
+}
+func (ht *HealthTest) IsHealthy() bool {
+	return ht.isHealthy()
+}
+
 func (ht *HealthTest) isHealthy() bool {
 	ht.healthyMutex.RLock()
 	h := ht.healthy
@@ -204,7 +214,7 @@ func (ht *HealthTest) setHealthy(h bool) {
 	ht.healthy = h
 	ht.healthyMutex.Unlock()
 	if old != h {
-		logPrintf("Changing health status of %s from %v to %v", ht.ipAddress, old, h)
+		applog.Printf("Changing health status of %s from %v to %v", ht.ipAddress, old, h)
 	}
 }
 
@@ -244,7 +254,7 @@ func (htr *HealthTestRunner) removeTest(ht *HealthTest, ref string) {
 		ht.healthyMutex.Unlock()
 		if len(t.references) == 0 {
 			// no more references, delete the test
-			t.stop()
+			t.Stop()
 			delete(htr.entries, key)
 		}
 	}
@@ -261,7 +271,7 @@ func (htr *HealthTestRunner) refAllGlobalHealthChecks(ref string, add bool) {
 				delete(t.references, ref)
 				if len(t.references) == 0 {
 					// no more references, delete the test
-					t.stop()
+					t.Stop()
 					delete(htr.entries, key)
 				}
 			}
@@ -269,6 +279,10 @@ func (htr *HealthTestRunner) refAllGlobalHealthChecks(ref string, add bool) {
 	}
 }
 
+func (htr *HealthTestRunner) IsHealthy(ht *HealthTest) bool {
+	return htr.isHealthy(ht)
+}
+
 func (htr *HealthTestRunner) isHealthy(ht *HealthTest) bool {
 	key := ht.String()
 	htr.entryMutex.RLock()
@@ -286,121 +300,39 @@ func (htr *HealthTestRunner) isHealthy(ht *HealthTest) bool {
 	return ht.isHealthy()
 }
 
-func (z *Zone) newHealthTest(l *Label, data interface{}) {
-	// First safely get rid of any old test. As label tests
-	// should never run this should never be executed
-	if l.Test != nil {
-		l.Test.stop()
-		l.Test = nil
-	}
+func NewFromMap(i map[string]interface{}) (*HealthTest, error) {
+	ts := typeutil.ToString(i["type"])
 
-	if data == nil {
-		return
+	if len(ts) == 0 {
+		return nil, fmt.Errorf("type required")
 	}
-	if i, ok := data.(map[string]interface{}); ok {
-		if t, ok := i["type"]; ok {
-			ts := valueToString(t)
-			htp := defaultHealthTestParameters()
-			if nh, ok := HealthTesterMap[ts]; !ok {
-				log.Printf("Bad health test type '%s'", ts)
-			} else {
-				htp.testName = ts
-				h := nh(i, &htp)
-
-				for k, v := range i {
-					switch k {
-					case "frequency":
-						htp.frequency = time.Duration(valueToInt(v)) * time.Second
-					case "retry_time":
-						htp.retryTime = time.Duration(valueToInt(v)) * time.Second
-					case "timeout":
-						htp.retryTime = time.Duration(valueToInt(v)) * time.Second
-					case "retries":
-						htp.retries = valueToInt(v)
-					case "healthy_initially":
-						htp.healthyInitially = valueToBool(v)
-						logPrintf("HealthyInitially for %s is %v", l.Label, htp.healthyInitially)
-					}
-				}
 
-				l.Test = newHealthTest(nil, htp, &h)
-			}
-		}
+	htp := defaultHealthTestParameters()
+	nh, ok := HealthTesterMap[ts]
+	if !ok {
+		return nil, fmt.Errorf("Bad health test type '%s'", ts)
 	}
-}
 
-func (z *Zone) StartStopHealthChecks(start bool, oldZone *Zone) {
-	logPrintf("Start/stop health checks on zone %s start=%v", z.Origin, start)
-	for labelName, label := range z.Labels {
-		for _, qtype := range healthQtypes {
-			if label.Records[qtype] != nil && len(label.Records[qtype]) > 0 {
-				for i := range label.Records[qtype] {
-					rr := label.Records[qtype][i].RR
-					var ip net.IP
-					switch rrt := rr.(type) {
-					case *dns.A:
-						ip = rrt.A
-					case *dns.AAAA:
-						ip = rrt.AAAA
-					default:
-						continue
-					}
-					var test *HealthTest
-					ref := fmt.Sprintf("%s/%s/%d/%d", z.Origin, labelName, qtype, i)
-					if start {
-						if test = label.Records[qtype][i].Test; test != nil {
-							// stop any old test
-							healthTestRunner.removeTest(test, ref)
-						} else {
-							if ltest := label.Test; ltest != nil {
-								test = ltest.copy(ip)
-								label.Records[qtype][i].Test = test
-							}
-						}
-						if test != nil {
-							test.ipAddress = ip
-							// if we are given an oldzone, let's see if we can find the old RR and
-							// copy over the initial health state, rather than use the initial health
-							// state provided from the label. This helps to stop health state bouncing
-							// when a zone file is reloaded for a purposes unrelated to the RR
-							if oldZone != nil {
-								oLabel, ok := oldZone.Labels[labelName]
-								if ok {
-									if oLabel.Test != nil {
-										for i := range oLabel.Records[qtype] {
-											oRecord := oLabel.Records[qtype][i]
-											var oip net.IP
-											switch orrt := oRecord.RR.(type) {
-											case *dns.A:
-												oip = orrt.A
-											case *dns.AAAA:
-												oip = orrt.AAAA
-											default:
-												continue
-											}
-											if oip.Equal(ip) {
-												if oRecord.Test != nil {
-													h := oRecord.Test.isHealthy()
-													logPrintf("Carrying over previous health state for %s: %v", oRecord.Test.ipAddress, h)
-													// we know the test is stopped (as we haven't started it) so we can write
-													// without the mutex and avoid a misleading log message
-													test.healthy = h
-												}
-												break
-											}
-										}
-									}
-								}
-							}
-							healthTestRunner.addTest(test, ref)
-						}
-					} else {
-						if test = label.Records[qtype][i].Test; test != nil {
-							healthTestRunner.removeTest(test, ref)
-						}
-					}
-				}
-			}
+	htp.testName = ts
+	h := nh(i, &htp)
+
+	for k, v := range i {
+		switch k {
+		case "frequency":
+			htp.frequency = time.Duration(typeutil.ToInt(v)) * time.Second
+		case "retry_time":
+			htp.retryTime = time.Duration(typeutil.ToInt(v)) * time.Second
+		case "timeout":
+			htp.retryTime = time.Duration(typeutil.ToInt(v)) * time.Second
+		case "retries":
+			htp.retries = typeutil.ToInt(v)
+		case "healthy_initially":
+			htp.healthyInitially = typeutil.ToBool(v)
+			// applog.Printf("HealthyInitially for %s is %v", l.Label, htp.healthyInitially)
 		}
 	}
+
+	tester := NewTest(nil, htp, &h)
+	return tester, nil
+
 }

healthtesters.go → health/healthtesters.go

@@ -1,4 +1,4 @@
-package main
+package health
 
 import (
 	"crypto/sha256"
@@ -15,6 +15,9 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/abh/geodns/applog"
+	"github.com/abh/geodns/typeutil"
 )
 
 /*
@@ -38,7 +41,9 @@ import (
  * Then add a single entry to the HealthTesterTypes map pointing to the third function
  */
 
-var HealthTesterMap = map[string]func(params map[string]interface{}, htp *HealthTestParameters) HealthTester{
+type healthTesterBuilder func(params map[string]interface{}, htp *HealthTestParameters) HealthTester
+
+var HealthTesterMap = map[string]healthTesterBuilder{
 	"tcp":      newTcpHealthTester,
 	"ntp":      newNtpHealthTester,
 	"exec":     newExecHealthTester,
@@ -72,7 +77,7 @@ func (t *TcpHealthTester) String() string {
 func newTcpHealthTester(params map[string]interface{}, htp *HealthTestParameters) HealthTester {
 	port := 80
 	if v, ok := params["port"]; ok {
-		port = valueToInt(v)
+		port = typeutil.ToInt(v)
 	}
 	return &TcpHealthTester{port: port}
 }
@@ -130,7 +135,7 @@ func (t *NtpHealthTester) String() string {
 func newNtpHealthTester(params map[string]interface{}, htp *HealthTestParameters) HealthTester {
 	maxStratum := 3
 	if v, ok := params["max_stratum"]; ok {
-		maxStratum = valueToInt(v)
+		maxStratum = typeutil.ToInt(v)
 	}
 	return &NtpHealthTester{maxStratum: maxStratum}
 }
@@ -159,7 +164,7 @@ func (t *ExecHealthTester) String() string {
 func newExecHealthTester(params map[string]interface{}, htp *HealthTestParameters) HealthTester {
 	cmd := "echo '%s'"
 	if v, ok := params["cmd"]; ok {
-		cmd = valueToString(v)
+		cmd = typeutil.ToString(v)
 	}
 	return &ExecHealthTester{cmd: cmd}
 }
@@ -184,7 +189,7 @@ type FileHealthTester struct {
 
 func (t *FileHealthTester) Test(ht *HealthTest) bool {
 	if len(t.path) == 0 {
-		logPrintf("No test file path specified")
+		applog.Printf("No test file path specified")
 		return false
 	}
 
@@ -235,7 +240,7 @@ func (t *FileHealthTester) String() string {
 func newFileHealthTester(params map[string]interface{}, htp *HealthTestParameters) HealthTester {
 	var path string
 	if v, ok := params["path"]; ok {
-		path = valueToString(v)
+		path = typeutil.ToString(v)
 	}
 	htp.global = true
 	return &FileHealthTester{path: path}
@@ -269,13 +274,8 @@ type NodepingHealthTester struct {
 func (t *NodepingHealthTester) Test(ht *HealthTest) bool {
 	token := t.token
 	if len(token) == 0 {
-		cfgMutex.RLock()
-		token = Config.Nodeping.Token
-		cfgMutex.RUnlock()
-		if len(token) == 0 {
-			logPrintf("No Nodeping API key specified")
-			return false
-		}
+		applog.Printf("No Nodeping API key specified")
+		return false
 	}
 
 	var vals url.Values = url.Values{}
@@ -305,8 +305,8 @@ func (t *NodepingHealthTester) Test(ht *HealthTest) bool {
 			for _, item := range m {
 				if result, ok := item.(map[string]interface{}); ok {
 					if ip, ok := result["label"]; ok {
-						host := valueToString(ip)
-						logPrintf("Nodeping host %s health set to false", host)
+						host := typeutil.ToString(ip)
+						applog.Printf("Nodeping host %s health set to false", host)
 						state[host] = false // only down or disabled events reported
 					}
 				}
@@ -327,7 +327,7 @@ func (t *NodepingHealthTester) String() string {
 func newNodepingHealthTester(params map[string]interface{}, htp *HealthTestParameters) HealthTester {
 	var token string
 	if v, ok := params["token"]; ok {
-		token = valueToString(v)
+		token = typeutil.ToString(v)
 	}
 	// as we can only detect down nodes, not all nodes, we should assume the default is health
 	htp.healthyInitially = true
@@ -399,55 +399,35 @@ type PingdomHealthTester struct {
 func (t *PingdomHealthTester) Test(ht *HealthTest) bool {
 	username := t.username
 	if len(username) == 0 {
-		cfgMutex.RLock()
-		username = Config.Pingdom.Username
-		cfgMutex.RUnlock()
-		if len(username) == 0 {
-			logPrintf("No Pingdom username specified")
-			return false
-		}
+		applog.Printf("No Pingdom username specified")
+		return false
 	}
 
 	password := t.password
 	if len(password) == 0 {
-		cfgMutex.RLock()
-		password = Config.Pingdom.Password
-		cfgMutex.RUnlock()
-		if len(password) == 0 {
-			logPrintf("No Pingdom password specified")
-			return false
-		}
+		applog.Printf("No Pingdom password specified")
+		return false
 	}
 
 	accountEmail := t.accountEmail
-	if len(accountEmail) == 0 {
-		cfgMutex.RLock()
-		accountEmail = Config.Pingdom.AccountEmail
-		cfgMutex.RUnlock()
-	}
 
 	appKey := t.appKey
 	if len(appKey) == 0 {
-		cfgMutex.RLock()
-		appKey = Config.Pingdom.AppKey
-		cfgMutex.RUnlock()
-		if len(appKey) == 0 {
-			appKey = "gyxtnd2fzco8ys29m8luk4syag4ybmc0"
-		}
+		applog.Printf("No Pingdom appkey specified")
 	}
 
 	stateMap := t.stateMap
 	if stateMap == nil {
-		cfgMutex.RLock()
-		stateMapString := Config.Pingdom.StateMap
-		cfgMutex.RUnlock()
-		if len(stateMapString) > 0 {
-			stateMap = make(map[string]bool)
-			if err := json.Unmarshal([]byte(stateMapString), &stateMap); err != nil {
-				logPrintf("Cannot decode configfile Pingdom state map JSON")
-				return false
-			}
-		}
+		// cfgMutex.RLock()
+		// stateMapString := Config.Pingdom.StateMap
+		// cfgMutex.RUnlock()
+		// if len(stateMapString) > 0 {
+		// 	stateMap = make(map[string]bool)
+		// 	if err := json.Unmarshal([]byte(stateMapString), &stateMap); err != nil {
+		// 		applog.Printf("Cannot decode configfile Pingdom state map JSON")
+		// 		return false
+		// 	}
+		// }
 		if stateMap == nil {
 			stateMap = defaultPingdomStateMap
 		}
@@ -498,11 +478,11 @@ func (t *PingdomHealthTester) Test(ht *HealthTest) bool {
 							if check, ok := checki.(map[string]interface{}); ok {
 								if ip, ok := check["name"]; ok {
 									if status, ok := check["status"]; ok {
-										s := valueToString(status)
+										s := typeutil.ToString(status)
 										if updown, ok := stateMap[s]; ok {
-											host := valueToString(ip)
+											host := typeutil.ToString(ip)
 											state[host] = updown
-											logPrintf("Pingdom host %s state %s health set to %v", host, s, updown)
+											applog.Printf("Pingdom host %s state %s health set to %v", host, s, updown)
 										}
 									}
 								}
@@ -537,22 +517,22 @@ func newPingdomHealthTester(params map[string]interface{}, htp *HealthTestParame
 	var appKey string
 	var stateMap map[string]bool = nil
 	if v, ok := params["username"]; ok {
-		username = valueToString(v)
+		username = typeutil.ToString(v)
 	}
 	if v, ok := params["password"]; ok {
-		password = valueToString(v)
+		password = typeutil.ToString(v)
 	}
 	if v, ok := params["account_email"]; ok {
-		accountEmail = valueToString(v)
+		accountEmail = typeutil.ToString(v)
 	}
 	if v, ok := params["app_key"]; ok {
-		appKey = valueToString(v)
+		appKey = typeutil.ToString(v)
 	}
 	if v, ok := params["state_map"]; ok {
 		if vv, ok := v.(map[string]interface{}); ok {
 			stateMap = make(map[string]bool)
 			for k, s := range vv {
-				stateMap[valueToString(k)] = valueToBool(s)
+				stateMap[typeutil.ToString(k)] = typeutil.ToBool(s)
 			}
 		}
 	}

picker.go

@@ -3,6 +3,8 @@ package main
 import (
 	"math/rand"
 
+	"github.com/abh/geodns/health"
+
 	"github.com/miekg/dns"
 )
 
@@ -36,7 +38,7 @@ func (label *Label) Picker(qtype uint16, max int, location *Location) Records {
 			tmpServers := servers[:0]
 			sum = 0
 			for i, s := range servers {
-				if servers[i].Test == nil || healthTestRunner.isHealthy(servers[i].Test) {
+				if servers[i].Test == nil || health.TestRunner.IsHealthy(servers[i].Test) {
 					tmpServers = append(tmpServers, s)
 					sum += s.Weight
 				}

serve.go

@@ -10,7 +10,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/abh/geodns/applog"
+	"github.com/abh/geodns/health"
 	"github.com/abh/geodns/querylog"
+
 	"github.com/miekg/dns"
 	"github.com/rcrowley/go-metrics"
 )
@@ -38,7 +41,7 @@ func (srv *Server) serve(w dns.ResponseWriter, req *dns.Msg, z *Zone) {
 		defer srv.queryLogger.Write(qle)
 	}
 
-	logPrintf("[zone %s] incoming  %s %s (id %d) from %s\n", z.Origin, qname,
+	applog.Printf("[zone %s] incoming  %s %s (id %d) from %s\n", z.Origin, qname,
 		dns.TypeToString[qtype], req.Id, w.RemoteAddr())
 
 	// Global meter
@@ -47,7 +50,7 @@ func (srv *Server) serve(w dns.ResponseWriter, req *dns.Msg, z *Zone) {
 	// Zone meter
 	z.Metrics.Queries.Mark(1)
 
-	logPrintln("Got request", req)
+	applog.Println("Got request", req)
 
 	label := getQuestionName(z, req)
 
@@ -84,7 +87,7 @@ func (srv *Server) serve(w dns.ResponseWriter, req *dns.Msg, z *Zone) {
 					// do stuff with e.Nsid
 				case *dns.EDNS0_SUBNET:
 					z.Metrics.EdnsQueries.Mark(1)
-					logPrintln("Got edns", e.Address, e.Family, e.SourceNetmask, e.SourceScope)
+					applog.Println("Got edns", e.Address, e.Family, e.SourceNetmask, e.SourceScope)
 					if e.Address != nil {
 						edns = e
 						ip = e.Address
@@ -239,7 +242,7 @@ func (srv *Server) serve(w dns.ResponseWriter, req *dns.Msg, z *Zone) {
 		m.Ns = append(m.Ns, z.SoaRR())
 	}
 
-	logPrintln(m)
+	applog.Println(m)
 
 	if qle != nil {
 		qle.LabelName = labels.Label
@@ -280,7 +283,7 @@ func (z *Zone) healthRR(label string, baseLabel string) []dns.RR {
 	h := dns.RR_Header{Ttl: 1, Class: dns.ClassINET, Rrtype: dns.TypeTXT}
 	h.Name = label
 
-	health := make(map[string]map[string]bool)
+	healthstatus := make(map[string]map[string]bool)
 
 	if l, ok := z.Labels[baseLabel]; ok {
 		for qt, records := range l.Records {
@@ -288,15 +291,15 @@ func (z *Zone) healthRR(label string, baseLabel string) []dns.RR {
 				hmap := make(map[string]bool)
 				for _, record := range records {
 					if record.Test != nil {
-						hmap[(*record.Test).ipAddress.String()] = healthTestRunner.isHealthy(record.Test)
+						hmap[(*record.Test).IP().String()] = health.TestRunner.IsHealthy(record.Test)
 					}
 				}
-				health[qts] = hmap
+				healthstatus[qts] = hmap
 			}
 		}
 	}
 
-	js, _ := json.Marshal(health)
+	js, _ := json.Marshal(healthstatus)
 
 	return []dns.RR{&dns.TXT{Hdr: h, Txt: []string{string(js)}}}
 }

server.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/abh/geodns/querylog"
+
 	"github.com/miekg/dns"
 )
 
@@ -49,14 +50,14 @@ func (srv *Server) addHandler(zones Zones, name string, config *Zone) {
 	oldZone := zones[name]
 	// across the recconfiguration keep a reference to all healthchecks to ensure
 	// the global map doesn't get destroyed
-	healthTestRunner.refAllGlobalHealthChecks(name, true)
-	defer healthTestRunner.refAllGlobalHealthChecks(name, false)
-	if oldZone != nil {
-		oldZone.StartStopHealthChecks(false, nil)
-	}
+	// health.TestRunner.refAllGlobalHealthChecks(name, true)
+	// defer health.TestRunner.refAllGlobalHealthChecks(name, false)
+	// if oldZone != nil {
+	// 	oldZone.StartStopHealthChecks(false, nil)
+	// }
 	config.SetupMetrics(oldZone)
 	zones[name] = config
-	config.StartStopHealthChecks(true, oldZone)
+	// config.StartStopHealthChecks(true, oldZone)
 	dns.HandleFunc(name, srv.setupServerFunc(config))
 }
 

typeutil/typeutil.go

@@ -0,0 +1,60 @@
+package typeutil
+
+import (
+	"log"
+	"strconv"
+)
+
+func ToBool(v interface{}) (rv bool) {
+	switch v.(type) {
+	case bool:
+		rv = v.(bool)
+	case string:
+		str := v.(string)
+		switch str {
+		case "true":
+			rv = true
+		case "1":
+			rv = true
+		}
+	case float64:
+		if v.(float64) > 0 {
+			rv = true
+		}
+	default:
+		log.Println("Can't convert", v, "to bool")
+		panic("Can't convert value")
+	}
+	return rv
+
+}
+
+func ToString(v interface{}) (rv string) {
+	switch v.(type) {
+	case string:
+		rv = v.(string)
+	case float64:
+		rv = strconv.FormatFloat(v.(float64), 'f', -1, 64)
+	default:
+		log.Println("Can't convert", v, "to string")
+		panic("Can't convert value")
+	}
+	return rv
+}
+
+func ToInt(v interface{}) (rv int) {
+	switch v.(type) {
+	case string:
+		i, err := strconv.Atoi(v.(string))
+		if err != nil {
+			panic("Error converting weight to integer")
+		}
+		rv = i
+	case float64:
+		rv = int(v.(float64))
+	default:
+		log.Println("Can't convert", v, "to integer")
+		panic("Can't convert value")
+	}
+	return rv
+}

zone.go

@@ -4,6 +4,9 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/abh/geodns/applog"
+	"github.com/abh/geodns/health"
+
 	"github.com/miekg/dns"
 	"github.com/rcrowley/go-metrics"
 )
@@ -26,7 +29,7 @@ type Record struct {
 	RR     dns.RR
 	Weight int
 	Loc    *Location
-	Test   *HealthTest
+	Test   *health.HealthTest
 }
 
 type Records []Record
@@ -45,7 +48,7 @@ type Label struct {
 	Records  map[uint16]Records
 	Weight   map[uint16]int
 	Closest  bool
-	Test     *HealthTest
+	Test     *health.HealthTest
 }
 
 type labels map[string]*Label
@@ -195,7 +198,6 @@ func (z *Zone) findLabels(s string, targets []string, qts qTypes) (*Label, uint1
 
 // Find the locations of all the A records within a zone. If we were being really clever
 // here we could use LOC records too. But for the time being we'll just use GeoIP
-
 func (z *Zone) SetLocations() {
 	qtypes := []uint16{dns.TypeA}
 	for _, label := range z.Labels {
@@ -216,3 +218,103 @@ func (z *Zone) SetLocations() {
 		}
 	}
 }
+
+func (z *Zone) newHealthTest(l *Label, data interface{}) {
+	// First safely get rid of any old test. As label tests
+	// should never run this should never be executed
+	if l.Test != nil {
+		l.Test.Stop()
+		l.Test = nil
+	}
+
+	if data == nil {
+		return
+	}
+
+	if i, ok := data.(map[string]interface{}); ok {
+		tester, err := health.NewFromMap(i)
+		if err != nil {
+			applog.Printf("Could not configure health check: %s", err)
+			return
+		}
+		l.Test = tester
+
+	}
+}
+
+func (z *Zone) StartStopHealthChecks(start bool, oldZone *Zone) {
+	// 	applog.Printf("Start/stop health checks on zone %s start=%v", z.Origin, start)
+	// 	for labelName, label := range z.Labels {
+	// 		for _, qtype := range health.Qtypes {
+	// 			if label.Records[qtype] != nil && len(label.Records[qtype]) > 0 {
+	// 				for i := range label.Records[qtype] {
+	// 					rr := label.Records[qtype][i].RR
+	// 					var ip net.IP
+	// 					switch rrt := rr.(type) {
+	// 					case *dns.A:
+	// 						ip = rrt.A
+	// 					case *dns.AAAA:
+	// 						ip = rrt.AAAA
+	// 					default:
+	// 						continue
+	// 					}
+
+	// 					var test *health.HealthTest
+	// 					ref := fmt.Sprintf("%s/%s/%d/%d", z.Origin, labelName, qtype, i)
+	// 					if start {
+	// 						if test = label.Records[qtype][i].Test; test != nil {
+	// 							// stop any old test
+	// 							health.TestRunner.removeTest(test, ref)
+	// 						} else {
+	// 							if ltest := label.Test; ltest != nil {
+	// 								test = ltest.copy(ip)
+	// 								label.Records[qtype][i].Test = test
+	// 							}
+	// 						}
+	// 						if test != nil {
+	// 							test.ipAddress = ip
+	// 							// if we are given an oldzone, let's see if we can find the old RR and
+	// 							// copy over the initial health state, rather than use the initial health
+	// 							// state provided from the label. This helps to stop health state bouncing
+	// 							// when a zone file is reloaded for a purposes unrelated to the RR
+	// 							if oldZone != nil {
+	// 								oLabel, ok := oldZone.Labels[labelName]
+	// 								if ok {
+	// 									if oLabel.Test != nil {
+	// 										for i := range oLabel.Records[qtype] {
+	// 											oRecord := oLabel.Records[qtype][i]
+	// 											var oip net.IP
+	// 											switch orrt := oRecord.RR.(type) {
+	// 											case *dns.A:
+	// 												oip = orrt.A
+	// 											case *dns.AAAA:
+	// 												oip = orrt.AAAA
+	// 											default:
+	// 												continue
+	// 											}
+	// 											if oip.Equal(ip) {
+	// 												if oRecord.Test != nil {
+	// 													h := oRecord.Test.IsHealthy()
+	// 													applog.Printf("Carrying over previous health state for %s: %v", oRecord.Test.ipAddress, h)
+	// 													// we know the test is stopped (as we haven't started it) so we can write
+	// 													// without the mutex and avoid a misleading log message
+	// 													test.healthy = h
+	// 												}
+	// 												break
+	// 											}
+	// 										}
+	// 									}
+	// 								}
+	// 							}
+	// 							health.TestRunner.addTest(test, ref)
+	// 						}
+	// 					} else {
+	// 						if test = label.Records[qtype][i].Test; test != nil {
+	// 							health.TestRunner.removeTest(test, ref)
+	// 						}
+	// 					}
+	// 				}
+	// 			}
+	// 		}
+	// 	}
+}

zones.go

@@ -16,6 +16,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/abh/geodns/applog"
+	"github.com/abh/geodns/typeutil"
+
 	"github.com/abh/errorutil"
 	"github.com/miekg/dns"
 )
@@ -56,10 +59,10 @@ func (srv *Server) zonesReadDir(dirName string, zones Zones) error {
 		if _, ok := lastRead[zoneName]; !ok || file.ModTime().After(lastRead[zoneName].time) {
 			modTime := file.ModTime()
 			if ok {
-				logPrintf("Reloading %s\n", fileName)
+				applog.Printf("Reloading %s\n", fileName)
 				lastRead[zoneName].time = modTime
 			} else {
-				logPrintf("Reading new file %s\n", fileName)
+				applog.Printf("Reading new file %s\n", fileName)
 				lastRead[zoneName] = &ZoneReadRecord{time: modTime}
 			}
 
@@ -88,7 +91,7 @@ func (srv *Server) zonesReadDir(dirName string, zones Zones) error {
 
 			sha256 := sha256File(filename)
 			if lastRead[zoneName].hash == sha256 {
-				logPrintf("Skipping new file %s as hash is unchanged\n", filename)
+				applog.Printf("Skipping new file %s as hash is unchanged\n", filename)
 				continue
 			}
 
@@ -193,13 +196,13 @@ func readZoneFile(zoneName, fileName string) (zone *Zone, zerr error) {
 
 		switch k {
 		case "ttl":
-			zone.Options.Ttl = valueToInt(v)
+			zone.Options.Ttl = typeutil.ToInt(v)
 		case "serial":
-			zone.Options.Serial = valueToInt(v)
+			zone.Options.Serial = typeutil.ToInt(v)
 		case "contact":
 			zone.Options.Contact = v.(string)
 		case "max_hosts":
-			zone.Options.MaxHosts = valueToInt(v)
+			zone.Options.MaxHosts = typeutil.ToInt(v)
 		case "closest":
 			zone.Options.Closest = v.(bool)
 			if zone.Options.Closest {
@@ -218,9 +221,9 @@ func readZoneFile(zoneName, fileName string) (zone *Zone, zerr error) {
 				for logger, v := range v.(map[string]interface{}) {
 					switch logger {
 					case "stathat":
-						logging.StatHat = valueToBool(v)
+						logging.StatHat = typeutil.ToBool(v)
 					case "stathat_api":
-						logging.StatHatAPI = valueToString(v)
+						logging.StatHatAPI = typeutil.ToString(v)
 						logging.StatHat = true
 					default:
 						log.Println("Unknown logger option", logger)
@@ -283,7 +286,7 @@ func setupZoneData(data map[string]interface{}, Zone *Zone) {
 		for rType, rdata := range dv {
 			switch rType {
 			case "max_hosts":
-				label.MaxHosts = valueToInt(rdata)
+				label.MaxHosts = typeutil.ToInt(rdata)
 				continue
 			case "closest":
 				label.Closest = rdata.(bool)
@@ -292,7 +295,7 @@ func setupZoneData(data map[string]interface{}, Zone *Zone) {
 				}
 				continue
 			case "ttl":
-				label.Ttl = valueToInt(rdata)
+				label.Ttl = typeutil.ToInt(rdata)
 				continue
 			case "test":
 				Zone.newHealthTest(label, rdata)
@@ -390,10 +393,10 @@ func setupZoneData(data map[string]interface{}, Zone *Zone) {
 						mx = mx + "."
 					}
 					if rec["weight"] != nil {
-						record.Weight = valueToInt(rec["weight"])
+						record.Weight = typeutil.ToInt(rec["weight"])
 					}
 					if rec["preference"] != nil {
-						pref = uint16(valueToInt(rec["preference"]))
+						pref = uint16(typeutil.ToInt(rec["preference"]))
 					}
 					record.RR = &dns.MX{
 						Hdr:        h,
@@ -412,13 +415,13 @@ func setupZoneData(data map[string]interface{}, Zone *Zone) {
 					}
 
 					if rec["srv_weight"] != nil {
-						srv_weight = uint16(valueToInt(rec["srv_weight"]))
+						srv_weight = uint16(typeutil.ToInt(rec["srv_weight"]))
 					}
 					if rec["port"] != nil {
-						port = uint16(valueToInt(rec["port"]))
+						port = uint16(typeutil.ToInt(rec["port"]))
 					}
 					if rec["priority"] != nil {
-						priority = uint16(valueToInt(rec["priority"]))
+						priority = uint16(typeutil.ToInt(rec["priority"]))
 					}
 					record.RR = &dns.SRV{
 						Hdr:      h,
@@ -484,7 +487,7 @@ func setupZoneData(data map[string]interface{}, Zone *Zone) {
 						recmap := rec.(map[string]interface{})
 
 						if weight, ok := recmap["weight"]; ok {
-							record.Weight = valueToInt(weight)
+							record.Weight = typeutil.ToInt(weight)
 						}
 						if t, ok := recmap["txt"]; ok {
 							txt = t.(string)
@@ -512,7 +515,7 @@ func setupZoneData(data map[string]interface{}, Zone *Zone) {
 						recmap := rec.(map[string]interface{})
 
 						if weight, ok := recmap["weight"]; ok {
-							record.Weight = valueToInt(weight)
+							record.Weight = typeutil.ToInt(weight)
 						}
 						if t, ok := recmap["spf"]; ok {
 							spf = t.(string)
@@ -647,60 +650,6 @@ func setupSOA(Zone *Zone) {
 
 }
 
-func valueToBool(v interface{}) (rv bool) {
-	switch v.(type) {
-	case bool:
-		rv = v.(bool)
-	case string:
-		str := v.(string)
-		switch str {
-		case "true":
-			rv = true
-		case "1":
-			rv = true
-		}
-	case float64:
-		if v.(float64) > 0 {
-			rv = true
-		}
-	default:
-		log.Println("Can't convert", v, "to bool")
-		panic("Can't convert value")
-	}
-	return rv
-
-}
-
-func valueToString(v interface{}) (rv string) {
-	switch v.(type) {
-	case string:
-		rv = v.(string)
-	case float64:
-		rv = strconv.FormatFloat(v.(float64), 'f', -1, 64)
-	default:
-		log.Println("Can't convert", v, "to string")
-		panic("Can't convert value")
-	}
-	return rv
-}
-
-func valueToInt(v interface{}) (rv int) {
-	switch v.(type) {
-	case string:
-		i, err := strconv.Atoi(v.(string))
-		if err != nil {
-			panic("Error converting weight to integer")
-		}
-		rv = i
-	case float64:
-		rv = int(v.(float64))
-	default:
-		log.Println("Can't convert", v, "to integer")
-		panic("Can't convert value")
-	}
-	return rv
-}
-
 func zoneNameFromFile(fileName string) string {
 	return fileName[0:strings.LastIndex(fileName, ".")]
 }