Experiment with generating prometheus metrics from log processor

geodns-influxdb/influx.go

@@ -10,7 +10,7 @@ import (
 	"github.com/kr/pretty"
 )
 
-const UserAgent = "geodns-logs/1.1"
+const UserAgent = "geodns-logs/1.2"
 
 type influxClient struct {
 	ServerID     string

geodns-influxdb/process-stats.go

@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 	"strings"
 	"sync"
@@ -13,6 +14,8 @@ import (
 
 	"github.com/hpcloud/tail"
 	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"github.com/abh/geodns/countries"
 	"github.com/abh/geodns/querylog"
@@ -49,6 +52,18 @@ func main() {
 		}
 	}
 
+	queries = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "dns_logs_total",
+			Help: "Number of served queries",
+		},
+		[]string{"zone", "vendor", "usercc", "poolcc", "qtype"},
+	)
+	prometheus.MustRegister(queries)
+
+	http.Handle("/metrics", promhttp.Handler())
+	go http.ListenAndServe(":8054", nil)
+
 	influx := NewInfluxClient()
 	influx.URL = os.Getenv("INFLUXDB_URL")
 	influx.Username = os.Getenv("INFLUXDB_USERNAME")
@@ -174,6 +189,7 @@ func processChan(in chan string, out chan<- *Stats, wg *sync.WaitGroup) error {
 			log.Printf("Can't unmarshal '%s': %s", line, err)
 			return err
 		}
+		e.Name = strings.ToLower(e.Name)
 
 		eMinute := ((e.Time - e.Time%int64(submitInterval)) / int64(time.Second))
 		e.Time = eMinute
@@ -192,7 +208,6 @@ func processChan(in chan string, out chan<- *Stats, wg *sync.WaitGroup) error {
 			}
 		}
 
-		e.Name = strings.ToLower(e.Name)
 		// fmt.Printf("%s %s\n", e.Origin, e.Name)
 
 		err = stats.Add(&e)

geodns-influxdb/stats.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/abh/geodns/querylog"
 	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 type statsEntry struct {
@@ -26,6 +27,8 @@ type Stats struct {
 	Map   map[string]*statsEntry
 }
 
+var queries *prometheus.CounterVec
+
 func NewStats() *Stats {
 	return &Stats{
 		Map: map[string]*statsEntry{},
@@ -88,6 +91,19 @@ func (stats *Stats) Add(e *querylog.Entry) error {
 
 	stats.Count++
 
+	qtypeString := dns.TypeToString[e.Qtype]
+
+	userCC := ""
+	for _, cc := range e.Targets {
+		if len(cc) == 2 {
+			userCC = cc
+			break
+		}
+	}
+
+	// []string{"zone", "vendor",  "usercc", "poolcc", "qtype"},
+	queries.WithLabelValues(e.Origin, vendor, userCC, poolCC, qtypeString).Inc()
+
 	key := stats.Key(e)
 
 	if s, ok := stats.Map[key]; ok {
@@ -101,7 +117,7 @@ func (stats *Stats) Add(e *querylog.Entry) error {
 			Vendor: vendor,
 			Label:  e.LabelName,
 			PoolCC: poolCC,
-			Qtype:  dns.TypeToString[e.Qtype],
+			Qtype:  qtypeString,
 			Count:  1,
 		}
 	}

server/serve.go

@@ -12,9 +12,9 @@ import (
 	"github.com/abh/geodns/applog"
 	"github.com/abh/geodns/querylog"
 	"github.com/abh/geodns/zones"
-	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/miekg/dns"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 func getQuestionName(z *zones.Zone, fqdn string) string {