Remove deprecated geodns-logs tool (use Avro logs instead)

.goreleaser.yml

@@ -20,28 +20,13 @@ builds:
         goarch: 386
       - goos: freebsd
         goarch: arm64
-  - id: geodns-logs
-    main: ./geodns-logs/
-    binary: geodns-logs
-    env:
-      - CGO_ENABLED=0
-    goos:
-      - linux
-      - freebsd
-      - darwin
-    ignore:
-      - goos: darwin
-        goarch: 386
-      - goos: freebsd
-        goarch: 386
-      - goos: freebsd
-        goarch: arm64
+
 archives:
   - files:
       - service/**
-      - service-logs/**
       - LICENSE
       - README.md
+
 checksum:
   name_template: "checksums.txt"
 snapshot:

Dockerfile

@@ -8,7 +8,6 @@ ENV CGO_ENABLED=0
 
 ADD applog/ applog/
 ADD countries/ countries/
-ADD geodns-logs/ geodns-logs/
 ADD health/ health/
 ADD monitor/ monitor/
 ADD querylog/ querylog/
@@ -17,7 +16,6 @@ ADD targeting/ targeting/
 ADD typeutil/ typeutil/
 ADD zones/ zones/
 ADD service/ service/
-ADD service-logs/ service-logs/
 ADD .git/ .git/
 ADD *.go build ./
 
@@ -29,5 +27,4 @@ RUN ln dist/* /
 
 FROM scratch
 COPY --from=build /geodns-linux-amd64 /geodns
-COPY --from=build /geodns-logs-linux-amd64 /geodns-logs
 ENTRYPOINT ["/geodns"]

build

@@ -17,11 +17,8 @@ go build -o dist/geodns-$OS-$ARCH \
   -trimpath \
   -ldflags "-X main.gitVersion=$REVISION -X main.buildTime=$BUILDTIME" \
   -v && \
-  (cd geodns-logs && go build -trimpath -v -o ../dist/geodns-logs-$OS-$ARCH && cd ..) && \
   cd dist && \
-  rm -f service service-logs && \
+  rm -f service && \
   ln -s ../service . && \
-  ln -s ../service-logs . && \
   tar -cvhf geodns-$OS-$ARCH.tar \
- --exclude \*~ geodns-$OS-$ARCH \
- geodns-logs-$OS-$ARCH service service-logs
+ --exclude \*~ geodns-$OS-$ARCH service

geodns-logs/process-stats.go

@@ -1,217 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"encoding/json"
-	"flag"
-	"log"
-	"net/http"
-	"os"
-	"strings"
-	"sync"
-
-	"github.com/miekg/dns"
-	"github.com/nxadm/tail"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-
-	"github.com/abh/geodns/v3/countries"
-	"github.com/abh/geodns/v3/querylog"
-)
-
-// TODO:
-// Add vendor yes/no
-// add server region tag (identifier)?
-
-var version string = "2.1"
-
-func main() {
-
-	log.Printf("Starting geodns-logs/%s", version)
-
-	identifierFlag := flag.String("identifier", "", "identifier (hostname, pop name or similar)")
-	// verboseFlag := flag.Bool("verbose", false, "verbose output")
-	flag.Parse()
-
-	var serverID string
-	// var serverGroups []string
-
-	if len(*identifierFlag) > 0 {
-		ids := strings.Split(*identifierFlag, ",")
-		serverID = ids[0]
-		// if len(ids) > 1 {
-		// serverGroups = ids[1:]
-		// }
-	}
-
-	if len(serverID) == 0 {
-		var err error
-		serverID, err = os.Hostname()
-		if err != nil {
-			log.Printf("Could not get hostname: %s", err)
-			os.Exit(2)
-		}
-	}
-
-	queries = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "dns_logs_total",
-			Help: "Number of served queries",
-		},
-		[]string{"zone", "vendor", "usercc", "poolcc", "qtype"},
-	)
-	prometheus.MustRegister(queries)
-
-	buildInfo := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "geodns_logs_build_info",
-			Help: "GeoDNS logs build information (in labels)",
-		},
-		[]string{"Version"},
-	)
-	prometheus.MustRegister(buildInfo)
-	buildInfo.WithLabelValues("geodns-logs/" + version).Set(1)
-
-	http.Handle("/metrics", promhttp.Handler())
-	go func() {
-		err := http.ListenAndServe(":8054", nil)
-		if err != nil {
-			log.Printf("could not start http server: %s", err)
-		}
-	}()
-
-	if len(flag.Args()) < 1 {
-		log.Printf("filename to process required")
-		os.Exit(2)
-	}
-
-	filename := flag.Arg(0)
-
-	logf, err := tail.TailFile(filename, tail.Config{
-		// Location:  &tail.SeekInfo{-1, 0},
-		Poll:      true, // inotify is flaky on EL6, so try this ...
-		ReOpen:    true,
-		MustExist: false,
-		Follow:    true,
-	})
-	if err != nil {
-		log.Printf("Could not tail '%s': %s", filename, err)
-	}
-
-	in := make(chan string)
-	go processChan(in, nil)
-
-	for line := range logf.Lines {
-		if line.Err != nil {
-			log.Printf("Error tailing file: %s", line.Err)
-		}
-		in <- line.Text
-	}
-
-}
-
-var extraValidLabels = map[string]struct{}{
-	"uk":       {},
-	"_status":  {},
-	"_country": {},
-	"www":      {},
-	"nag-test": {},
-}
-
-func validCC(label string) bool {
-	if _, ok := countries.CountryContinent[label]; ok {
-		return true
-	}
-	if _, ok := countries.ContinentCountries[label]; ok {
-		return true
-	}
-	if _, ok := countries.RegionGroupRegions[label]; ok {
-		return true
-	}
-	if _, ok := countries.RegionGroups[label]; ok {
-		return true
-	}
-	if _, ok := extraValidLabels[label]; ok {
-		return true
-	}
-	return false
-}
-
-func getPoolCC(label string) (string, bool) {
-	l := dns.SplitDomainName(label)
-	// log.Printf("LABEL: %+v", l)
-	if len(l) == 0 {
-		return "", true
-	}
-
-	for _, cc := range l {
-		if validCC(cc) {
-			return cc, true
-		}
-	}
-
-	if len(l[0]) == 1 && strings.ContainsAny(l[0], "01234") {
-		if len(l) == 1 {
-			return "", true
-		}
-	}
-
-	// log.Printf("LABEL '%s' unhandled cc...", label)
-	return "", false
-}
-
-func processChan(in chan string, wg *sync.WaitGroup) error {
-	e := querylog.Entry{}
-
-	stats := NewStats()
-
-	for line := range in {
-		err := json.Unmarshal([]byte(line), &e)
-		if err != nil {
-			log.Printf("unmarshal error '%s': %s", line, err)
-			continue
-		}
-		e.Name = strings.ToLower(e.Name)
-
-		// fmt.Printf("%s %s\n", e.Origin, e.Name)
-
-		err = stats.Add(&e)
-		if err != nil {
-			log.Printf("stats error: %s", err)
-			continue
-		}
-	}
-
-	if wg != nil {
-		wg.Done()
-	}
-	return nil
-}
-
-func processFile(file string, out chan<- *Stats) error {
-	fh, err := os.Open(file)
-	if err != nil {
-		return err
-	}
-
-	in := make(chan string)
-
-	wg := sync.WaitGroup{}
-	wg.Add(1)
-	go processChan(in, &wg)
-
-	scanner := bufio.NewScanner(fh)
-
-	for scanner.Scan() {
-		in <- scanner.Text()
-	}
-	if err := scanner.Err(); err != nil {
-		log.Println("reading standard input:", err)
-	}
-
-	close(in)
-
-	wg.Wait()
-
-	return nil
-}

geodns-logs/process-stats_test.go

@@ -1,25 +0,0 @@
-package main
-
-import "testing"
-
-func TestGetPoolCC(t *testing.T) {
-	tests := []struct {
-		label string
-		cc    string
-		ok    bool
-	}{
-		{"1.debian.pool.ntp.org.", "", false},
-		{"2.dk.pool.ntp.org.", "dk", true},
-		{"dk.pool.ntp.org.", "dk", true},
-		{"0.asia.pool.ntp.org.", "asia", true},
-		{"1.pool.ntp.org.", "", false},
-	}
-
-	for _, input := range tests {
-		cc, ok := getPoolCC(input.label)
-		if cc != input.cc || ok != input.ok {
-			t.Logf("%q got %q (%t), expected %q (%t)", input.label, cc, ok, input.cc, input.ok)
-			t.Fail()
-		}
-	}
-}

geodns-logs/stats.go

@@ -1,139 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"strings"
-	"time"
-
-	"github.com/miekg/dns"
-	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/abh/geodns/v3/querylog"
-)
-
-type statsEntry struct {
-	Time   int64
-	Origin string
-	Name   string
-	Vendor string
-	Label  string
-	Qtype  string
-	PoolCC string
-	Count  int
-}
-
-type Stats struct {
-	Count int
-	Map   map[string]*statsEntry
-}
-
-var queries *prometheus.CounterVec
-
-func NewStats() *Stats {
-	return &Stats{
-		Map: map[string]*statsEntry{},
-	}
-}
-
-func (s *Stats) Key(e *querylog.Entry) string {
-	return fmt.Sprintf("%s %s %s %d", e.Origin, e.Name, e.LabelName, e.Qtype)
-}
-
-func vendorName(n string) string {
-	idx := strings.Index(n, ".pool.ntp.org.")
-	// log.Printf("IDX for %s: %d", n, idx)
-	if idx <= 0 {
-		return ""
-	}
-	n = n[0:idx]
-
-	l := dns.SplitDomainName(n)
-
-	v := l[len(l)-1]
-
-	if len(v) == 1 && strings.ContainsAny(v, "01234") {
-		return ""
-	}
-
-	if len(v) == 2 {
-		// country code
-		return "_country"
-	}
-
-	if v == "asia" || v == "north-america" || v == "europe" || v == "south-america" || v == "oceania" || v == "africa" {
-		return "_continent"
-	}
-
-	return v
-}
-
-func (stats *Stats) Add(e *querylog.Entry) error {
-	if e.Rcode > 0 {
-		// NXDOMAIN, count separately?
-		return nil
-	}
-	if e.AnswerCount == 0 {
-		// No answers, count separately?
-		return nil
-	}
-
-	var vendor, poolCC string
-
-	if e.Origin == "pool.ntp.org" || strings.HasSuffix(e.Origin, "ntppool.org") {
-		vendor = vendorName(e.Name)
-
-		if len(vendor) == 0 && e.Name != e.Origin {
-			var ok bool
-			poolCC, ok = getPoolCC(e.LabelName)
-			if !ok {
-				log.Printf("Could not get valid poolCC label for %+v", e)
-			}
-		}
-	}
-
-	stats.Count++
-
-	qtypeString := dns.TypeToString[e.Qtype]
-
-	userCC := ""
-	for _, cc := range e.Targets {
-		if len(cc) == 2 {
-			userCC = cc
-			break
-		}
-	}
-
-	// []string{"zone", "vendor",  "usercc", "poolcc", "qtype"},
-	queries.WithLabelValues(e.Origin, vendor, userCC, poolCC, qtypeString).Inc()
-
-	key := stats.Key(e)
-
-	if s, ok := stats.Map[key]; ok {
-		s.Count++
-	} else {
-		stats.Map[key] = &statsEntry{
-			// Time:   time.Unix(e.Time/int64(time.Second), 0),
-			Time:   e.Time,
-			Origin: e.Origin,
-			Name:   e.Name,
-			Vendor: vendor,
-			Label:  e.LabelName,
-			PoolCC: poolCC,
-			Qtype:  qtypeString,
-			Count:  1,
-		}
-	}
-
-	return nil
-}
-
-func (stats *Stats) Summarize() {
-	// pretty.Println(stats)
-	var timeStamp int64
-	for k := range stats.Map {
-		timeStamp = stats.Map[k].Time
-		break
-	}
-	fmt.Printf("Stats %s count total: %d, summarized: %d\n", time.Unix(timeStamp, 0).String(), stats.Count, len(stats.Map))
-}

geodns-logs/stats_test.go

@@ -1,65 +0,0 @@
-package main
-
-import (
-	"flag"
-	"os"
-	"testing"
-)
-
-func TestMain(m *testing.M) {
-	flag.Parse()
-	os.Exit(m.Run())
-}
-
-func TestPoolCC(t *testing.T) {
-	tests := []struct {
-		Input    string
-		Expected string
-		Ok       bool
-	}{
-		{"pool.ntp.org", "", false},
-		{"2.pool.ntp.org", "", false},
-		{"us.pool.ntp.org", "us", true},
-		{"0.us.pool.ntp.org", "us", true},
-		{"asia.pool.ntp.org", "asia", true},
-		{"3.asia.pool.ntp.org", "asia", true},
-		{"3.example.pool.ntp.org", "", false},
-	}
-
-	for _, x := range tests {
-		got, ok := getPoolCC(x.Input)
-		if got != x.Expected {
-			t.Logf("Got '%s' but expected '%s' for '%s'", got, x.Expected, x.Input)
-			t.Fail()
-		}
-		if ok != x.Ok {
-			t.Logf("Got '%t' but expected '%t' for '%s'", ok, x.Ok, x.Input)
-			t.Fail()
-		}
-	}
-}
-
-func TestVendorName(t *testing.T) {
-	tests := []struct {
-		Input    string
-		Expected string
-	}{
-
-		{"pool.ntp.org.", ""},
-		{"2.pool.ntp.org.", ""},
-		{"us.pool.ntp.org.", "_country"},
-		{"2.us.pool.ntp.org.", "_country"},
-		{"europe.pool.ntp.org.", "_continent"},
-		{"2.europe.pool.ntp.org.", "_continent"},
-		{"0.example.pool.ntp.org.", "example"},
-		{"3.example.pool.ntp.org.", "example"},
-	}
-
-	for _, x := range tests {
-		got := vendorName(x.Input)
-		if got != x.Expected {
-			t.Logf("Got '%s' but expected '%s' for '%s'", got, x.Expected, x.Input)
-			t.Fail()
-		}
-	}
-}

service-logs/log/run

@@ -1,4 +0,0 @@
-#!/bin/sh
-mkdir -p /var/log/geodns-logs
-chown nobody /var/log/geodns-logs
-exec setuidgid nobody multilog s10000000 n3 /var/log/geodns-logs

service-logs/run

@@ -1,17 +0,0 @@
-#!/bin/sh
-exec 2>&1
-sleep 1 # just in case we spin for some reason
-
-cd /opt/geodns
-
-ulimit -n 8192
-
-ID=""
-if [ -e env/ID ]; then
-  ID=`head -1 env/ID`
-  if [ ! -z "$ID" ]; then
-      ID="--identifier=$ID"
-  fi
-fi
-
-exec softlimit -d200000000 envdir ./env setuidgid nobody ./geodns-logs $ID /var/log/geodns/queries.log