Add 'top labels' to status page

monitor.go

@@ -278,7 +278,11 @@ func StatusServer(zones Zones) func(http.ResponseWriter, *http.Request) {
 
 		for name, zone := range zones {
 			count := zone.Metrics.Queries.Count()
-			rates = append(rates, &rate{Name: name, Count: count, Metrics: zone.Metrics})
+			rates = append(rates, &rate{
+				Name:    name,
+				Count:   count,
+				Metrics: zone.Metrics,
+			})
 		}
 
 		sort.Sort(RatesByCount{rates})

serve.go

@@ -38,6 +38,8 @@ func serve(w dns.ResponseWriter, req *dns.Msg, z *Zone) {
 
 	label := getQuestionName(z, req)
 
+	z.Metrics.LabelStats.Add(label)
+
 	var ip string
 	var edns *dns.EDNS0_SUBNET
 	var opt_rr *dns.OPT

status.html

@@ -119,6 +119,21 @@
 		<td>{{printf "%.2f" .Metrics.EdnsQueries.RateMean}}</td>
 	</tr>
 
+	{{ $counts := .Metrics.LabelStats.TopCounts 10 }}
+
+	{{ if $counts }}
+	<tr class="info">
+		<td colspan=5>Labels</td>
+	</tr>
+
+	{{range $counts }}
+		<tr>
+			<td colspan=4>{{.Label}}</td><td>{{.Count}}</td>
+		</tr>
+	{{end}}
+
+	{{end}}
+
 {{end}}
 
 </table>

status.html.go

@@ -11,66 +11,74 @@ import (
 // status_html returns raw, uncompressed file data.
 func status_html() []byte {
 	gz, err := gzip.NewReader(bytes.NewBuffer([]byte{
-0x1f,0x8b,0x08,0x00,0x00,0x09,0x6e,0x88,0x00,0xff,0xb4,0x56,
-0xcf,0x6f,0xda,0x30,0x14,0x3e,0xc3,0x5f,0x61,0x45,0xea,0x6d,
-0x75,0x4a,0x37,0xa4,0xb1,0x85,0x9c,0x5a,0x75,0x17,0xaa,0xb6,
-0x68,0x3b,0xec,0x66,0xe2,0x07,0xb1,0x16,0x6c,0x64,0xbf,0x96,
-0x76,0x51,0xff,0xf7,0xd9,0xf9,0x45,0x80,0xc0,0xc0,0xd3,0x2e,
-0x89,0xf3,0xd9,0x9f,0xde,0x7b,0xdf,0xe7,0xf8,0x39,0x4a,0x71,
-0x99,0xc5,0x51,0x0a,0x8c,0xc7,0x11,0x0a,0xcc,0x20,0xbe,0x03,
-0x75,0x73,0x3f,0x25,0x79,0x4e,0xe8,0x0f,0xd0,0x46,0x28,0x49,
-0xde,0xdf,0xa3,0xb0,0x9c,0xec,0x47,0x99,0x90,0xbf,0x48,0xaa,
-0x61,0x3e,0x0e,0xc2,0x50,0x02,0x72,0xc9,0xe8,0x4c,0x29,0x34,
-0xa8,0xd9,0x2a,0xe1,0x92,0x26,0x6a,0x19,0xe2,0x5a,0x20,0x82,
-0xbe,0x6c,0x26,0xc2,0x6b,0xfa,0x91,0x0e,0xc2,0xc4,0x98,0xb0,
-0xc1,0x2e,0xed,0xca,0x99,0x90,0xc0,0xe9,0x52,0x58,0x9a,0x31,
-0x01,0xd1,0x90,0x8d,0x03,0x83,0x6f,0x19,0x98,0x14,0x00,0x83,
-0x53,0xe3,0x15,0xc0,0x9a,0x61,0x92,0xd6,0x81,0x40,0x3f,0x67,
-0xc0,0xe4,0x26,0xda,0xd1,0x20,0xc5,0x57,0xdc,0xef,0x21,0xa7,
-0xbf,0x95,0x04,0xc9,0x96,0xf0,0xc1,0x8e,0x9d,0x2c,0xa0,0x49,
-0x4e,0xe6,0x4a,0xe2,0xe5,0x1a,0xc4,0x22,0xc5,0x2f,0x64,0xa6,
-0x32,0xfe,0x95,0xbc,0xf7,0xa3,0xb0,0xa2,0x45,0x33,0xc5,0xdf,
-0xe2,0x7e,0x3f,0xe2,0xe2,0x85,0x24,0x19,0x33,0x66,0x1c,0x24,
-0x96,0xc1,0x6c,0x71,0x3a,0x70,0x13,0xe9,0x20,0xbe,0xcb,0xd4,
-0x8c,0x65,0x51,0x68,0x87,0xdb,0x2b,0xb5,0x5a,0xbb,0x35,0xbd,
-0x36,0x66,0x56,0x4c,0x7e,0x2e,0xd0,0x5e,0x84,0x6c,0x96,0x41,
-0x3d,0x51,0x7e,0x14,0x4f,0x2b,0xae,0xb6,0xd9,0x01,0xaf,0x3e,
-0x6d,0x44,0x0e,0xd2,0x00,0xb7,0xbc,0x9e,0xe3,0xe9,0xe2,0x6d,
-0x07,0xbc,0x66,0x0b,0x39,0x57,0xa4,0x2c,0x2a,0xa8,0x12,0x22,
-0x06,0x19,0x1a,0x6b,0x2f,0x6f,0x56,0xc7,0x03,0x62,0xb5,0xda,
-0x86,0x86,0xfb,0xd0,0xa0,0x03,0x9b,0x58,0xc9,0x1b,0xc4,0x0e,
-0x74,0x51,0xc2,0x56,0x2e,0xf1,0xe3,0x33,0x68,0x01,0x3b,0x21,
-0xf3,0x7c,0xa5,0x85,0xc4,0x39,0x09,0x2e,0xe8,0xf5,0x3c,0x20,
-0xb4,0xcc,0x8e,0x56,0x8b,0xe9,0x13,0x43,0x18,0x14,0xdb,0xf0,
-0x4c,0xd2,0xd0,0x87,0x34,0xf0,0x62,0xb9,0xe2,0x5b,0xbc,0xb2,
-0xfc,0xe2,0xed,0xfc,0xf9,0xbf,0x66,0x7e,0x13,0x06,0xd5,0x42,
-0xb3,0xe5,0x8e,0xae,0x93,0xa3,0x0e,0x15,0xc8,0xe8,0xea,0x62,
-0x07,0x18,0xed,0x01,0x74,0x17,0x9a,0xb0,0xd7,0x6d,0x60,0x8a,
-0xfc,0x06,0x5e,0xfe,0xe6,0xfd,0x54,0xc8,0x04,0xdc,0x96,0xd3,
-0x78,0x50,0x5f,0x1e,0x10,0xd2,0xe8,0xdb,0xd4,0x45,0x6d,0x25,
-0xa7,0x9a,0xd2,0x22,0x6d,0x5b,0x72,0x22,0xeb,0x21,0xc1,0xd1,
-0x15,0xf1,0xe3,0x8d,0x7c,0x79,0x47,0x89,0x87,0x34,0x61,0xaf,
-0xe7,0x47,0x2b,0x9d,0x22,0x7b,0x3b,0x75,0xcf,0xac,0x27,0x48,
-0x40,0x9e,0xeb,0x53,0x49,0xf2,0x72,0xab,0xa6,0xfa,0x78,0x56,
-0x71,0x3d,0x9d,0x6b,0xb1,0x3d,0xfc,0x6b,0xb3,0x3d,0x5c,0xac,
-0xcb,0xf6,0xf0,0xb2,0xa2,0x1e,0x71,0xb4,0x75,0xf8,0x44,0xa1,
-0x6d,0x30,0x5d,0x8d,0xe6,0xd3,0x3f,0x35,0x9a,0x7a,0xcb,0x1c,
-0x3a,0x99,0xbe,0xaf,0x50,0x2c,0xa1,0xc8,0xac,0xac,0x89,0x96,
-0x08,0xbd,0x61,0x6f,0x53,0xb4,0xf5,0x2d,0xea,0xbc,0xfb,0x9d,
-0x67,0x66,0x9d,0x76,0xf3,0xb6,0x0d,0xf4,0xa7,0xed,0xd1,0xa6,
-0x6c,0xa5,0x5e,0x59,0x17,0x39,0x47,0x98,0xc6,0x36,0x4e,0x5a,
-0x8e,0x8a,0x9e,0xf7,0x8c,0xb0,0x41,0x86,0x15,0x62,0x5a,0x8b,
-0x3a,0x30,0xb7,0x5d,0xc9,0xe3,0xc3,0xb4,0x42,0x4a,0xe1,0xf3,
-0x5c,0x33,0xb9,0x00,0xeb,0x76,0x91,0xaa,0xb5,0xc6,0x55,0x82,
-0xba,0xad,0x50,0xa5,0x9e,0x55,0x4d,0x65,0xce,0x86,0xf1,0xb0,
-0x9e,0xad,0xaf,0x20,0x81,0x53,0xeb,0xde,0x0e,0x1a,0x67,0x6b,
-0x5f,0x2b,0xd1,0x3b,0xda,0x69,0xe7,0xbe,0x99,0x80,0x15,0x3a,
-0x31,0x87,0xba,0xe9,0xc9,0x9c,0xa1,0x07,0x67,0xe0,0x43,0xda,
-0x3a,0x03,0x3a,0x8a,0xbe,0x75,0x37,0xd4,0x73,0x2a,0xbf,0xe5,
-0xd2,0xf8,0x54,0xbf,0xcb,0x3b,0xb5,0x98,0xbd,0x78,0xbe,0xc4,
-0x2e,0x25,0xf2,0x1c,0x24,0x77,0x5b,0x6a,0xf3,0x9b,0x6c,0xfe,
-0x92,0xf2,0x2e,0x6a,0x7f,0x0e,0x77,0xb1,0xef,0xff,0x09,0x00,
-0x00,0xff,0xff,0x48,0x16,0xe4,0x27,0xe0,0x0b,0x00,0x00,
+0x1f,0x8b,0x08,0x00,0x00,0x09,0x6e,0x88,0x00,0xff,0xb4,0x57,
+0x4d,0x73,0xe2,0x38,0x10,0x3d,0xc3,0xaf,0x50,0xb9,0x36,0xb7,
+0x8d,0x1c,0x67,0x43,0xd5,0x92,0x35,0xbe,0x6c,0x52,0xd9,0xc3,
+0x92,0x4a,0xc2,0xee,0x1c,0xe6,0x26,0xac,0x06,0xbb,0xc6,0x48,
+0x94,0xd4,0x84,0x64,0x5c,0xf9,0xef,0x23,0xcb,0x1f,0x18,0x0c,
+0x04,0x34,0x35,0x17,0x2c,0x3f,0xe9,0xa9,0x3f,0x5e,0x5b,0x2d,
+0xc2,0x04,0x17,0x59,0x14,0x26,0xc0,0x78,0x14,0x62,0x8a,0x19,
+0x44,0x0f,0x20,0xef,0x1e,0x27,0x24,0xcf,0x09,0xfd,0x02,0x4a,
+0xa7,0x52,0x90,0x8f,0x8f,0xd0,0x2f,0x27,0xfb,0x61,0x96,0x8a,
+0x6f,0x24,0x51,0x30,0x1b,0x79,0xbe,0x2f,0x00,0xb9,0x60,0x74,
+0x2a,0x25,0x6a,0x54,0x6c,0x19,0x73,0x41,0x63,0xb9,0xf0,0x71,
+0x9d,0x22,0x82,0xba,0x6c,0x26,0xfc,0x6b,0xfa,0x07,0x0d,0xfc,
+0x58,0x6b,0xbf,0xc1,0x2e,0xcd,0xca,0x69,0x2a,0x80,0xd3,0x45,
+0x6a,0x68,0x5a,0x7b,0x44,0x41,0x36,0xf2,0x34,0xbe,0x67,0xa0,
+0x13,0x00,0xf4,0x4e,0xb5,0x67,0x81,0x35,0xc3,0x38,0xa9,0x0d,
+0x81,0x5a,0x65,0xc0,0xc4,0xc6,0xda,0x51,0x23,0xf6,0x2d,0xea,
+0xf7,0x90,0xd3,0xef,0x52,0x80,0x60,0x0b,0xf8,0xdd,0x8c,0x8b,
+0xb4,0x80,0x22,0x39,0x99,0x49,0x81,0x97,0x6b,0x48,0xe7,0x09,
+0xde,0x92,0xa9,0xcc,0xf8,0x5f,0xe4,0xa3,0x1f,0xfa,0x15,0x2d,
+0x9c,0x4a,0xfe,0x1e,0xf5,0xfb,0x21,0x4f,0x5f,0x49,0x9c,0x31,
+0xad,0x47,0x5e,0x6c,0x18,0xcc,0x04,0xa7,0xbc,0x62,0x22,0x09,
+0xa2,0x87,0x4c,0x4e,0x59,0x16,0xfa,0x66,0xb8,0xbd,0x52,0xc9,
+0x75,0xb1,0xa6,0xd7,0xc6,0xf4,0x92,0x89,0x3f,0x2d,0xda,0x0b,
+0x91,0x4d,0x33,0xa8,0x27,0xca,0x17,0xfb,0x6b,0x92,0xab,0x8c,
+0x77,0xc0,0xab,0x57,0x63,0x91,0x83,0xd0,0xc0,0x0d,0xaf,0x57,
+0xf0,0x94,0x7d,0x9a,0x01,0xaf,0xd9,0xa9,0x98,0x49,0x52,0x06,
+0xe5,0x55,0x0e,0x11,0x8d,0x0c,0xb5,0x91,0x97,0x37,0xab,0xa3,
+0x80,0x98,0x5c,0x6d,0x43,0x83,0x2e,0x14,0xec,0xc1,0xc6,0x26,
+0xe5,0x0d,0x62,0x06,0xca,0x86,0xb0,0xe5,0x4b,0xf4,0xbc,0x02,
+0x95,0xc2,0x8e,0xc9,0x3c,0x5f,0xaa,0x54,0xe0,0x8c,0x78,0x17,
+0xf4,0x7a,0xe6,0x11,0x5a,0x7a,0x47,0xab,0xc5,0xf4,0x85,0x21,
+0x04,0xb6,0x0c,0xcf,0x24,0x0d,0x5c,0x48,0x81,0x13,0xab,0x08,
+0xbe,0xc5,0x2b,0xc3,0xb7,0xcf,0x42,0x9f,0x5f,0x2b,0xe6,0x3f,
+0xa9,0x46,0x39,0x57,0x6c,0xb1,0x93,0xd7,0xf1,0x51,0x85,0x2c,
+0x32,0xbc,0xba,0xd8,0x01,0x86,0x1d,0x80,0xee,0x42,0x63,0xf6,
+0xb6,0x0d,0x4c,0x90,0xdf,0xc1,0xeb,0x67,0xda,0x4f,0x52,0x11,
+0x43,0x51,0x72,0x0a,0x0f,0xe6,0x97,0x7b,0x84,0x34,0xf9,0x6d,
+0xe2,0xa2,0x26,0x92,0x53,0x45,0x69,0x91,0xb6,0x25,0x39,0x91,
+0xf5,0x14,0xe3,0xf0,0x8a,0xb8,0xf1,0x86,0xae,0xbc,0xa3,0xc4,
+0x43,0x39,0x61,0x6f,0xe7,0x5b,0x2b,0x95,0x22,0x9d,0x4a,0xed,
+0x88,0xf5,0x02,0x31,0x88,0x73,0x75,0x2a,0x49,0x4e,0x6a,0xd5,
+0x54,0x17,0xcd,0x2a,0xae,0xa3,0x72,0x2d,0xb6,0x83,0x7e,0x6d,
+0xb6,0x83,0x8a,0x75,0xd8,0x0e,0x5a,0x56,0xd4,0x23,0x8a,0xb6,
+0x0e,0x9f,0xd0,0x37,0x0d,0x66,0x5f,0xa3,0xb9,0xf9,0xa9,0x46,
+0x53,0x97,0x4c,0xeb,0x64,0xaa,0x0f,0xa5,0xff,0x97,0x98,0x2e,
+0x60,0xe3,0x94,0x0d,0x89,0x96,0x28,0xbd,0x63,0xef,0x13,0x34,
+0xe1,0xcd,0x37,0x6e,0x37,0x27,0xe6,0xe1,0x2d,0x9f,0x32,0x86,
+0x33,0xa9,0x16,0xbb,0x9b,0xd6,0xf8,0x9e,0xcd,0xba,0x19,0x68,
+0x9e,0xa6,0x17,0x7f,0x35,0xed,0x5e,0x97,0x5d,0xd9,0x29,0x01,
+0xd6,0xd7,0x10,0x93,0xc8,0xd8,0x49,0xca,0x91,0x6d,0x9f,0x2b,
+0x84,0x0d,0x32,0xa8,0x10,0xdd,0x5a,0xb4,0x07,0x2b,0x2a,0x9f,
+0x3c,0x3f,0x4d,0x2a,0xa4,0xd4,0x30,0xcf,0x15,0x13,0x73,0x30,
+0x85,0x63,0x5d,0x35,0x2a,0x17,0x91,0xa0,0x6a,0xb7,0x81,0x4a,
+0x08,0x93,0x2d,0x99,0x15,0x8a,0x8e,0x06,0xf5,0x6c,0x7d,0x9b,
+0xf1,0x8a,0x24,0x3d,0x9a,0x41,0x93,0xa0,0xba,0x44,0xc2,0x3a,
+0xe7,0x9d,0xce,0xbc,0xb7,0x04,0xc7,0x60,0x44,0x8b,0xf5,0xa1,
+0xc6,0x7c,0x32,0x67,0xe0,0xc0,0x09,0x5c,0x48,0x5b,0xc7,0xc9,
+0x9e,0xa0,0xef,0x8b,0xcb,0xee,0x39,0x91,0xdf,0x73,0xa1,0x5d,
+0xa2,0xdf,0xe5,0x9d,0x1a,0x4c,0xc7,0x9e,0x2b,0x71,0x6f,0x26,
+0xcc,0x35,0xff,0xb7,0x58,0xae,0x04,0x6a,0x72,0x3b,0xda,0x70,
+0xff,0x65,0x53,0xc8,0x26,0xc5,0xd5,0x90,0xfe,0x27,0x97,0x7f,
+0x97,0x0b,0x82,0xab,0xb2,0xfa,0x0c,0x27,0x9d,0x35,0x34,0x03,
+0x7d,0x5e,0x8f,0x91,0xdd,0x50,0x77,0x8c,0x97,0xb5,0xdd,0xde,
+0x6a,0xfb,0xeb,0xaf,0xf8,0x37,0x45,0xf9,0xda,0x2d,0xaa,0x00,
+0xaa,0xcf,0xde,0x3a,0xd6,0xf9,0xe6,0xf3,0x1c,0x04,0xaf,0x3c,
+0xad,0x46,0xcd,0x60,0x73,0x1c,0x6c,0x4e,0x83,0xf2,0xfa,0x6e,
+0x0e,0x81,0xe2,0xbf,0x50,0xff,0x47,0x00,0x00,0x00,0xff,0xff,
+0xf9,0x56,0x1c,0xc4,0x13,0x0d,0x00,0x00,
 	}))
 
 	if err != nil {

zone.go

@@ -46,6 +46,7 @@ type labels map[string]*Label
 type ZoneMetrics struct {
 	Queries     *metrics.StandardMeter
 	EdnsQueries *metrics.StandardMeter
+	LabelStats  *zoneLabelStats
 }
 
 type Zone struct {

zone_stats.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"sort"
+)
+
+type zoneLabelStats struct {
+	pos     int
+	rotated bool
+	log     []string
+	in      chan string
+	out     chan []string
+	reset   chan bool
+	close   chan bool
+}
+
+type labelStats []labelStat
+
+func (s labelStats) Len() int      { return len(s) }
+func (s labelStats) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+type labelStatsByCount struct{ labelStats }
+
+func (s labelStatsByCount) Less(i, j int) bool { return s.labelStats[i].Count > s.labelStats[j].Count }
+
+type labelStat struct {
+	Label string
+	Count int
+}
+
+func NewZoneLabelStats(size int) *zoneLabelStats {
+	zs := &zoneLabelStats{
+		log:   make([]string, size),
+		in:    make(chan string, 100),
+		out:   make(chan []string),
+		reset: make(chan bool),
+		close: make(chan bool),
+	}
+	go zs.receiver()
+	return zs
+}
+
+func (zs *zoneLabelStats) receiver() {
+
+	for {
+		select {
+		case new := <-zs.in:
+			zs.add(new)
+		case zs.out <- zs.log:
+		case <-zs.reset:
+			zs.pos = 0
+			zs.log = make([]string, len(zs.log))
+			zs.rotated = false
+		case <-zs.close:
+			close(zs.in)
+			return
+		}
+	}
+
+}
+
+func (zs *zoneLabelStats) Close() {
+	zs.close <- true
+}
+
+func (zs *zoneLabelStats) Reset() {
+	zs.reset <- true
+}
+
+func (zs *zoneLabelStats) Add(l string) {
+	zs.in <- l
+}
+
+func (zs *zoneLabelStats) add(l string) {
+	zs.log[zs.pos] = l
+	zs.pos++
+	if zs.pos+1 > len(zs.log) {
+		zs.rotated = true
+		zs.pos = 0
+	}
+}
+
+func (zs *zoneLabelStats) TopCounts(n int) labelStats {
+	cm := zs.Counts()
+	top := make(labelStats, len(cm))
+	i := 0
+	for l, c := range cm {
+		top[i] = labelStat{l, c}
+		i++
+	}
+	sort.Sort(labelStatsByCount{top})
+	if len(top) > n {
+		others := 0
+		for _, t := range top[n:] {
+			others += t.Count
+		}
+		top = append(top[:n], labelStat{"Others", others})
+	}
+	return top
+}
+
+func (zs *zoneLabelStats) Counts() map[string]int {
+	log := (<-zs.out)
+
+	counts := make(map[string]int)
+	for i, l := range log {
+		if zs.rotated == false && i >= zs.pos {
+			break
+		}
+		counts[l]++
+	}
+	return counts
+}

zone_stats_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	. "launchpad.net/gocheck"
+)
+
+type ZoneStatsSuite struct {
+}
+
+var _ = Suite(&ZoneStatsSuite{})
+
+func (s *ZoneStatsSuite) TestZoneStats(c *C) {
+	zs := NewZoneLabelStats(4)
+	c.Assert(zs, NotNil)
+	c.Log("adding 4 entries")
+	zs.Add("abc")
+	zs.Add("foo")
+	zs.Add("def")
+	zs.Add("abc")
+	c.Log("getting counts")
+	co := zs.Counts()
+	c.Check(co["abc"], Equals, 2)
+	c.Check(co["foo"], Equals, 1)
+	zs.Add("foo")
+	co = zs.Counts()
+	//	close
+	c.Check(co["abc"], Equals, 1)
+	c.Check(co["foo"], Equals, 2)
+	zs.Close()
+
+	zs = NewZoneLabelStats(10)
+	zs.Add("a")
+	zs.Add("a")
+	zs.Add("a")
+	zs.Add("b")
+	zs.Add("c")
+	zs.Add("c")
+	zs.Add("d")
+	zs.Add("d")
+	zs.Add("e")
+	zs.Add("f")
+
+	top := zs.TopCounts(2)
+	c.Check(top, HasLen, 3)
+	c.Check(top[0].Label, Equals, "a")
+
+	zs.Reset()
+	c.Check(zs.Counts(), HasLen, 0)
+
+	zs.Close()
+
+}

zones.go

@@ -36,6 +36,7 @@ func addHandler(zones Zones, name string, config *Zone) {
 		config.Metrics.EdnsQueries = metrics.NewMeter()
 		metrics.Register(config.Origin+" queries", config.Metrics.Queries)
 		metrics.Register(config.Origin+" EDNS queries", config.Metrics.EdnsQueries)
+		config.Metrics.LabelStats = NewZoneLabelStats(1000)
 	}
 
 	zones[name] = config
@@ -98,6 +99,7 @@ func zonesReadDir(dirName string, zones Zones) error {
 		log.Println("Removing zone", zone.Origin)
 		metrics.Unregister(zone.Origin + " queries")
 		metrics.Unregister(zone.Origin + " EDNS queries")
+		zone.Metrics.LabelStats.Close()
 		dns.HandleRemove(zoneName)
 		delete(zones, zoneName)
 	}