Reimplement ranking analytics to save (lots of) memory

Make production build more efficient for fe, update styling a bit

.travis.yml

@@ -11,8 +11,6 @@ before_install:
   - nvm install 6 && nvm use 6
 
 install:
-  - echo `npm -v`
-  - echo `node -v`
   - export GO15VENDOREXPERIMENT=1
   - go get github.com/rakyll/statik
   - go install github.com/rakyll/statik

dashboard/README.md

@@ -1,7 +1,20 @@
 # Dashboard
 
-The dashboard package gathers data about Guerrilla while it is running and provides an analytics web dashboard. To activate the dashboard, add it to your configuration file.
+The dashboard package gathers data about Guerrilla while it is running and provides an analytics web dashboard. To activate the dashboard, add it to your configuration file as specified in the example configuration.
+
+## Backend
+
+The backend is a Go package that collects and stores data from guerrillad, serves the dashboard to web clients, and updates clients with new analytics data over WebSockets. The backend uses [statik](https://github.com/rakyll/statik) to convert the `build` folder into a http-servable Go package. When deploying, the frontend should be built first, then the `statik` package should be created. An example of this process is in the `.travis.yml`.
 
 ## Frontend
 
-The front-end is written in React and uses WebSockets to accept data from the backend and [Victory](https://formidable.com/open-source/victory/) to render charts.
+The front-end is written in React and uses WebSockets to accept data from the backend and [Victory](https://formidable.com/open-source/victory/) to render charts. The `js` directory is an NPM module that contains all frontend code. All commands below should be run within the `js` directory.
+
+To install frontend dependencies:
+`npm install`
+
+To build the frontend code:
+`npm run build`
+
+To run the HMR development server (serves frontend on port 3000 rather than through `dashboard` package):
+`npm start`

dashboard/datastore.go

@@ -17,7 +17,8 @@ const (
 	// Number of entries to show in top N charts
 	topClientsSize = 5
 	// Frequency at which we update top client rankings
-	topClientsUpdateInterval = time.Minute * 5
+	rankingUpdateInterval = time.Hour * 6
+	nRankingBuffers       = int(maxWindow / rankingUpdateInterval)
 	// Redux action type names
 	initMessageType = "INIT"
 	tickMessageType = "TICK"
@@ -32,7 +33,6 @@ var (
 // Keeps track of connection data that is buffered in the topClients
 // so the data can be removed after `maxWindow` interval has occurred.
 type conn struct {
-	addedTime        int64
 	helo, domain, ip string
 }
 
@@ -43,8 +43,10 @@ type dataStore struct {
 	// List of samples of number of connected clients
 	nClientTicks []point
 	// Up-to-date number of clients
-	nClients uint64
-	topClients
+	nClients  uint64
+	topDomain bufferedRanking
+	topHelo   bufferedRanking
+	topIP     bufferedRanking
 	// For notifying the store about new connections
 	newConns chan conn
 	subs     map[string]chan<- *message
@@ -56,56 +58,83 @@ func newDataStore() *dataStore {
 	ds := &dataStore{
 		ramTicks:     make([]point, 0, maxTicks),
 		nClientTicks: make([]point, 0, maxTicks),
-		topClients: topClients{
-			TopDomain: make(map[string]int),
-			TopHelo:   make(map[string]int),
-			TopIP:     make(map[string]int),
-		},
-		newConns: newConns,
-		subs:     subs,
+		topDomain:    newBufferedRanking(nRankingBuffers),
+		topHelo:      newBufferedRanking(nRankingBuffers),
+		topIP:        newBufferedRanking(nRankingBuffers),
+		newConns:     newConns,
+		subs:         subs,
 	}
-	go ds.topClientsManager()
+	go ds.rankingManager()
 	return ds
 }
 
-// Manages the list of top clients by domain, helo, and IP by incrementing
-// records upon a new connection and scheduling a decrement after the `maxWindow`
-// interval has passed.
-func (ds *dataStore) topClientsManager() {
-	bufferedConns := []conn{}
-	ticker := time.NewTicker(topClientsUpdateInterval)
+// Keeps track of top domain/helo/ip rankings, but buffered into multiple
+// maps so that old records can be efficiently kept track of and quickly removed
+type bufferedRanking []map[string]int
+
+func newBufferedRanking(nBuffers int) bufferedRanking {
+	br := make([]map[string]int, nBuffers)
+	for i := 0; i < nBuffers; i++ {
+		br[i] = make(map[string]int)
+	}
+	return br
+}
+
+// Manages the list of top clients by domain, helo, and IP by updating buffered
+// record maps. At each `rankingUpdateInterval` we shift the maps and remove the
+// oldest, so rankings are always at most as old as `maxWindow`
+func (ds *dataStore) rankingManager() {
+	ticker := time.NewTicker(rankingUpdateInterval)
 
 	for {
 		select {
 		case c := <-ds.newConns:
-			bufferedConns = append(bufferedConns, c)
-
 			ds.lock.Lock()
-			ds.TopDomain[c.domain]++
-			ds.TopHelo[c.helo]++
-			ds.TopIP[c.ip]++
+			ds.topDomain[0][c.domain]++
+			ds.topHelo[0][c.helo]++
+			ds.topIP[0][c.ip]++
 			ds.lock.Unlock()
 
 		case <-ticker.C:
-			cutoff := time.Now().Add(-maxWindow).Unix()
-			cutoffI := 0
-
 			ds.lock.Lock()
-			for i, bc := range bufferedConns {
-				// We make an assumption here that conns come in in-order, which probably
-				// isn't exactly true, but close enough to not make much of a difference
-				if bc.addedTime > cutoff {
-					cutoffI = i
-					break
-				}
-				ds.TopDomain[bc.domain]--
-				ds.TopHelo[bc.helo]--
-				ds.TopIP[bc.ip]--
-			}
+			// Add empty map at index 0 and shift other maps one down
+			ds.topDomain = append(
+				[]map[string]int{map[string]int{}},
+				ds.topDomain[:len(ds.topDomain)-1]...)
+			ds.topHelo = append(
+				[]map[string]int{map[string]int{}},
+				ds.topHelo[:len(ds.topHelo)-1]...)
+			ds.topIP = append(
+				[]map[string]int{map[string]int{}},
+				ds.topHelo[:len(ds.topIP)-1]...)
 			ds.lock.Unlock()
+		}
+	}
+}
 
-			bufferedConns = bufferedConns[cutoffI:]
+// Aggregates the rankings from the ranking buffer into a single map
+// for each of domain, helo, ip. This is what we send to the frontend.
+func (ds *dataStore) aggregateRankings() ranking {
+	topDomain := make(map[string]int, len(ds.topDomain[0]))
+	topHelo := make(map[string]int, len(ds.topHelo[0]))
+	topIP := make(map[string]int, len(ds.topIP[0]))
+
+	for i := 0; i < nRankingBuffers; i++ {
+		for domain, count := range ds.topDomain[i] {
+			topDomain[domain] += count
+		}
+		for helo, count := range ds.topHelo[i] {
+			topHelo[helo] += count
 		}
+		for ip, count := range ds.topIP[i] {
+			topIP[ip] += count
+		}
+	}
+
+	return ranking{
+		TopDomain: topDomain,
+		TopHelo:   topHelo,
+		TopIP:     topIP,
 	}
 }
 
@@ -179,15 +208,15 @@ func dataListener(interval time.Duration) {
 		store.addRAMPoint(ramPoint)
 		store.addNClientPoint(nClientPoint)
 		store.notify(&message{tickMessageType, dataFrame{
-			Ram:        ramPoint,
-			NClients:   nClientPoint,
-			topClients: store.topClients,
+			Ram:      ramPoint,
+			NClients: nClientPoint,
+			ranking:  store.aggregateRankings(),
 		}})
 	}
 }
 
 // Keeps track of top clients by helo, ip, and domain
-type topClients struct {
+type ranking struct {
 	TopHelo   map[string]int `json:"topHelo"`
 	TopIP     map[string]int `json:"topIP"`
 	TopDomain map[string]int `json:"topDomain"`
@@ -196,13 +225,13 @@ type topClients struct {
 type dataFrame struct {
 	Ram      point `json:"ram"`
 	NClients point `json:"nClients"`
-	topClients
+	ranking
 }
 
 type initFrame struct {
 	Ram      []point `json:"ram"`
 	NClients []point `json:"nClients"`
-	topClients
+	ranking
 }
 
 // Format of messages to be sent over WebSocket
@@ -247,10 +276,9 @@ func (h logHook) Fire(e *log.Entry) error {
 		store.lock.Unlock()
 	case "mailfrom":
 		store.newConns <- conn{
-			addedTime: time.Now().Unix(),
-			domain:    domain,
-			helo:      helo,
-			ip:        ip,
+			domain: domain,
+			helo:   helo,
+			ip:     ip,
 		}
 	case "disconnect":
 		store.lock.Lock()

dashboard/exe/main.go

@@ -1,11 +0,0 @@
-package main
-
-import (
-	"github.com/flashmob/go-guerrilla/dashboard"
-)
-
-func main() {
-	dashboard.Run(&dashboard.Config{
-		ListenInterface: ":8080",
-	})
-}

dashboard/js/src/components/App.js

@@ -1,7 +1,5 @@
 import React, { Component } from 'react';
 import {connect} from 'react-redux';
-// import {init, tick} from '../action-creators';
-// import ActionTypes from '../action-types';
 import LineChart from './LineChart';
 import RankingTable from './RankingTable';
 
@@ -25,8 +23,12 @@ const styles = {
 	}
 }
 
-// TODO make this not hard-coded
-const WS_URL = 'ws://localhost:8080/ws';
+let WS_URL = `ws://${location.host}/ws`
+if (process.env.NODE_ENV === 'development') {
+	WS_URL = `ws://localhost:8080/ws`
+}
+
+// Maximum size of ranking tables
 const RANKING_SIZE = 5;
 
 const _computeRanking = mapping => {
@@ -38,14 +40,13 @@ const _computeRanking = mapping => {
 
 class App extends Component {
 	constructor(props) {
-		super();
+		super(props);
 		const ws = new WebSocket(WS_URL);
 		ws.onerror = err => console.log(err);
 		ws.onopen = event => console.log(event);
 		ws.onclose = event => console.log(event);
 		ws.onmessage = event => {
 			const message = JSON.parse(event.data);
-			console.log(message);
 			props.dispatch(message);
 		};
 

dashboard/js/src/components/LineChart.js

@@ -6,7 +6,6 @@ import simplify from 'simplify-js';
 import theme from './../theme';
 
 const _formatIndependentAxis = tick => {
-	console.log("_formatIndependentAxis", tick, '$$$$', moment(tick).format('HH:mm:ss'));
 	return moment(tick).format('HH:mm:ss');
 };
 
@@ -17,11 +16,12 @@ const _formatDependentAxis = (tick, format) => (
 );
 
 // Uses simplifyJS to simplify the data from the backend (there can be up to
-// 8000 points so this step is necessary)
+// 8000 points so this step is necessary). Because of the format expectations
+// of simplifyJS, we need to convert x's to ints and back to moments.
 const _simplify = data => {
 	if (data.length === 0) return [];
 	return simplify(
-		data.map(d => ({x: moment(d.x).unix(), y: d.y}))
+		data.map(d => ({x: moment(d.x).valueOf(), y: d.y}))
 	).map(d => ({x: moment(d.x), y: d.y}));
 }
 
@@ -40,8 +40,8 @@ const LineChart = ({data, format, title}) => {
 			<h1 style={styles.title}>{title}</h1>
 			<VictoryChart
 				theme={theme}
-				height={150}
-				width={1000}>
+				height={200}
+				width={1500}>
 				<VictoryAxis
 					scale="time"
 					tickCount={4}

dashboard/js/src/components/RankingTable.js

@@ -45,23 +45,27 @@ const RankingTable = ({ranking, rankType}) => {
 		<div>
 			<table style={styles.table}>
 				<caption>{`Top Clients by ${rankType}`}</caption>
-				<tr style={styles.row}>
-					<th style={{...styles.header, ...styles.rank}}>Rank</th>
-					<th style={{...styles.header, ...styles.rankType}}>{rankType}</th>
-					<th style={{...styles.header, ...styles.count}}># Clients</th>
-				</tr>
-				{
-					ranking.map((record, i) => (
-						<tr style={Object.assign({},
-							styles.row,
-							i % 2 === 0 && styles.odd
-						)} key={record.value}>
-							<td style={styles.cell}>{i + 1}</td>
-							<td style={styles.cell}>{record.value}</td>
-							<td style={styles.cell}>{record.count}</td>
-						</tr>
-					))
-				}
+				<thead>
+					<tr style={styles.row}>
+						<th style={{...styles.header, ...styles.rank}}>Rank</th>
+						<th style={{...styles.header, ...styles.rankType}}>{rankType}</th>
+						<th style={{...styles.header, ...styles.count}}># Clients</th>
+					</tr>
+				</thead>
+				<tbody>
+					{
+						ranking.map((record, i) => (
+							<tr style={Object.assign({},
+								styles.row,
+								i % 2 === 0 && styles.odd
+							)} key={record.value}>
+								<td style={styles.cell}>{i + 1}</td>
+								<td style={styles.cell}>{record.value}</td>
+								<td style={styles.cell}>{record.count}</td>
+							</tr>
+						))
+					}
+				</tbody>
 			</table>
 		</div>
 	)

dashboard/js/src/index.js

@@ -2,19 +2,21 @@ import React from 'react';
 import ReactDOM from 'react-dom';
 import App from './components/App';
 import reducer from './reducer';
-import {applyMiddleware, createStore} from 'redux';
-import {Provider} from 'react-redux';
+import { applyMiddleware, createStore } from 'redux';
+import { Provider } from 'react-redux';
 import createLogger from 'redux-logger';
 import './index.css';
 
-const logger = createLogger({
-	stateTransformer: state => state.toJS()
-});
+let store = createStore(reducer);
+
+if (process.env.NODE_ENV === 'development') {
+	store = createStore(
+		reducer, applyMiddleware(createLogger({
+			stateTransformer: state => state.toJS()
+		})
+	));
+}
 
-const store = createStore(
-	reducer,
-	applyMiddleware(logger)
-);
 
 ReactDOM.render(
 	<Provider store={store}>

dashboard/js/src/theme.js

@@ -71,7 +71,7 @@ const theme = {
 			}),
 			grid: {
 				fill: "transparent",
-				stroke: "transparent"
+				stroke: "#f0f0f0"
 			},
 			ticks: {
 				fill: "transparent",
@@ -124,7 +124,7 @@ const theme = {
 		style: {
 			data: {
 				fill: "transparent",
-				stroke: charcoal,
+				stroke: "#969696",
 				strokeWidth: 2
 			},
 			labels: assign({}, baseLabelStyles, {

mocks/client_mock.go

@@ -7,15 +7,6 @@ import (
 	"time"
 )
 
-const (
-	URL = "127.0.0.1:2500"
-)
-
-var (
-	helos = []string{"hi", "hello", "ahoy", "bonjour", "hey!"}
-	froms = []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"}
-)
-
 func lastWords(message string, err error) {
 	fmt.Println(message, err.Error())
 	return
@@ -23,8 +14,8 @@ func lastWords(message string, err error) {
 }
 
 type Client struct {
-	helo         string
-	emailAddress string
+	Helo         string
+	EmailAddress string
 }
 
 func (c *Client) SendMail(to, url string) {
@@ -38,11 +29,11 @@ func (c *Client) SendMail(to, url string) {
 	// Introduce some artificial delay
 	time.Sleep(time.Millisecond * (time.Duration(rand.Int() % 50)))
 
-	if err = sc.Hello(c.helo); err != nil {
+	if err = sc.Hello(c.Helo); err != nil {
 		lastWords("Hello ", err)
 	}
 
-	if err = sc.Mail(c.emailAddress); err != nil {
+	if err = sc.Mail(c.EmailAddress); err != nil {
 		lastWords("Mail ", err)
 	}
 
@@ -60,7 +51,7 @@ func (c *Client) SendMail(to, url string) {
 	defer wr.Close()
 
 	msg := fmt.Sprint("Subject: something\n")
-	msg += "From: " + c.emailAddress + "\n"
+	msg += "From: " + c.EmailAddress + "\n"
 	msg += "To: " + to + "\n"
 	msg += "\n\n"
 	msg += "hello\n"

mocks/cmd/client.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/flashmob/go-guerrilla/mocks"
+)
+
+const (
+	URL = "127.0.0.1:2500"
+)
+
+var (
+	helos  = []string{"hi", "hello", "ahoy", "bonjour", "hey!", "whats up"}
+	emails = []string{
+		"[email protected]",
+		"[email protected]",
+		"[email protected]",
+		"[email protected]",
+		"[email protected]",
+		"[email protected]",
+		"[email protected]",
+		"[email protected]",
+		"[email protected]",
+		"[email protected]",
+	}
+)
+
+func main() {
+	c := make(chan int)
+	for i := 0; i < 100; i++ {
+		go sendMailForever(time.Millisecond * time.Duration(rand.Int()%500))
+	}
+	<-c
+}
+
+func sendMailForever(wait time.Duration) {
+	c := mocks.Client{
+		Helo:         helos[rand.Int()%len(helos)],
+		EmailAddress: emails[rand.Int()%len(emails)],
+	}
+	fmt.Println(c)
+
+	for {
+		c.SendMail("[email protected]", URL)
+		time.Sleep(wait)
+	}
+}