pike: Import pike.top rpc call from modules_s/pike

Thanks to Ovidiu for pointers in the right direction.

modules_k/pike/README

@@ -36,7 +36,11 @@ Bogdan-Andrei Iancu
 
               5.1. pike_list
 
-   2. Developer Guide
+   2. RPC calls
+
+        1. pike.top
+
+   3. Developer Guide
 
    List of Examples
 
@@ -45,7 +49,7 @@ Bogdan-Andrei Iancu
    1.3. Set remove_latency parameter
    1.4. Set pike_log_level parameter
    1.5. pike_check_req usage
-   2.1. Tree of IP addresses
+   3.1. Tree of IP addresses
 
 Chapter 1. Admin Guide
 
@@ -170,7 +174,7 @@ modparam("pike", "pike_log_level", -1)
 
    4.1. pike_check_req()
 
-4.1. pike_check_req()
+4.1.  pike_check_req()
 
    Process the source IP of the current request and returns false if the
    IP was exceeding the blocking limit.
@@ -196,7 +200,7 @@ if (!pike_check_req()) { exit; };
 
    5.1. pike_list
 
-5.1. pike_list
+5.1.  pike_list
 
    Lists the nodes in the pike tree.
 
@@ -208,12 +212,38 @@ if (!pike_check_req()) { exit; };
                 :pike_list:_reply_fifo_file_
                 _empty_line_
 
-Chapter 2. Developer Guide
+Chapter 2. RPC calls
+
+   Table of Contents
+
+   1. pike.top
+
+1.  pike.top
+
+   Pike.top behaves like a 'top' command and shows source IP addresses of
+   incoming requestes to pike_check_req() function.
+
+   The IP list is sorted by sum of leaf hits (prev and curr) descending
+   and in second level by hits.
+
+   Some IPs could be marked as HOT depending on theirs request rates.
+
+   pike.top command takes one string parameter which specifies what kind
+   of nodes you are interested in. Possible values are HOT or ALL. If no
+   argument is given, it behaves as HOT was used.
+
+   Marking nodes HOT is done on server side, client only presents given
+   data and make some postprocessing like sorting.
+
+   Output of this command is a simple dump of ip_tree nodes marked as
+   ip-leafs.
+
+Chapter 3. Developer Guide
 
    One single tree (for both IPv4 and IPv6) is used. Each node contains a
    byte, the IP addresses stretching from root to the leafs.
 
-   Example 2.1. Tree of IP addresses
+   Example 3.1. Tree of IP addresses
            / 193 - 175 - 132 - 164
 tree root /                  \ 142
           \ 195 - 37 - 78 - 163

modules_k/pike/doc/pike.xml

@@ -38,6 +38,6 @@
 	
 	<xi:include href="pike_admin.xml"/>
 	<xi:include href="pike_devel.xml"/>
-	
+	<xi:include href="pike_rpc.xml"/>
 	
 </book>

modules_k/pike/doc/pike_rpc.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN" 
+   "http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
+
+<section id="pike.rpc" xmlns:xi="http://www.w3.org/2001/XInclude">
+	<sectioninfo>
+	</sectioninfo>
+
+	<title>RPC calls</title>
+
+	<section id="pike.top">
+		<title>
+			<function>pike.top</function>
+		</title>
+		<simpara>
+			Pike.top behaves like a 'top' command and shows source IP addresses of incoming requestes to pike_check_req() function.
+		</simpara>
+		<simpara>
+			The IP list is sorted by sum of leaf hits (prev and curr) descending and in second level by hits.
+		</simpara>
+		<simpara>
+			Some IPs could be marked as HOT depending on theirs request rates.
+		</simpara>
+		<simpara>
+			pike.top command takes one string parameter which specifies what kind of nodes you are interested in. Possible values
+			are HOT or ALL. If no argument is given, it behaves as HOT was used.
+		</simpara>
+		<simpara>
+			Marking nodes HOT is done on server side, client only presents given data and make some postprocessing like sorting.
+		</simpara>
+		<simpara>
+			Output of this command is a simple dump of ip_tree nodes marked as ip-leafs.
+		</simpara>
+	</section>
+</section>

modules_k/pike/ip_tree.c

@@ -148,7 +148,7 @@ error:
 	return -1;
 }
 
-
+unsigned int get_max_hits() { return root != 0 ? root->max_hits : -1; } 
 
 /* destroy an ip_node and all nodes under it; the nodes must be first removed
  * from any other lists/timers */
@@ -263,6 +263,20 @@ int is_node_hot_leaf(struct ip_node *node)
 	return is_hot_leaf(node);
 }
 
+/*! \brief Used by the rpc function */
+char *node_status_array[] = {"", "WARM", "HOT", "ALL"};
+node_status_t node_status(struct ip_node *node)
+{
+        if ( is_hot_leaf(node) )
+                return NODE_STATUS_HOT;
+
+        if ( is_warm_leaf(node) )
+                return NODE_STATUS_WARM;
+
+        return NODE_STATUS_OK;
+}
+
+
 
 /* mark with one more hit the given IP address - */
 struct ip_node* mark_node(unsigned char *ip,int ip_len,
@@ -369,3 +383,44 @@ void remove_node(struct ip_node *node)
 	destroy_ip_node(node);
 }
 
+static void print_node(struct ip_node *node,int sp, FILE *f)
+{
+	struct ip_node *foo;
+
+	/* print current node */
+	if (!f) {
+		DBG("[l%d] node %p; brh=%d byte=%d flags=%d, hits={%d,%d} , "
+			"leaf_hits={%d,%d]\n",
+			sp, node, node->branch, node->byte, node->flags,
+			node->hits[PREV_POS],node->hits[CURR_POS],
+			node->leaf_hits[PREV_POS],node->leaf_hits[CURR_POS]);
+	} else {
+		fprintf(f,"[l%d] node %p; brh=%d byte=%d flags=%d, hits={%d,%d} , "
+			"leaf_hits={%d,%d]\n",
+			sp, node, node->branch, node->byte, node->flags,
+			node->hits[PREV_POS],node->hits[CURR_POS],
+			node->leaf_hits[PREV_POS],node->leaf_hits[CURR_POS]);
+	}
+
+	/* print all the kids */
+	foo = node->kids;
+	while(foo){
+		print_node(foo,sp+1,f);
+		foo = foo->next;
+	}
+}
+
+void print_tree(  FILE *f )
+{
+	int i;
+
+	DBG("DEBUG:pike:print_tree: printing IP tree\n");
+	for(i=0;i<MAX_IP_BRANCHES;i++) {
+		if (prv_get_tree_branch(i)==0)
+			continue;
+		prv_lock_tree_branch(i);
+		if (prv_get_tree_branch(i))
+			print_node( prv_get_tree_branch(i), 0, f);
+		prv_unlock_tree_branch(i);
+	}
+}

modules_k/pike/ip_tree.h

@@ -96,6 +96,17 @@ void lock_tree_branch(unsigned char b);
 void unlock_tree_branch(unsigned char b);
 struct ip_node* get_tree_branch(unsigned char b);
 
+typedef enum {
+        NODE_STATUS_OK    = 0,
+        NODE_STATUS_WARM  = 1,
+        NODE_STATUS_HOT   = 2,
+        NODE_STATUS_ALL   = 3   /** used for status matching */
+} node_status_t;
+node_status_t node_status(struct ip_node *node);
+extern char *node_status_array[];
+unsigned int get_max_hits();
+
+void print_tree( FILE *f);
 
 
 #endif

modules_k/pike/pike.c

@@ -49,6 +49,8 @@
 #include "timer.h"
 #include "pike_mi.h"
 #include "pike_funcs.h"
+#include "../../rpc_lookup.h"
+#include "pike_rpc.h"
 
 MODULE_VERSION
 
@@ -110,11 +112,17 @@ struct module_exports exports= {
 
 static int pike_init(void)
 {
+	LOG(L_INFO, "PIKE - initializing\n");
+
 	if(register_mi_mod(exports.name, mi_cmds)!=0)
 	{
 		LM_ERR("failed to register MI commands\n");
 		return -1;
 	}
+	if (rpc_register_array(pike_rpc_methods)!=0) {
+		LM_ERR("failed to register RPC commands\n");
+		return -1;
+	}
 
 	/* alloc the timer lock */
 	timer_lock=lock_alloc();

modules_k/pike/pike_rpc.c

@@ -0,0 +1,193 @@
+#include "ip_tree.h"
+#include "../../rpc_lookup.h"
+/*??? #include "rpc.h" */
+/*??? #include "top.h" */
+#include "../../timer.h"	/* ticks_t */	
+
+#include "../../dprint.h"
+#include "pike_top.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <assert.h>
+// TODO FIXME LCH remove arpa/inet.h after testing
+#include <arpa/inet.h>
+
+// IPv6 address is a 16 bytes long
+#define MAX_DEPTH 16
+
+static unsigned int g_max_hits = 0;
+
+static void traverse_subtree( struct ip_node *node, int depth, int options )
+{
+	static unsigned char ip_addr[MAX_DEPTH];
+
+	struct ip_node *foo;
+	
+	DBG("pike:rpc traverse_subtree, depth: %d, byte: %d", depth, node->byte);
+
+	assert( depth < MAX_DEPTH );
+	
+	ip_addr[depth] = node->byte;
+
+	if ( node->flags & NODE_IPLEAF_FLAG ) {
+		int ns = node_status(node);
+		DBG("pike:traverse_subtree: options: 0x%02x, node status: 0x%02x", options, ns);
+		/* add to the result list if it has requested status */
+		switch (options) {
+			case NODE_STATUS_HOT:
+						if ( ns & NODE_STATUS_HOT )
+							pike_top_add_entry(ip_addr, depth+1, node->leaf_hits, node->hits, node->expires - get_ticks(), ns);
+						break;
+			case NODE_STATUS_ALL:
+						pike_top_add_entry(ip_addr, depth+1, node->leaf_hits, node->hits, node->expires - get_ticks(), ns);
+						break;
+		}
+	}
+	else if (! node->kids) {	/* TODO non IP leaf of ip_tree - it is possible to report WARM nodes here */
+/*		if ( options == ns )
+			pike_top_add_entry(ip_addr, depth+1, node->leaf_hits, node->hits, node->expires - get_ticks(), ns);
+*/	}
+	else {	/* not a any kind of leaf - inner node */
+		DBG("pike:rpc traverse_subtree, not IP leaf, depth: %d, ip: %d.%d.%d.%d   hits[%d,%d], expires: %d",
+			depth, ip_addr[0], ip_addr[1], ip_addr[2], ip_addr[3], node->hits[0], node->hits[1], node->expires - get_ticks());
+	}
+	
+	foo = node->kids;
+	while (foo) {
+		traverse_subtree( foo, depth + 1, options );
+		foo = foo->next;
+	}
+}
+
+static void collect_data(int options)
+{
+	int i;
+
+	g_max_hits = get_max_hits();
+
+	DBG("pike: collect_data");
+	
+	// maybe try_lock first and than do the rest?
+	for(i=0;i<MAX_IP_BRANCHES;i++) {
+		if (get_tree_branch(i)==0)
+			continue;
+		DBG("pike: collect_data: branch %d", i);
+		lock_tree_branch(i);
+		if (get_tree_branch(i))
+			traverse_subtree( get_tree_branch(i), 0, options );
+		unlock_tree_branch(i);
+    }
+}
+
+/* do not use static buffer with this function */
+static const char *concat_err = "ERROR while concatenating string";
+static char *concat(char *buff, size_t buffsize, const char *first, int second)
+{
+	int rv;
+	size_t size;
+	
+	while ( (rv = snprintf(buff, buffsize, "%s%d", first, second)) >= buffsize ) {
+		size = rv > 128 ? rv : 128;
+		buff = (char *)realloc(buff, size);
+		if ( buff == 0 )
+			return (char*)concat_err;
+		buffsize = size;
+		DBG("pike:rpc:concat: new buffer size for %s: %d", first,
+				(int)buffsize);
+	}
+	return buff;
+}
+
+static void pike_top(rpc_t *rpc, void *c)
+{
+	int i;
+	void *handle;
+	struct TopListItem_t *top_list_root;
+	struct TopListItem_t *ti = 0;
+	char addr_buff[40];
+	char *ip_addr = 0;
+	char *leaf_hits_prev = 0;
+	char *leaf_hits_curr = 0;
+	char *expires = 0;
+	char *status = 0;
+	size_t ip_addr_size = 0;
+	size_t leaf_hits_prev_size = 0;
+	size_t leaf_hits_curr_size = 0;
+	size_t expires_size = 0;
+	size_t status_size = 0;
+	char *stropts;
+	int   options = 0;
+
+	DBG("pike: top");
+	
+	/* obtain params */
+	if (rpc->scan(c, "s", &stropts) <= 0)
+		stropts = "HOT";
+
+	DBG("pike:top: string options: '%s'", stropts);
+	if ( strstr(stropts, "ALL") ) { 
+		options = NODE_STATUS_ALL;
+	} else if ( strstr(stropts, "HOT") ) {
+		options |= NODE_STATUS_HOT;
+	} else if ( strstr(stropts, "WARM") ) {
+		options |= NODE_STATUS_WARM;
+	}
+	DBG("pike:top: options: 0x%02x\n", options);
+	
+	
+	print_tree( 0 );
+	
+	collect_data(options);
+	top_list_root = pike_top_get_root();
+	DBG("pike_top: top_list_root = %p", top_list_root);
+	
+	rpc->add(c, "{", &handle);
+	rpc->struct_add(handle, "d", "max_hits", get_max_hits());
+	i = 0; // it is passed as number of rows
+	if ( top_list_root == 0 ) {
+		DBG("pike_top: no data");
+	}
+	else {
+		for( ti = top_list_root, i = 0; ti != 0; ti = ti->next, ++i ) {
+			pike_top_print_addr(ti->ip_addr, ti->addr_len, addr_buff, sizeof(addr_buff));
+			DBG("pike:top: result[%d]: %s leaf_hits[%d,%d] hits[%d,%d] expires: %d status: 0x%02x",
+					i, addr_buff, ti->leaf_hits[0], ti->leaf_hits[1],
+					ti->hits[0], ti->hits[1], ti->expires, ti->status); 
+			rpc->struct_add(handle, "sddds",
+							concat(ip_addr, ip_addr_size, "ip_addr", i), addr_buff,
+							concat(leaf_hits_prev, leaf_hits_prev_size, "leaf_hits_prev", i), ti->leaf_hits[0],
+							concat(leaf_hits_curr, leaf_hits_curr_size, "leaf_hits_curr", i), ti->leaf_hits[1],
+							concat(expires, expires_size, "expires", i), ti->expires,
+							concat(status, status_size, "status", i), node_status_array[ti->status]);
+		}
+	}
+	rpc->struct_add(handle, "d", "number_of_rows", i);
+	/* free buffers */
+	free(ip_addr);
+	free(leaf_hits_prev);
+	free(leaf_hits_curr);
+	free(expires);
+	free(status);
+	pike_top_list_clear();
+	
+	rpc->send(c);
+}
+
+/* ----- exported data structure with methods ----- */
+
+// TODO check documentation
+static const char* pike_top_doc[] = {
+	"pike.top doc.",  /* Documentation string */
+	0                 /* Method signature(s) */
+};
+
+/* 
+ * RPC Methods exported by this module 
+ */
+
+rpc_export_t pike_rpc_methods[] = {
+	{"pike.top",   pike_top,     pike_top_doc, 0},
+	{0, 0, 0, 0}
+};
+

modules_k/pike/pike_rpc.h

@@ -0,0 +1,9 @@
+#ifndef __PIKE_RPC_H
+#define __PIKE_RPC_H
+
+#include "../../rpc.h"
+#include "../../rpc_lookup.h"
+extern rpc_export_t pike_rpc_methods[];
+
+
+#endif

modules_k/pike/pike_top.c

@@ -0,0 +1,91 @@
+#include "pike_top.h"
+
+#include "../../dprint.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <arpa/inet.h>
+#include "ip_tree.h"
+
+
+static struct TopListItem_t *top_list_root = 0;
+static struct TopListItem_t *top_list_iter = 0;
+
+static char buff[128];
+
+struct TopListItem_t *pike_top_get_root() { return top_list_root; }
+
+char *pike_top_print_addr( unsigned char *ip, int iplen, char *buff, int buffsize )
+{
+	unsigned short *ipv6_ptr = (unsigned short *)ip;
+	memset( buff, 0, sizeof(buff));
+	
+	DBG("pike:top:print_addr(iplen: %d, buffsize: %d)", iplen, buffsize);
+	
+	if ( iplen == 4 ) {
+		inet_ntop(AF_INET, ip, buff, buffsize);
+	}
+	else if ( iplen == 16 ) {
+		inet_ntop(AF_INET6, ip, buff, buffsize);
+	}
+	else {
+		sprintf( buff, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x",
+				 htons(ipv6_ptr[0]), htons(ipv6_ptr[1]), htons(ipv6_ptr[2]), htons(ipv6_ptr[3]),
+				 htons(ipv6_ptr[4]), htons(ipv6_ptr[5]), htons(ipv6_ptr[6]), htons(ipv6_ptr[7]) );
+	}
+	
+	return buff;
+}
+
+/* if you do not need global buffer, you can use this simpler call */
+static char *print_addr(unsigned char *ip, int iplen)
+{
+	return pike_top_print_addr(ip, iplen, buff, sizeof(buff));
+}
+
+int pike_top_add_entry( unsigned char *ip_addr, int addr_len, unsigned int leaf_hits[2], unsigned int hits[2], unsigned int expires, node_status_t status )
+{
+	struct TopListItem_t *new_item = (struct TopListItem_t *)malloc(sizeof(struct TopListItem_t));
+	
+	print_addr(ip_addr, addr_len);
+	DBG("pike_top_add_enrty(ip: %s, leaf_hits[%d,%d], hits[%d,%d],"
+			" expires: %d, status: %d)",
+			buff, leaf_hits[0], leaf_hits[1], hits[0], hits[1],
+			expires, status);
+	assert(new_item != 0);
+	
+	memset( (void *)new_item, 0, sizeof(struct TopListItem_t) );
+	
+	new_item->status  = status;
+	new_item->expires = expires;
+	new_item->hits[0] = hits[0];
+	new_item->hits[1] = hits[1];
+	new_item->leaf_hits[0] = leaf_hits[0];
+	new_item->leaf_hits[1] = leaf_hits[1];
+	
+	assert( addr_len <= 16 );
+	
+	new_item->addr_len = addr_len;
+	memcpy(new_item->ip_addr, ip_addr, addr_len);
+
+	new_item->next = top_list_root;
+	top_list_root  = new_item;
+
+	return 1;
+}
+
+void pike_top_list_clear()
+{
+	struct TopListItem_t *ptr;
+	
+	top_list_iter = top_list_root;
+	while (top_list_iter) {
+		ptr = top_list_iter->next;
+		free(top_list_iter);
+		top_list_iter = ptr;
+	}
+	top_list_root = 0;
+	memset(buff, 0, sizeof(buff));
+}

modules_k/pike/pike_top.h

@@ -0,0 +1,28 @@
+#ifndef __PIKE_TOP_H
+#define __PIKE_TOP_H
+
+#include "ip_tree.h"
+#include "../../ip_addr.h"
+
+struct TopListItem_t {
+	int             addr_len;
+	unsigned char   ip_addr[45];	/*!< Make room for IPv6 */
+	unsigned int  	leaf_hits[2];
+	unsigned int  	hits[2];
+	unsigned int    expires;	/*!< in seconds */
+	node_status_t   status;
+
+	struct TopListItem_t *next;
+};
+
+// returns 1 when OK and 0 when failed
+int pike_top_add_entry( unsigned char *ip_addr, int iplen, unsigned int leaf_hits[2], unsigned int hits[2], unsigned int expires, node_status_t status );
+
+struct TopListItem_t *pike_top_get_root();
+void pike_top_list_clear();
+
+/* helpful functions */
+char *pike_top_print_addr( unsigned char *ip_addr, int addrlen, char *buff, int buffsize );
+
+
+#endif // PIKE_TOP_H