- added callbacks/hooks for dst_blacklist_add: one can register such

a callback and deny adding a destination to the blacklist list, or
 it can change the destinations flags.

- autodetect if locks or lock_set are not limited and if so use
 one lock per hash bucket or a lock_set of hash table size (should work
  better on multiple cpus, less cacheline invalidations)

dst_blacklist.c

@@ -29,6 +29,7 @@
 /* History:
  * --------
  *  2006-07-29  created by andrei
+ *  2007-05-39  added hooks for add; more locks to reduce contention (andrei)
  */
 
 
@@ -43,6 +44,7 @@
 #include "ip_addr.h"
 #include "error.h"
 #include "rpc.h"
+#include "compiler_opt.h"
 
 
 
@@ -66,7 +68,54 @@ struct dst_blst_entry{
 #define DEFAULT_BLST_TIMER_INTERVAL		60 /* 1 min */
 
 
+/* lock method */
+#ifdef GEN_LOCK_T_UNLIMITED
+#define BLST_LOCK_PER_BUCKET
+#elif defined GEN_LOCK_SET_T_UNLIMITED
+#define BLST_LOCK_SET
+#else
+#define BLST_ONE_LOCK
+#endif
+
+
+#ifdef BLST_LOCK_PER_BUCKET
+/* lock included in the hash bucket */
+#define LOCK_BLST(h)		lock_get(&dst_blst_hash[(h)].lock)
+#define UNLOCK_BLST(h)		lock_release(&dst_blst_hash[(h)].lock)
+#elif defined BLST_LOCK_SET
+static gen_lock_set_t* blst_lock_set=0;
+#define LOCK_BLST(h)		lock_set_get(blst_lock_set, (h))
+#define UNLOCK_BLST(h)		lock_set_release(blst_lock_set, (h))
+#else
+/* use only one lock */
 static gen_lock_t* blst_lock=0;
+#define LOCK_BLST(h)		lock_get(blst_lock)
+#define UNLOCK_BLST(h)		lock_release(blst_lock)
+#endif
+
+
+
+
+#define BLST_HASH_STATS
+
+#ifdef BLST_HASH_STATS
+#define BLST_HASH_STATS_DEC(h) dst_blst_hash[(h)].entries--
+#define BLST_HASH_STATS_INC(h) dst_blst_hash[(h)].entries++
+#else
+#define BLST_HASH_STATS_DEC(h) do{}while(0)
+#define BLST_HASH_STATS_INC(h) do{}while(0)
+#endif
+
+struct dst_blst_lst_head{
+	struct dst_blst_entry* first;
+#ifdef BLST_LOCK_PER_BUCKET
+	gen_lock_t	lock;
+#endif
+#ifdef BLST_HASH_STATS
+	unsigned int entries;
+#endif
+};
+
 static struct timer_ln* blst_timer_h=0;
 
 static volatile unsigned int* blst_mem_used=0;
@@ -74,11 +123,96 @@ unsigned int  blst_max_mem=DEFAULT_BLST_MAX_MEM; /* maximum memory used
 													for the blacklist entries*/
 unsigned int blst_timeout=DEFAULT_BLST_TIMEOUT;
 unsigned int blst_timer_interval=DEFAULT_BLST_TIMER_INTERVAL;
-struct dst_blst_entry** dst_blst_hash=0;
+struct dst_blst_lst_head* dst_blst_hash=0;
+
+
+
+#ifdef DST_BLACKLIST_HOOKS
+
+#define MAX_BLST_HOOKS 1
+
+static struct blacklist_hook* blacklist_hooks;
+static unsigned int blacklist_max_hooks=MAX_BLST_HOOKS;
+static int last_blacklist_hook_idx=0;
+
+static int init_blacklist_hooks()
+{
+	blacklist_hooks=pkg_malloc(blacklist_max_hooks*
+								sizeof(struct blacklist_hook));
+	if (blacklist_hooks==0)
+		goto error;
+	memset(blacklist_hooks, 0,
+				blacklist_max_hooks*sizeof(struct blacklist_hook));
+	return 0;
+error:
+	LOG(L_ERR, "blacklist_hooks: memory allocation failure\n");
+	return -1;
+}
+
+
+static void destroy_blacklist_hooks()
+{
+	int r;
+
+	if (blacklist_hooks){
+		for (r=0; r<last_blacklist_hook_idx; r++){
+			if (blacklist_hooks[r].destroy)
+				blacklist_hooks[r].destroy();
+		}
+		pkg_free(blacklist_hooks);
+		blacklist_hooks=0;
+	}
+}
 
 
-#define LOCK_BLST()		lock_get(blst_lock)
-#define UNLOCK_BLST()	lock_release(blst_lock)
+/* allocates a new hook
+ * returns 0 on success and -1 on error
+ * must be called from mod init (from the main process, before forking)*/
+int register_blacklist_hook(struct blacklist_hook *h)
+{
+	struct blacklist_hook* tmp;
+	int new_max_hooks;
+	
+	if (blacklist_max_hooks==0)
+		goto error;
+	if (last_blacklist_hook_idx >= blacklist_max_hooks){
+		new_max_hooks=2*blacklist_max_hooks;
+		tmp=pkg_realloc(blacklist_hooks, 
+				new_max_hooks*sizeof(struct blacklist_hook));
+		if (tmp==0){
+			goto error;
+		}
+		blacklist_hooks=tmp;
+		/* init the new chunk */
+		memset(&blacklist_hooks[last_blacklist_hook_idx+1], 0, 
+					(new_max_hooks-blacklist_max_hooks-1)*
+						sizeof(struct blacklist_hook));
+		blacklist_max_hooks=new_max_hooks;
+	}
+	blacklist_hooks[last_blacklist_hook_idx]=*h;
+	last_blacklist_hook_idx++;
+	return 0;
+error:
+	return -1;
+}
+
+
+int blacklist_run_hooks(struct dest_info* si, unsigned char* flags)
+{
+	int r;
+	int ret;
+	
+	ret=DST_BLACKLIST_ACCEPT; /* default, if no hook installed accept 
+								blacklist operation */
+	for (r=0; r<last_blacklist_hook_idx; r++){
+		ret=blacklist_hooks[r].on_blst_add(si, flags);
+		if (ret!=DST_BLACKLIST_ACCEPT) break;
+	}
+	return ret;
+}
+
+
+#endif /* DST_BLACKLIST_HOOKS */
 
 
 inline static void blst_destroy_entry(struct dst_blst_entry* e)
@@ -133,14 +267,26 @@ void destroy_dst_blacklist()
 		timer_free(blst_timer_h);
 		blst_timer_h=0;
 	}
+#ifdef BLST_LOCK_PER_BUCKET
+		for(r=0; r<DST_BLST_HASH_SIZE; r++)
+			lock_destroy(&dst_blst_hash[r].lock);
+#elif defined BLST_LOCK_SET
+		if (blst_lock_set){
+			lock_set_destroy(blst_lock_set);
+			lock_set_dealloc(blst_lock_set);
+			blst_lock_set=0;
+		}
+#else
 	if (blst_lock){
 		lock_destroy(blst_lock);
 		lock_dealloc(blst_lock);
 		blst_lock=0;
 	}
+#endif
+	
 	if (dst_blst_hash){
 		for(r=0; r<DST_BLST_HASH_SIZE; r++){
-			for (crt=&dst_blst_hash[r], tmp=&(*crt)->next; *crt; 
+			for (crt=&dst_blst_hash[r].first, tmp=&(*crt)->next; *crt; 
 					crt=tmp, tmp=&(*crt)->next){
 			e=*crt;
 			*crt=(*crt)->next;
@@ -154,6 +300,9 @@ void destroy_dst_blacklist()
 		shm_free((void*)blst_mem_used);
 		blst_mem_used=0;
 	}
+#ifdef DST_BLACKLIST_HOOKS
+	destroy_blacklist_hooks();
+#endif
 }
 
 
@@ -161,22 +310,51 @@ void destroy_dst_blacklist()
 int init_dst_blacklist()
 {
 	int ret;
+#ifdef BLST_LOCK_PER_BUCKET
+	int r;
+#endif
 	
 	ret=-1;
+#ifdef DST_BLACKLIST_HOOKS
+	if (init_blacklist_hooks()!=0){
+		ret=E_OUT_OF_MEM;
+		goto error;
+	}
+#endif
 	blst_mem_used=shm_malloc(sizeof(*blst_mem_used));
 	if (blst_mem_used==0){
 		ret=E_OUT_OF_MEM;
 		goto error;
 	}
 	*blst_mem_used=0;
-	dst_blst_hash=shm_malloc(sizeof(struct dst_blst_entry*) *
+	dst_blst_hash=shm_malloc(sizeof(struct dst_blst_lst_head) *
 											DST_BLST_HASH_SIZE);
 	if (dst_blst_hash==0){
 		ret=E_OUT_OF_MEM;
 		goto error;
 	}
-	memset(dst_blst_hash, 0, sizeof(struct dst_blst_entry*) *
+	memset(dst_blst_hash, 0, sizeof(struct dst_blst_lst_head) *
 								DST_BLST_HASH_SIZE);
+#ifdef BLST_LOCK_PER_BUCKET
+	for (r=0; r<DST_BLST_HASH_SIZE; r++){
+		if (lock_init(&dst_blst_hash[r].lock)==0){
+			ret=-1;
+			goto error;
+		}
+	}
+#elif defined BLST_LOCK_SET
+	blst_lock_set=lock_set_alloc(DST_BLST_HASH_SIZE);
+	if (blst_lock_set==0){
+		ret=E_OUT_OF_MEM;
+		goto error;
+	}
+	if (lock_set_init(blst_lock_set)==0){
+		lock_set_dealloc(blst_lock_set);
+		blst_lock_set=0;
+		ret=-1;
+		goto error;
+	}
+#else /* BLST_ONE_LOCK */
 	blst_lock=lock_alloc();
 	if (blst_lock==0){
 		ret=E_OUT_OF_MEM;
@@ -188,6 +366,7 @@ int init_dst_blacklist()
 		ret=-1;
 		goto error;
 	}
+#endif /* BLST*LOCK*/
 	blst_timer_h=timer_alloc();
 	if (blst_timer_h==0){
 		ret=E_OUT_OF_MEM;
@@ -240,6 +419,7 @@ inline static struct dst_blst_entry* _dst_blacklist_lst_find(
 	type=(ip->af==AF_INET6)*BLST_IS_IPV6;
 	for (crt=head, tmp=&(*head)->next; *crt; crt=tmp, tmp=&(*crt)->next){
 		e=*crt;
+		prefetch_loc_r((*crt)->next, 1);
 		/* remove old expired entries */
 		if ((s_ticks_t)(now-(*crt)->expire)>=0){
 			*crt=(*crt)->next;
@@ -260,8 +440,6 @@ inline static struct dst_blst_entry* _dst_blacklist_lst_find(
 /* frees all the expired entries until either there are no more of them
  *  or the total memory used is <= target (to free all of them use -1 for 
  *  targer)
- *  It must be called with LOCK_BLST held (or call dst_blacklist_clean_expired
- *   instead).
  *  params:   target  - free expired entries until no more then taget memory 
  *                      is used  (use 0 to free all of them)
  *            delta   - consider an entry expired if it expires after delta
@@ -271,9 +449,9 @@ inline static struct dst_blst_entry* _dst_blacklist_lst_find(
  *  returns: number of deleted entries
  *  This function should be called periodically from a timer
  */
-inline static int _dst_blacklist_clean_expired_unsafe(unsigned int target,
-												ticks_t delta,
-												ticks_t timeout)
+inline static int dst_blacklist_clean_expired(unsigned int target,
+									  ticks_t delta,
+									  ticks_t timeout)
 {
 	static unsigned short start=0;
 	unsigned short h;
@@ -283,57 +461,41 @@ inline static int _dst_blacklist_clean_expired_unsafe(unsigned int target,
 	ticks_t start_time;
 	ticks_t now;
 	int no=0;
+	int i;
 	
 	now=start_time=get_ticks_raw();
 	for(h=start; h!=(start+DST_BLST_HASH_SIZE); h++){
-		for (crt=&dst_blst_hash[h%DST_BLST_HASH_SIZE], tmp=&(*crt)->next;
-				*crt; crt=tmp, tmp=&(*crt)->next){
-			e=*crt;
-			if ((s_ticks_t)(now+delta-(*crt)->expire)>=0){
-				*crt=(*crt)->next;
-				*blst_mem_used-=DST_BLST_ENTRY_SIZE(*e);
-				blst_destroy_entry(e);
-				no++;
-				if (*blst_mem_used<=target)
-					goto skip;
+		i=h%DST_BLST_HASH_SIZE;
+		if (dst_blst_hash[i].first){
+			LOCK_BLST(i);
+			for (crt=&dst_blst_hash[i].first, tmp=&(*crt)->next;
+					*crt; crt=tmp, tmp=&(*crt)->next){
+				e=*crt;
+				prefetch_loc_r((*crt)->next, 1);
+				if ((s_ticks_t)(now+delta-(*crt)->expire)>=0){
+					*crt=(*crt)->next;
+					*blst_mem_used-=DST_BLST_ENTRY_SIZE(*e);
+					blst_destroy_entry(e);
+					BLST_HASH_STATS_DEC(i);
+					no++;
+					if (*blst_mem_used<=target){
+						UNLOCK_BLST(i);
+						goto skip;
+					}
+				}
+			}
+			UNLOCK_BLST(i);
+			/* check for timeout only "between" hash cells */
+			now=get_ticks_raw();
+			if ((now-start_time)>=timeout){
+				DBG("_dst_blacklist_clean_expired_unsafe: timeout: %d > %d\n",
+						TICKS_TO_MS(now-start_time), TICKS_TO_MS(timeout));
+				goto skip;
 			}
-		}
-		/* check for timeout only "between" hash cells */
-		now=get_ticks_raw();
-		if ((now-start_time)>=timeout){
-			DBG("_dst_blacklist_clean_expired_unsafe: timeout: %d > %d\n",
-					TICKS_TO_MS(now-start_time), TICKS_TO_MS(timeout));
-			goto skip;
 		}
 	}
 skip:
 	start=h; /* next time we start where we left */
-	return no;
-}
-
-
-
-/* frees all the expired entries until either there are no more of them
- *  or the total memory used is <= target (to free all of them use -1 for 
- *  targer)
- *  params:   target  - free expired entries until no more then taget memory 
- *                      is used  (use 0 to free all of them)
- *            delta   - consider an entry expired if it expires after delta
- *                      ticks from now
- *            timeout - exit after timeout ticks
- *
- *  returns: number of deleted entries
- *  This function should be called periodically from a timer
- */
-inline static int dst_blacklist_clean_expired(unsigned int target,
-												ticks_t delta,
-												ticks_t timeout)
-{
-	int no;
-	
-	LOCK_BLST();
-		no=_dst_blacklist_clean_expired_unsafe(target, delta, timeout);
-	UNLOCK_BLST();
 	if (no){
 		DBG("dst_blacklist_clean_expired, %d entries removed\n", no);
 	}
@@ -376,24 +538,28 @@ inline static int dst_blacklist_add_ip(unsigned char err_flags,
 	now=get_ticks_raw();
 	hash=dst_blst_hash_no(proto, ip, port);
 	/* check if the entry already exists */
-	LOCK_BLST();
-		e=_dst_blacklist_lst_find(&dst_blst_hash[hash], ip, proto, port, now);
+	LOCK_BLST(hash);
+		e=_dst_blacklist_lst_find(&dst_blst_hash[hash].first, ip, proto,
+																port, now);
 		if (e){
 			e->flags|=err_flags;
 			e->expire=now+S_TO_TICKS(blst_timeout); /* update the timeout */
 		}else{
-			if ((*blst_mem_used+size)>=blst_max_mem){
+			if (unlikely((*blst_mem_used+size)>=blst_max_mem)){
+				UNLOCK_BLST(hash);
 				/* first try to free some memory  (~ 12%), but don't
 				 * spend more then 250 ms*/
-				_dst_blacklist_clean_expired_unsafe(*blst_mem_used/16*14, 0, 
+				dst_blacklist_clean_expired(*blst_mem_used/16*14, 0, 
 															MS_TO_TICKS(250));
-				if (*blst_mem_used+size>=blst_max_mem){
+				if (unlikely(*blst_mem_used+size>=blst_max_mem)){
 					ret=-1;
 					goto error;
 				}
+				LOCK_BLST(hash);
 			}
 			e=shm_malloc(size);
 			if (e==0){
+				UNLOCK_BLST(hash);
 				ret=E_OUT_OF_MEM;
 				goto error;
 			}
@@ -404,10 +570,11 @@ inline static int dst_blacklist_add_ip(unsigned char err_flags,
 			memcpy(e->ip, ip->u.addr, ip->len);
 			e->expire=now+S_TO_TICKS(blst_timeout); /* update the timeout */
 			e->next=0;
-			dst_blacklist_lst_add(&dst_blst_hash[hash], e);
+			dst_blacklist_lst_add(&dst_blst_hash[hash].first, e);
+			BLST_HASH_STATS_INC(hash);
 		}
+	UNLOCK_BLST(hash);
 error:
-	UNLOCK_BLST();
 	return ret;
 }
 
@@ -426,12 +593,13 @@ inline static int dst_is_blacklisted_ip(unsigned char proto,
 	ret=0;
 	now=get_ticks_raw();
 	hash=dst_blst_hash_no(proto, ip, port);
-	LOCK_BLST();
-		e=_dst_blacklist_lst_find(&dst_blst_hash[hash], ip, proto, port, now);
+	LOCK_BLST(hash);
+		e=_dst_blacklist_lst_find(&dst_blst_hash[hash].first, ip, proto,
+									port, now);
 		if (e){
 			ret=e->flags;
 		}
-	UNLOCK_BLST();
+	UNLOCK_BLST(hash);
 	return ret;
 }
 
@@ -440,6 +608,11 @@ inline static int dst_is_blacklisted_ip(unsigned char proto,
 int dst_blacklist_add(unsigned char err_flags,  struct dest_info* si)
 {
 	struct ip_addr ip;
+
+#ifdef DST_BLACKLIST_HOOKS
+	if (blacklist_run_hooks(si, &err_flags)!=DST_BLACKLIST_ACCEPT)
+		return 0;
+#endif
 	su2ip_addr(&ip, &si->to);
 	return dst_blacklist_add_ip(err_flags, si->proto, &ip,
 								su_getport(&si->to));
@@ -491,9 +664,9 @@ void dst_blst_debug(rpc_t* rpc, void* ctx)
 	struct ip_addr ip;
 	
 	now=get_ticks_raw();
-	LOCK_BLST();
 		for(h=0; h<DST_BLST_HASH_SIZE; h++){
-			for(e=dst_blst_hash[h]; e; e=e->next){
+			LOCK_BLST(h);
+			for(e=dst_blst_hash[h].first; e; e=e->next){
 				dst_blst_entry2ip(&ip, e);
 				rpc->add(ctx, "ssddd", get_proto_name(e->proto), 
 										ip_addr2a(&ip), e->port, 
@@ -502,9 +675,33 @@ void dst_blst_debug(rpc_t* rpc, void* ctx)
 										-TICKS_TO_S(now-e->expire) ,
 										e->flags);
 			}
+			UNLOCK_BLST(h);
+		}
+}
+
+/* only for debugging, it helds the lock too long for "production" use */
+void dst_blst_hash_stats(rpc_t* rpc, void* ctx)
+{
+	int h;
+	struct dst_blst_entry* e;
+#ifdef BLST_HASH_STATS
+	int n;
+	
+	n=0;
+#endif
+	
+		for(h=0; h<DST_BLST_HASH_SIZE; h++){
+#ifdef BLST_HASH_STATS
+			LOCK_BLST(h);
+			for(e=dst_blst_hash[h].first; e; e=e->next) n++;
+			UNLOCK_BLST(h);
+			rpc->add(ctx, "dd", h, n);
+#else
+			rpc->add(ctx, "dd", h, dst_blst_hash[h].entries);
+#endif
 		}
-	UNLOCK_BLST();
 }
 
+
 #endif /* USE_DST_BLACKLIST */
 

dst_blacklist.h

@@ -46,6 +46,22 @@
 #define BLST_ADM_PROHIBITED	(1<<6)	/* administratively prohibited */
 #define BLST_PERMANENT		(1<<7)  /* never deleted, never expires */
 
+
+/*#define DST_BLACKLIST_HOOKS*/
+
+#define DST_BLACKLIST_ACCEPT 0
+#define DST_BLACKLIST_DENY  -1
+
+#ifdef DST_BLACKLIST_HOOKS
+struct blacklist_hook{
+	int (*on_blst_add)(struct dest_info* si, unsigned char* err_flags);
+	/* called before ser shutdown */
+	void (*destroy)(void);
+};
+
+int register_blacklist_hook(struct blacklist_hook *h);
+#endif /* DST_BLACKLIST_HOOKS */
+
 int init_dst_blacklist();
 void destroy_dst_blacklist();