started doxygen documentation of CDS library and little changes which seemed

useful when creating documentation (reference counters, removed library init
and cleanup function because it is not needed more)

lib/cds/cds.c

@@ -3,6 +3,8 @@
 #include <cds/sync.h>
 #include <cds/logger.h>
 
+#if 0
+
 typedef struct {
 	int init_cnt;
 } init_data_t;
@@ -55,3 +57,4 @@ void cds_cleanup()
 	}
 }
 
+#endif

lib/cds/cds.h

@@ -1,17 +1,77 @@
 #ifndef __CDS_H
 #define __CDS_H
 
-/* declaration of initialization/destruction functions */
+/** \defgroup cds CDS library - Common Data Structures 
+ *
+ * This library contains many useful functions and data structures. It is
+ possible to use it with Sip Express Router (SER) or without it. In the first
+ case it is able to use some internal SER's data types like strings.
+ *
+ * \section cds_conventions Conventions
+ *  - data types (structures, enums, ...) have their names with suffix \c _t
+ *  (\c str_t, \c dstring_t, ...)
+ *  - many functions have prefix according to data structure on which are
+ *  operating (like dstr_append which operates on dstring_t data structure)
+ *  - most functions return 0 as successful result and nonzero value as error
+ *
+ * \section cds_dependencies Dependencies
+ * This library depends only on standard C libraries and needs no other
+ * external libraries.
+ *
+ * \section cds_ser_usage Usage with SER
+ * There can be problems with using shared libraries on different platforms.
+ * Currently supported solution is that user must supply LD_LIBRARY_PATH
+ * (or something similar on his OS) with the path to the library before 
+ * starting SER with modules needed the library.
+ *
+ * \section cds_nonser_usage Usage without SER
+ * The library can be compiled without dependencies on SER and can be linked
+ * to whatever program as any other dynamic library. This style of usage is
+ * probably used only when debugging problems which is hard to find with SER
+ * (outside of SER is possible to use tools like Valgrind) and for author's
+ * little programs and need not to be documented more... ;-)
+ *
+ * \par History
+ * There were following reasons to introduce this library:
+ *  - many duplicated functions in modules (copy&pasted between modules)
+ *  without touching SER's core
+ *  - possibility to debug programs outside of SER due to its simplicity 
+ *  and many usefull tools
+ *
+ * @{ */
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-	
+
+#if 0
+/* not used */
+
+/** \defgroup cds_init Initialization/destruction
+ * Library needs to be initialized before using it and
+ * un-initialized after it is not used more. Use \ref cds_initialize and 
+ * \ref cds_cleanup for this purpose.
+ * @{ */
+
+/** Initializes CDS library. 
+ *
+ * Currently initializes reference counter which is experimental and 
+ * probably will be removed in the future because seems to be rather 
+ * useless here. */
 int cds_initialize();
+
+/** Cleans up after CDS. After calling this function can't be called
+ * reference counter functions. */
 void cds_cleanup();
 
+#endif
+
+/** @} */
+
 #ifdef __cplusplus
 }
 #endif
 
+/** @} */
+
 #endif

lib/cds/memory.h

@@ -32,6 +32,17 @@
 extern "C" {
 #endif
 
+/** \ingroup cds
+ * @{ 
+ *
+ * \defgroup cds_memory Memory management
+ *
+ * Memory operations are common for whole CDS library. Because it must work
+ together with SER's memory management and must work without it too, there are
+ wrapper macros for memory allocation/deallocation.
+ * 
+ * @{ */
+
 /* typedef void*(*cds_malloc_func)(unsigned int size);
 typedef void(*cds_free_func)(void *ptr);
 
@@ -40,16 +51,64 @@ extern cds_free_func cds_free;
 
 void cds_set_memory_functions(cds_malloc_func _malloc, cds_free_func _free); */
 
+/** \def cds_malloc(s)
+ * Function/macro for memory allocation. Which function is choosen depends on
+ * SER and TRACE_CDS_MEMORY defines. 
+ *
+ * When SER is defined shm_malloc is choosen, standard malloc otherwise. */
+
+/** \def cds_free(p)
+ * Function/macro for memory deallocation. Which function is choosen depends
+ * on SER and TRACE_CDS_MEMORY defines. 
+ *
+ * If SER is defined shm_free is choosen, standard free otherwise. */
+
+/** \def cds_malloc_ptr
+ * Function/macro for memory allocation when pointer to function needed. Which
+ * function is choosen depends on SER and TRACE_CDS_MEMORY defines. 
+ *
+ * If SER is defined shm_malloc is choosen, standard malloc otherwise.  */
+
+/** \def cds_free_ptr
+ * Function/macro for memory deallocation when pointer to function needed.
+ * Which function is choosen depends on SER and TRACE_CDS_MEMORY defines. 
+ *
+ * If SER is defined shm_free is choosen, standard free otherwise.  */
+
+/** \def cds_malloc_pkg(s)
+ * Function/macro for 'local' memory allocation. Which function is choosen
+ * depends on SER and TRACE_CDS_MEMORY defines. 
+ *
+ * When SER is defined pkg_malloc is choosen, standard malloc otherwise. */
+
+/** \def cds_free_pkg(p)
+ * Function/macro for 'local' memory deallocation. Which function is choosen
+ * depends on SER and TRACE_CDS_MEMORY defines. 
+ *
+ * When SER is defined pkg_free is choosen, standard free otherwise. */
+
 #ifdef TRACE_CDS_MEMORY
 
+/** \internal Debugging variant of alloc function */
 void *debug_malloc(int size, const char *file, int line);
+
+/** \internal Debugging variant of free function */
 void debug_free(void *block, const char *file, int line);
+
+/** \internal Another debugging variant of alloc function - used when pointer
+ * to function needed. */
 void *debug_malloc_ex(unsigned int size);
+
+/** \internal Another debugging variant of free function - used when pointer to
+ * function needed. */
 void debug_free_ex(void *block);
 
-/* trace function */
+/* \internal Helper function for debugging - shows some debugging information about
+ * memory allocations (currently only the number of allocated blocks). */
 void cds_memory_trace(char *dst, int dst_len);
-/* initializes internal variables for memory tracing */
+
+/** \internal Helper function which is useful for memory debugging only - initializes
+ * internal variables for memory tracing */
 void cds_memory_trace_init();
 
 #define cds_malloc(s)	debug_malloc(s,__FILE__, __LINE__)
@@ -95,6 +154,8 @@ void shm_free_x(void *ptr);
 }
 #endif
 
+/** @} 
+ * @} */
 
 #endif
 

lib/cds/msg_queue.c

@@ -93,14 +93,14 @@ int push_message(msg_queue_t *q, mq_message_t *m)
 	if ((!q) || (!m)) return -1;
 	m->next = NULL;
 	
-	if (q->use_mutex) cds_mutex_lock(&q->q_mutex);
+	if (q->flags & MQ_USE_MUTEX) cds_mutex_lock(&q->q_mutex);
 	if (q->last) q->last->next = m;
 	else {
 		q->first = m;
 		q->last = m;
 	}
 	q->last = m;
-	if (q->use_mutex) cds_mutex_unlock(&q->q_mutex);
+	if (q->flags & MQ_USE_MUTEX) cds_mutex_unlock(&q->q_mutex);
 	
 	return 0;
 }
@@ -110,11 +110,11 @@ int mq_add_to_top(msg_queue_t *q, mq_message_t *m)
 	if ((!q) || (!m)) return -1;
 	m->next = NULL;
 	
-	if (q->use_mutex) cds_mutex_lock(&q->q_mutex);
+	if (q->flags & MQ_USE_MUTEX) cds_mutex_lock(&q->q_mutex);
 	m->next = q->first;
 	q->first = m;
 	if (!q->last) q->last = m;
-	if (q->use_mutex) cds_mutex_unlock(&q->q_mutex);
+	if (q->flags & MQ_USE_MUTEX) cds_mutex_unlock(&q->q_mutex);
 	
 	return 0;
 }
@@ -124,7 +124,7 @@ mq_message_t *pop_message(msg_queue_t *q)
 	mq_message_t *m;
 	if (!q) return NULL;
 
-	if (q->use_mutex) cds_mutex_lock(&q->q_mutex);
+	if (q->flags & MQ_USE_MUTEX) cds_mutex_lock(&q->q_mutex);
 	m = q->first;
 	if (m) {
 		if (q->first == q->last) {
@@ -134,18 +134,17 @@ mq_message_t *pop_message(msg_queue_t *q)
 		else q->first = m->next;
 		m->next = NULL;
 	}
-	if (q->use_mutex) cds_mutex_unlock(&q->q_mutex);
+	if (q->flags & MQ_USE_MUTEX) cds_mutex_unlock(&q->q_mutex);
 		
 	return m;
 }
 
-/** 1 ... empty, 0 ...  NOT empty !! */
 int is_msg_queue_empty(msg_queue_t *q)
 {
 	int res = 1;
-	if (q->use_mutex) cds_mutex_lock(&q->q_mutex);
+	if (q->flags & MQ_USE_MUTEX) cds_mutex_lock(&q->q_mutex);
 	if (q->first) res = 0;
-	if (q->use_mutex) cds_mutex_unlock(&q->q_mutex);
+	if (q->flags & MQ_USE_MUTEX) cds_mutex_unlock(&q->q_mutex);
 	return res;
 }
 
@@ -156,20 +155,31 @@ int msg_queue_init(msg_queue_t *q)
 
 int msg_queue_init_ex(msg_queue_t *q, int synchronize)
 {
-	if (synchronize) cds_mutex_init(&q->q_mutex);
-	init_reference_counter(&q->ref);
-	q->use_mutex = synchronize;
+	if (synchronize) {
+		cds_mutex_init(&q->q_mutex);
+		q->flags = MQ_USE_MUTEX;
+	}
+	else q->flags = 0;
 	q->first = NULL;
 	q->last = NULL;
 	return 0;
 }
 
-void msg_queue_destroy(msg_queue_t *q)
+/** \internal Destroys all internal data of message queue and 
+ * optionaly frees it if no more references exist. */
+static inline void msg_queue_destroy_and_free(msg_queue_t *q, int do_free)
 {
 	mq_message_t *m,*n;
 	if (!q) return;
 	
-	if (q->use_mutex) cds_mutex_lock(&q->q_mutex);
+	if (q->flags & MQ_USE_REF_CNTR) {
+		if (!remove_reference(&q->ref)) {
+			/* this was NOT the last reference */
+			return;
+		}
+	}
+
+	if (q->flags & MQ_USE_MUTEX) cds_mutex_lock(&q->q_mutex);
 	m = q->first;
 	while (m) {
 		n = m->next;
@@ -178,17 +188,29 @@ void msg_queue_destroy(msg_queue_t *q)
 	}
 	q->first = NULL;
 	q->last = NULL;
-	if (q->use_mutex) {
+	if (q->flags & MQ_USE_MUTEX) {
 		cds_mutex_unlock(&q->q_mutex);
 		cds_mutex_destroy(&q->q_mutex);
 	}
+	if (do_free) cds_free(q);
+}
+
+void msg_queue_destroy(msg_queue_t *q)
+{
+	msg_queue_destroy_and_free(q, 0);
 }
 
 void msg_queue_free(msg_queue_t *q)
 {
-	if (remove_reference(&q->ref)) {
-		msg_queue_destroy(q);
-		cds_free(q);
+	msg_queue_destroy_and_free(q, 1);
+}
+
+void msg_queue_init_ref_cnt(msg_queue_t *q, reference_counter_group_t *grp)
+{
+	if (grp) {
+		init_reference_counter(grp, &q->ref);
+		q->flags |= MQ_USE_REF_CNTR;
 	}
+	else q->flags &= ~MQ_USE_REF_CNTR; /* don't use reference counter */
 }
 

lib/cds/msg_queue.h

@@ -33,8 +33,31 @@
 extern "C" {
 #endif
 
+/** \ingroup cds
+ * @{ 
+ *
+ * \defgroup cds_msg_queue Message Queue
+ *
+ * Message queue is a structure useful for sending data between processes.
+ * It can be synchronized via its own mutex or the synchronization can
+ * be left on caller. It can use reference counter which is useful
+ * when accessing dynamicaly allocated queue destroyed by its last user.
+ *
+ * @{
+ * */
+
 typedef void (*destroy_function_f)(void *);
 
+/** Structure holding message which can be put
+ * into message queue.
+ *
+ * There is a need to allow destroing the message without knowing its
+ * internals (destroying message queue with non-processed messages) and thus
+ * the destroy_function able to fully destroy whole data hold by message must
+ * be given. It is mostly needed to choose the function manually only for
+ * complex data with pointers which content need to be freed too.
+ * For simple structures it is set automaticaly during the message creation.
+ */
 typedef struct _mq_message_t {
 	void *data;
 	int data_len;
@@ -47,50 +70,90 @@ typedef struct _mq_message_t {
 	char data_buf[1];
 } mq_message_t;
 
+
+/** Message queue flag meaning that internal queue mutex is used for
+ * synchronization. It is set during message queue initialization via \ref
+ * msg_queue_init or \ref msg_queue_init_ex. */
+#define MQ_USE_MUTEX	1
+
+/** Message queue flag meaning that reference counters are used. 
+ * To set this flag is needed to call \ref msg_queue_init_ref_cnt with
+ * non-NULL group parameter. */ 
+#define MQ_USE_REF_CNTR	2
+
 typedef struct msg_queue {
 	reference_counter_data_t ref;
 	mq_message_t *first;
 	mq_message_t *last;
 	cds_mutex_t q_mutex;
-	int use_mutex;
+	unsigned int flags;
 } msg_queue_t;
 
 #define get_message_data(msg)		(msg ? msg->data: NULL)
 #define get_message_data_len(msg)	(msg ? msg->data_len: 0)
 
-/** the space for data is allocated in messages data
+/** The space for data is allocated in messages data
  * (they are automaticaly freed!)! Pointer to allocated
  * data bytes is in data variable in the message structure. */
 mq_message_t *create_message_ex(int data_len);
 
-/** data must be allocated using cds_malloc and they
- * are automacicaly freed! */
+/** Creates message holding data allocated using cds_malloc.
+ * Warning: data must be allocated using cds_malloc and they
+ * are automacicaly freed by free_message! */
 mq_message_t *create_message(void *data, int data_len);
 
+/** Sets function which will be called by free_message to destroy data. 
+ *
+ * This function may be useful when a complex structure with pointers is added
+ * as data parameter.  */
 void set_data_destroy_function(mq_message_t *msg, destroy_function_f func);
 
-/** initializes message, 
- * if auto_free set, data must be allocated using cds_malloc and are automaticaly freed by free_message 
- * (and if msg_queue_destroy called) */
+/** Initializes message.
+ * If auto_free set, data must be allocated using cds_malloc and are
+ * automaticaly freed by free_message (and if msg_queue_destroy called) */
 void init_message_ex(mq_message_t *m, void *data, int data_len, destroy_function_f func);
 
-/** frees the message and data holding by the message !!!! */
+/** Frees the message and data hold by the message. */
 void free_message(mq_message_t *msg);
 
+/** Put message into queue. */
 int push_message(msg_queue_t *q, mq_message_t *m);
+
+/** Remove message from queue. */
 mq_message_t *pop_message(msg_queue_t *q);
 
-/** 1 ... empty, 0 ...  NOT empty !! */
+/** Tests if message queue holds a message.
+ * \retval 1 if empty
+ * \retval 0 if NOT empty. */
 int is_msg_queue_empty(msg_queue_t *q);
 
-/** initializes synchronized message queue */
+/** Initializes message queue with a mutex guarding queue operations. */
 int msg_queue_init(msg_queue_t *q);
+
+/** Initializes message queue. If synchronize is set it initializes
+ * a mutex guarding queue operations otherwise the message queue remains
+ * unsynchronized. */
 int msg_queue_init_ex(msg_queue_t *q, int synchronize);
+
+/** Initializes reference counter for given message queue
+ * \param grp specifies group of reference counters to use. The message
+ * queue will stop using the reference counter if NULL. 
+ * \param q specifies the message queue */
+void msg_queue_init_ref_cnt(msg_queue_t *q, reference_counter_group_t *grp);
+
+/** Destroys message queue if no more references exist. 
+ * This function destroys all message queue internal data but doesn't free
+ * the message queue itself. It can be useful for staticaly allocated queues
+ * or when allocated not using cds_malloc. */
 void msg_queue_destroy(msg_queue_t *q);
 
-/* removes reference to message queue and frees it if no other references exist */
+/** Destroys and frees the message queue if no more references exist.
+ * It uses cds_free for freeing the memory. */
 void msg_queue_free(msg_queue_t *q);
 
+/** @} 
+ @} */
+
 #ifdef __cplusplus
 }
 #endif

lib/cds/ref_cntr.c

@@ -2,42 +2,50 @@
 #include <cds/logger.h>
 #include <cds/memory.h>
 
-/* One global mutex for reference counting may be enough. 
- * If problems try to create pool of precreated mutexes
- * and use them randomly.
- */
-static cds_mutex_t *ref_cntr_mutex = NULL;
+/* functions for initialization and destruction */
 
-/* global functions for initialization and destruction */
-
-int reference_counter_initialize()
+reference_counter_group_t *create_reference_counter_group(int mutex_cnt)
 {
-	if (!ref_cntr_mutex) {
-		ref_cntr_mutex = (cds_mutex_t*)cds_malloc(sizeof(cds_mutex_t));
-		if (ref_cntr_mutex) {
-			cds_mutex_init(ref_cntr_mutex);
-			return 0;
-		}
+	reference_counter_group_t *g;
+	int i;
+
+	g = cds_malloc(sizeof(*g) + mutex_cnt * sizeof(cds_mutex_t));
+	if (!g) {
+		ERROR_LOG("can't allocate memory\n");
+		return NULL;
+	}
+
+	for (i = 0; i < mutex_cnt; i++) {
+		cds_mutex_init(&g->mutexes[i]);
 	}
-	return -1;
+	g->mutex_to_assign = 0;
+	g->mutex_cnt = mutex_cnt;
+
+	return g;
 }
 
-void reference_counter_cleanup()
+void free_reference_counter_group(reference_counter_group_t *grp)
 {
-	if (ref_cntr_mutex) {
-		cds_mutex_destroy(ref_cntr_mutex);
-		cds_free((void*)ref_cntr_mutex);
-		ref_cntr_mutex = NULL;
+	int i;
+	if (grp) {
+		for (i = 0; i < grp->mutex_cnt; i++) {
+			cds_mutex_destroy(&grp->mutexes[i]);
+		}
+		cds_free(grp);
 	}
 }
 
 /* -------------------------------------------------------------------- */
 
-void init_reference_counter(reference_counter_data_t *ref)
+void init_reference_counter(reference_counter_group_t *grp, reference_counter_data_t *ref)
 {
-	if (ref) {
+	int m;
+	if (ref && grp) {
+		m = grp->mutex_to_assign;
 		ref->cntr = 1;
-		ref->mutex = ref_cntr_mutex;
+		ref->mutex = grp->mutexes + m;
+		m = (m + 1) % grp->mutex_cnt; /* can't be less than zero */
+		grp->mutex_to_assign = m;
 	}
 }
 

lib/cds/ref_cntr.h

@@ -5,36 +5,151 @@
 extern "C" {
 #endif
 
+/** \ingroup cds
+ * @{ 
+ *
+ * \defgroup cds_ref_cnt Reference counting
+ *
+ * Experimental functions for reference counting (to simplify
+ * this code elsewhere).
+ * 
+ * Reference counter (\ref reference_counter_data_t) holds integer number which
+ * should be changed using functions \ref add_reference and \ref
+ * remove_reference. Using these functions is the number read/changed from
+ * locked section guarded by one mutex from a set of 'group mutexes'.
+ *
+ * Often used scenario:
+ * - list of structures, change in the list (adding, removing) is guarded by
+ *   one mutex, 
+ * - each structure has reference counter, when the count is zero, the
+ *   structure can be removed from the list and freed
+ *
+ * Note that mutex for adding references is needed because references are not
+ * added from critical section guarded by main list mutex but can be added whenever.
+ *
+ * Typical usage:
+ * \code
+ * struct some_structure {
+ *     ...
+ *     reference_counter_data_t ref;
+ *     ...
+ * };
+ *
+ * reference_counter_group_t *some_grp;
+ * 
+ * void init()
+ * {
+ *     some_grp = init_reference_counter_group(16);
+ * }
+ *
+ * void destroy()
+ * {
+ *     free_reference_counter_group(some_grp);
+ * }
+ * 
+ * ...
+ *
+ * // adding a member:
+ * struct some_struct *ss = malloc(...);
+ * init_reference_counter(some_grp, &ss->ref);
+ * lock(list);
+ * add_to_list(ss);
+ * unlock(list);
+ *
+ * ...
+ * // adding a reference doesn't need to lock list
+ * // can be done only by a reference owner
+ * add_reference(&ss->ref);
+ *
+ * // releasing a member when not needed for caller and there is
+ * // no way how to obtain reference for released member
+ * // (no way how to find a member in list)
+ * if (remove_reference(&ss->ref)) {
+ *     // last reference removed
+ *     lock(list);
+ *     remove_from_list(ss);
+ *     free(ss);
+ *     unlock(list);
+ * }
+ *  
+ * // or 
+ * // releasing a member when not needed for caller and it is possible 
+ * // to 'search' in the list (reference to released member can be somehow 
+ * // obtained by inspecting the list):
+ * lock(list);
+ * if (remove_reference(&ss->ref)) {
+ *     // last reference removed
+ *     remove_from_list(ss);
+ *     free(ss);
+ * } 
+ * unlock(list);
+ * \endcode
+ *
+ * \todo use atomic operations instead of locking
+ * @{ */
+
 #include <cds/sync.h>
-	
+
+/** Structure holding reference counter value. */
 typedef struct {
-	int cntr;
-	cds_mutex_t *mutex;
+	int cntr; /**< counter value */
+	cds_mutex_t *mutex; /**< mutex asigned to this reference counter */
 } reference_counter_data_t;
-	
-/* these functions can be called only by owner of at least one reference */
-/* owner is somebody who:
- *    - created a referenced structure
- *    - added a reference
- *    - got a reference by an ovner
- */
-
-void init_reference_counter(reference_counter_data_t *ref);
+
+/** Structure holding information about group of reference counters.  
+ * It holds array of mutexes which are assigned to single reference 
+ * counters in this group. The assignment is done using an operation 
+ * on pointer to reference_counter_data_t. */
+typedef struct {
+	int mutex_cnt; /**< number of mutexes for this group */
+
+	/** number of next mutex to be assigned - this member is NOT 
+	 * read/changed from critical section but it doesn't matter
+	 * if it will be rewritten between processes because it is used
+	 * for load distributing only (the worst thing that can happen
+	 * is that the same mutex is assigned twice) */
+	int mutex_to_assign; 
+	cds_mutex_t mutexes[1]; /**< array of mutexes (allocated together with the structure)*/
+} reference_counter_group_t;
+
+/** Initializes reference counter - sets its value to 1. 
+ * After call to this function, the caller is owner of first 
+ * reference. From now it can call other functions like
+ * \ref add_reference or \ref remove_reference. 
+ *
+ * This function initializes the mutex - it chooses one from 
+ * group mutexes. The mutex can not be changed after this 
+ * call (it is only for reading). */
+void init_reference_counter(reference_counter_group_t *grp, reference_counter_data_t *ref);
+
+/** Adds reference - increments reference counter.
+ * This function can be called only by owner of at least one reference! */
 void add_reference(reference_counter_data_t *ref);
+
+/** Returns the value of reference counter. This function is mostly
+ * useless. */
 int get_reference_count(reference_counter_data_t *ref);
 
-/* returns:
- * 0 if reference removed, but exist other references
- * 1 if removed last refernce and the element SHOULD be freed
+/** Removes reference - decrements reference counter.
+ * This function can be called only by owner of at least one reference!
  *
- * usage:
- * 
- * some_structure *ss;
- * ...
- * if (remove_reference(&ss->ref)) cds_free(&ss->ref);
+ * \retval 0 if reference removed, but other references exist
+ * \retval 1 if removed last reference
  *  */
 int remove_reference(reference_counter_data_t *ref);
 
+/** Creates and initializes group of reference counters. All reference 
+ * counters 'belonging' to this group are using the same set of mutexes. */
+reference_counter_group_t *create_reference_counter_group(int mutex_cnt);
+
+/** Destroys all resources used by reference counter group.
+ * After this function call no reference counter initialized
+ * by this group can be used. */
+void free_reference_counter_group(reference_counter_group_t *grp);
+
+/** @} 
+ * @} */
+
 #ifdef __cplusplus
 }
 #endif