Merge pull request #2435 from kamailio/vhernando/xhttp_prom_v2

xhttp_prom: add support for histogram metrics.

src/modules/xhttp_prom/doc/xhttp_prom_admin.xml

@@ -23,7 +23,7 @@
 	</para>
 	<para>
 	  The module generates metrics based on &kamailio; statistics, and also the user
-	  can create his own metrics (currently counters and gauges).
+	  can create his own metrics (currently counters, gauges and histograms).
 	</para>
 	<para>
 	  The xHTTP_PROM module uses the xHTTP module to handle HTTP requests.
@@ -112,6 +112,10 @@ modparam("xhttp_prom", "xhttp_prom_buf_size", 1024)
 		Specifies a timeout in minutes. A metric not used during this timeout is
 		automatically deleted. Listing metrics does not count as using them.
 	  </para>
+	  <para>
+		<emphasis>If set to 0 timeout is disabled.</emphasis>
+ 		Negative values are not allowed.
+	  </para>
 	  <para>
 		<emphasis>
 		  Default value is 60 minutes.
@@ -151,6 +155,14 @@ modparam("xhttp_prom", "xhttp_prom_timeout", 600)
 		  Default value is "", meaning do not display any &kamailio; statistics.
 		</emphasis>
 	  </para>
+	  <para>
+		<emphasis>
+		  IMPORTANT: &kamailio; internal statistics are parsed to convert - into _, so they
+		  accomplish with Prometheus guidelines for metric names.
+		  <ulink url="https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels">https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels</ulink>
+		</emphasis>
+		User generated statistics and label names are not parsed.
+	  </para>
 	  <example>
 		<title>Set <varname>xhttp_prom_stats</varname> parameter</title>
 		<programlisting format="linespecific">
@@ -166,6 +178,32 @@ modparam("xhttp_prom", "xhttp_prom_stats", "200_replies")
 
 # Do not display internal &kamailio; statistics. This is the default option.
 modparam("xhttp_prom", "xhttp_prom_stats", "")
+...
+		</programlisting>
+	  </example>
+	</section>
+	<section id="xhttp_prom.p.xhttp_prom_beginning">
+	  <title><varname>xhttp_prom_beginning</varname> (str)</title>
+	  <para>
+		Specifies beginning of string the metrics are build with.
+	  </para>
+	  <para>
+		<emphasis>It defaults to "kamailio_"</emphasis>, so if not specified every
+		metric will start with "kamailio_".
+	  </para>
+	  <para>
+		Void string "" is also allowed, meaning no prefix string for every metric
+		name.
+	  </para>
+	  <example>
+		<title>Set <varname>xhttp_prom_beginning</varname> parameter</title>
+		<programlisting format="linespecific">
+...
+# All metrics will start with "my_metric_".
+modparam("xhttp_prom", "xhttp_prom_beginning", "my_metric_")
+
+# No string at the beginning.
+modparam("xhttp_prom", "xhttp_prom_beginning", "");
 ...
 		</programlisting>
 	  </example>
@@ -283,6 +321,86 @@ modparam("xhttp_prom", "prom_gauge", "name=gg_second;");
 # Create gg_third gauge with two labels method and handler:
 modparam("xhttp_prom", "prom_gauge", "name=gg_third; label=method:handler;");
 
+...
+		</programlisting>
+	  </example>
+	</section>
+	<section id="xhttp_prom.p.prom_histogram">
+	  <title><varname>prom_histogram</varname> (str)</title>
+	  <para>
+		Create a histogram metric.
+	  </para>
+	  <para>
+		This function declares a histogram but the actual histogram is only created
+		when observing it.
+	  </para>
+	  <para>
+		It takes a list of attribute=value separated by semicolon, the attributes can
+		be name, label and buckets.
+	  </para>
+	  <itemizedlist>
+		<listitem>
+		  <para>
+			<emphasis>name</emphasis> - name of the histogram. This attribute is mandatory.
+			It is used to generate the metric name. Each name is unique, no metric shall
+			repeat a name.
+		  </para>
+		</listitem>
+		<listitem>
+		  <para>
+			<emphasis>label</emphasis> - names of labels in the histogram. Optional.
+			Only one label parameter at most allowed in histograms.
+			Each label name is separated by <emphasis>:</emphasis> without spaces.
+			At most only three label names allowed in each label parameter.
+			<example><title><varname>prom_histogram</varname> label example</title>
+			<programlisting format="linespecific">
+# Create two labels called method and handler
+label = method:handler
+This would generate  {method="whatever", handler="whatever2"} when building
+the metric.
+			</programlisting>
+			</example>
+		  </para>
+		</listitem>
+		<listitem>
+		  <para>
+			<emphasis>buckets</emphasis> - specifies upper bounds for buckets in the
+			histogram. This attribute is optional.
+		  </para>
+		  <para>
+			Bucket values are separated by ":". Each value has to be a number.
+		  </para>
+		  <para>
+			"+Inf" upper bucket is always automatically included.
+		  </para>
+		  <para>At least one bucket value is needed (other than +Inf).</para>
+		  <para>Every bucket value has to increase in the list.</para>
+		  <para>
+			If no buckets specified, default bucket list is set to these values:
+			<para>[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]</para>
+		  </para>
+		</listitem>
+	  </itemizedlist>
+	  <example>
+		<title>Set <varname>prom_histogram</varname> parameter</title>
+		<programlisting format="linespecific">
+...
+		  
+# Create my_histo histogram with no labels and default buckets.
+modparam("xhttp_prom", "prom_histogram", "name=my_histo;");
+
+# Create second_histo histogram with one label and default buckets.
+# modparam("xhttp_prom", "prom_histogram", "name=second_histo; label=my_lbl");
+
+# Create a histogram with no labels and buckets 3.1, 5, 6.5
+modparam("xhttp_prom", "prom_histogram", "name=histo_third; buckets=3.1:5:6.5");
+
+# Create a histogram with a label and buckets 3.1, 5, 6.5
+modparam("xhttp_prom", "prom_histogram", "name=histo_fourth; label=lbl; buckets=3.1:5:6.5");
+
+These lines declare the histogram but the actual metric will be created when
+using it by prom_histogram_observe function.
+
 ...
 		</programlisting>
 	  </example>
@@ -454,6 +572,60 @@ modparam("xhttp_prom", "prom_gauge", "name=cnt02; label=method:IP;");
 prom_gauge_set("gg02", "2.8", "push", "192.168.0.1");
 # When listed the metric it will show a line like this:
 kamailio_gg02 {method="push", IP="192.168.0.1"} 2.8 1234567890
+...
+		</programlisting>
+	  </example>
+	</section>
+	<section id="xhttp_prom.f.prom_histogram_observe">
+	  <title>
+		<function moreinfo="none">prom_histogram_observe(name, number, l0, l1, l2)</function>
+	  </title>
+	  <para>
+		Get a histogram identified by its name and labels and observe a value in it.
+		If histogram does not exist it creates the histogram and accumulate the value in its
+		buckets, counter and sum.
+	  </para>
+	  <para>
+		Name is mandatory, number is mandatory.
+		Number is a string that will be parsed as a float.
+		l0, l1, l2 are values of labels and are optional.
+	  </para>
+	  <para>
+		name value and number of labels have to match a previous histogram definition with prom_histogram.
+	  </para>
+	  <para>
+		This function accepts pseudovariables on its parameters.
+	  </para>
+	  <para>
+		Available via KEMI framework as <emphasis>histogram_observe_l0</emphasis>,
+		<emphasis>histogram_observe_l1</emphasis>,
+		<emphasis>histogram_observe_l2</emphasis> and
+		<emphasis>histogram_observe_l3</emphasis>.
+	  </para>
+	  <example>
+		<title><function>prom_histogram_observe</function> usage</title>
+		<programlisting format="linespecific">
+...
+# Definition of hist01 histogram with no labels and default buckets.
+modparam("xhttp_prom", "prom_histogram", "name=hist01;");
+...
+# Observe -12.5 value in hist01 histogram (with no labels). If histogram does not exist it gets created:
+prom_histogram_observe("hist01", "-12.5");
+...
+
+# Definition of hist02 histogram with two labels method and IP and buckets [2.3, 5.8, +Inf]:
+modparam("xhttp_prom", "prom_histogram", "name=hist02; label=method:IP; buckets=2.3:5.8");
+...
+# Observe 2.8 value in hist02 histogram with labels method and IP.
+# It creates the histogram if it does not exist.
+prom_histogram_observe("hist02", "2.8", "push", "192.168.0.1");
+# When listed the metric it will show lines like this:
+hist02_bucket{method="push", IP="192.168.0.1", le="2.300000"} 0 1592574659768
+hist02_bucket{method="push", IP="192.168.0.1", le="5.800000"} 1 1592574659768
+hist02_bucket{method="push", IP="192.168.0.1", le="+Inf"} 1 1592574659768
+hist02_sum{method="push", IP="192.168.0.1"} 2.800000 1592574659768
+hist02_count{method="push", IP="192.168.0.1"} 1 1592574659768
+
 ...
 		</programlisting>
 	  </example>
@@ -705,6 +877,31 @@ event_route[xhttp:request] {
 		  ...
 		</programlisting>
 	  </example>
+    </section>
+	<section  id="xhttp_prom.rpc.histogram_observe">
+	  <title><function moreinfo="none">xhttp_prom.histogram_observe</function></title>
+	  <para>
+		Observe a number in a histogram. Select the histogram by its name and labels.
+	  </para>
+      <para>
+        Name: <emphasis>xhttp_prom.histogram_observe</emphasis>
+      </para>
+      <para>Parameters:</para>
+	  <itemizedlist>
+		<listitem><para><emphasis>name</emphasis>: name of the histogram (mandatory)</para></listitem>
+		<listitem><para><emphasis>number</emphasis>: float value to observe in the histogram (mandatory)</para></listitem>
+		<listitem><para><emphasis>l0</emphasis>: value of the first label (optional)</para></listitem>
+		<listitem><para><emphasis>l1</emphasis>: value of second label (optional)</para></listitem>
+		<listitem><para><emphasis>l2</emphasis>: value of the third label (optional)</para></listitem>
+	  </itemizedlist>
+	  <example>
+		<title><function>xhttp_prom.histogram_observe</function> usage</title>
+		<programlisting format="linespecific">
+		  ...
+		  &kamcmd; xhttp_prom.histogram_observe "hist01" -- -5.2
+		  ...
+		</programlisting>
+	  </example>
     </section>
 	<section  id="xhttp_prom.rpc.metric_list_print">
 	  <title><function moreinfo="none">xhttp_prom.metric_list_print</function></title>
@@ -715,6 +912,10 @@ event_route[xhttp:request] {
         Name: <emphasis>xhttp_prom.metric_list_print</emphasis>
       </para>
       <para>Parameters:<emphasis>none</emphasis></para>
+	  <para>
+		<emphasis>NOTE:</emphasis>: If you list a lot of metrics you may need to increase
+		buffer size of your RPC transport layer.
+	  </para>
 	  <example>
 		<title><function>xhttp_prom.metric_list_print</function> usage</title>
 		<programlisting format="linespecific">

src/modules/xhttp_prom/prom.c

@@ -22,7 +22,10 @@
  */
 
 /**
- * Functionality of prometheus module.
+ * @file
+ * @brief xHTTP_PROM :: Functionality of xhttp_prom module.
+ * @ingroup xhttp_prom
+ * - Module: @ref xhttp_prom
  */
 
 #include <string.h>
@@ -35,9 +38,10 @@
 
 #include "prom.h"
 #include "prom_metric.h"
+#include "xhttp_prom.h"
 
 /**
- * Delete current buffer data.
+ * @brief Delete current buffer data.
  */
 void prom_body_delete(prom_ctx_t *ctx)
 {
@@ -45,10 +49,10 @@ void prom_body_delete(prom_ctx_t *ctx)
 }
 
 /**
- * Write some data in prom_body buffer.
+ * @brief Write some data in prom_body buffer.
  *
- * /return number of bytes written.
- * /return -1 on error.
+ * @return number of bytes written.
+ * @return -1 on error.
  */
 int prom_body_printf(prom_ctx_t *ctx, char *fmt, ...)
 {
@@ -87,10 +91,63 @@ error:
 }
 
 /**
- * Get current timestamp in milliseconds.
+ * @brief Write a metric name in prom_body buffer.
  *
- * /param ts pointer to timestamp integer.
- * /return 0 on success.
+ * Based on prom_body_printf function.
+ *
+ * @return number of bytes written.
+ * @return -1 on error.
+ */
+int prom_body_name_printf(prom_ctx_t *ctx, char *fmt, ...)
+{
+	struct xhttp_prom_reply *reply = &ctx->reply;
+	
+	va_list ap;
+	
+	va_start(ap, fmt);
+
+	LM_DBG("Body current length: %d\n", reply->body.len);
+
+	char *p = reply->buf.s + reply->body.len;
+	int remaining_len = reply->buf.len - reply->body.len;
+	LM_DBG("Remaining length: %d\n", remaining_len);
+
+	/* int vsnprintf(char *str, size_t size, const char *format, va_list ap); */
+	int len = vsnprintf(p, remaining_len, fmt, ap);
+	if (len < 0) {
+		LM_ERR("Error printing body buffer\n");
+		goto error;
+	} else if (len >= remaining_len) {
+		LM_ERR("Error body buffer overflow: %d (%d)\n", len, remaining_len);
+		goto error;
+	} else {
+		/* Buffer printed OK. */
+
+		/* Change - into _ to accomplish with Prometheus guidelines for metric names */
+		int i;
+		for (i=0; i<len; i++) {
+			if (p[i] == '-') {
+				p[i] = '_';
+			}
+		}
+		
+		reply->body.len += len;
+		LM_DBG("Body new length: %d\n", reply->body.len);
+	}
+
+	va_end(ap);
+	return len;
+
+error:
+	va_end(ap);
+	return -1;
+}
+
+/**
+ * @brief Get current timestamp in milliseconds.
+ *
+ * @param ts pointer to timestamp integer.
+ * @return 0 on success.
  */
 int get_timestamp(uint64_t *ts)
 {
@@ -109,9 +166,9 @@ int get_timestamp(uint64_t *ts)
 }
 
 /**
- * Generate a string suitable for a Prometheus metric.
+ * @brief Generate a string suitable for a Prometheus metric.
  *
- * /return 0 on success.
+ * @return 0 on success.
  */
 static int metric_generate(prom_ctx_t *ctx, str *group, str *name, counter_handle_t *h)
 {
@@ -123,17 +180,20 @@ static int metric_generate(prom_ctx_t *ctx, str *group, str *name, counter_handl
 		LM_ERR("Error getting current timestamp\n");
 		return -1;
 	}
-	LM_DBG("Timestamp: %" PRIu64 "\n", ts);
 
-	
-	/* LM_DBG("%.*s:%.*s = %lu\n",
-	   group->len, group->s, name->len, name->s, counter_val); */
-	LM_DBG("kamailio_%.*s_%.*s %lu %" PRIu64 "\n",
+	LM_DBG("metric -> group: %.*s  name: %.*s  value: %lu  TS: %" PRIu64 "\n",
 		   group->len, group->s, name->len, name->s,
 		   counter_val, (uint64_t)ts);
 
-	if (prom_body_printf(ctx, "kamailio_%.*s_%.*s %lu %" PRIu64 "\n",
-						 group->len, group->s, name->len, name->s,
+	/* Print metric name. */
+	if (prom_body_name_printf(ctx, "%.*s%.*s_%.*s",
+						 xhttp_prom_beginning.len, xhttp_prom_beginning.s,
+						 group->len, group->s, name->len, name->s) == -1) {
+		LM_ERR("Fail to print\n");
+		return -1;
+	}
+						 
+	if (prom_body_printf(ctx, " %lu %" PRIu64 "\n",
 						 counter_val, (uint64_t)ts) == -1) {
 		LM_ERR("Fail to print\n");
 		return -1;
@@ -143,7 +203,7 @@ static int metric_generate(prom_ctx_t *ctx, str *group, str *name, counter_handl
 }
 
 /**
- * Statistic getter callback.
+ * @brief Statistic getter callback.
  */
 static void prom_get_grp_vars_cbk(void* p, str* g, str* n, counter_handle_t h)
 {
@@ -151,7 +211,7 @@ static void prom_get_grp_vars_cbk(void* p, str* g, str* n, counter_handle_t h)
 }
 
 /**
- * Group statistic getter callback.
+ * @brief Group statistic getter callback.
  */
 static void prom_get_all_grps_cbk(void* p, str* g)
 {
@@ -161,9 +221,9 @@ static void prom_get_all_grps_cbk(void* p, str* g)
 #define STATS_MAX_LEN 1024
 
 /**
- * Get statistics (based on stats_get_all)
+ * @brief Get statistics (based on stats_get_all)
  *
- * /return 0 on success
+ * @return 0 on success
  */
 int prom_stats_get(prom_ctx_t *ctx, str *stat)
 {

src/modules/xhttp_prom/prom.h

@@ -22,7 +22,10 @@
  */
 
 /**
- * Header for functionality of prometheus module.
+ * @file
+ * @brief xHTTP_PROM :: Header for functionality of xhttp_prom module.
+ * @ingroup xhttp_prom
+ * - Module: @ref xhttp_prom
  */
 
 #ifndef _PROM_H_
@@ -31,25 +34,25 @@
 #include "xhttp_prom.h"
 
 /**
- * Get current timestamp in milliseconds.
+ * @brief Get current timestamp in milliseconds.
  *
- * /param ts pointer to timestamp integer.
- * /return 0 on success.
+ * @param ts pointer to timestamp integer.
+ * @return 0 on success.
  */
 int get_timestamp(uint64_t *ts);
 
 /**
- * Write some data in prom_body buffer.
+ * @brief Write some data in prom_body buffer.
  *
- * /return number of bytes written.
- * /return -1 on error.
+ * @return number of bytes written.
+ * @return -1 on error.
  */
 int prom_body_printf(prom_ctx_t *ctx, char *fmt, ...);
 
 /**
- * Get statistics (based on stats_get_all)
+ * @brief Get statistics (based on stats_get_all)
  *
- * /return 0 on success
+ * @return 0 on success
  */
 int prom_stats_get(prom_ctx_t *ctx, str *stat);
 

src/modules/xhttp_prom/prom_metric.h

@@ -22,7 +22,10 @@
  */
 
 /**
- * Header user defined metrics for Prometheus.
+ * @file
+ * @brief xHTTP_PROM :: Header for user defined metrics.
+ * @ingroup xhttp_prom
+ * - Module: @ref xhttp_prom
  */
 
 #ifndef _PROM_METRIC_H_
@@ -31,48 +34,73 @@
 #include "xhttp_prom.h"
 
 /**
- * Initialize user defined metrics.
+ * @brief Initialize user defined metrics.
  */
-int prom_metric_init(int timeout_minutes);
+int prom_metric_init();
 
 /**
- * Close user defined metrics.
+ * @brief Close user defined metrics.
  */
 void prom_metric_close();
 
 /**
- * Create a counter and add it to list.
+ * @brief Create a counter and add it to list.
  */
 int prom_counter_create(char *spec);
 
 /**
- * Create a gauge and add it to list.
+ * @brief Create a gauge and add it to list.
  */
 int prom_gauge_create(char *spec);
 
 /**
- * Print user defined metrics.
+ * @brief Create a histogram and add it to list.
+ *
+ * @return 0 on success.
+ */
+int prom_histogram_create(char *spec);
+
+/**
+ * @brief Print user defined metrics.
  */
 int prom_metric_list_print(prom_ctx_t *ctx);
 
 /**
- * Reset a counter.
+ * @brief Reset a counter.
  */
 int prom_counter_reset(str *s_name, str *l1, str *l2, str *l3);
 
 /**
- * Reset value in a gauge.
+ * @brief Reset value in a gauge.
  */
 int prom_gauge_reset(str *s_name, str *l1, str *l2, str *l3);
 
 /**
- * Add some positive amount to a counter.
+ * @brief Add some positive amount to a counter.
  */
 int prom_counter_inc(str *s_name, int number, str *l1, str *l2, str *l3);
 
 /**
- * Set a value in a gauge.
+ * @brief Set a value in a gauge.
  */
 int prom_gauge_set(str *s_name, double number, str *l1, str *l2, str *l3);
 
+/**
+ * @brief Observe a value in a histogram.
+ *
+ * @param number value to observe.
+ */
+int prom_histogram_observe(str *s_name, double number, str *l1, str *l2, str *l3);
+
+/**
+ * @brief Parse a string and convert to double.
+ *
+ * @param s_number pointer to number string.
+ * @param pnumber double passed as reference.
+ *
+ * @return 0 on success.
+ * On error value pointed by pnumber is undefined.
+ */
+int double_parse_str(str *s_number, double *pnumber);
+
 #endif // _PROM_METRIC_H_

src/modules/xhttp_prom/xhttp_prom.h

@@ -21,11 +21,11 @@
  *
  */
 
-/*!
- * \file
- * \brief XHTTP_PROM :: 
- * \ingroup xhttp_prom
- * Module: \ref xhttp_prom
+/**
+ * @file
+ * @brief xHTTP_PROM :: General structures and variables.
+ * @ingroup xhttp_prom
+ * - Module: @ref xhttp_prom
  */
 
 #ifndef _XHTTP_PROM_H
@@ -40,7 +40,8 @@
 
 
 
-/** Representation of the xhttp_prom reply being constructed.
+/**
+ * @brief Representation of the xhttp_prom reply being constructed.
  *
  * This data structure describes the xhttp_prom reply that is being constructed
  * and will be sent to the client.
@@ -55,7 +56,8 @@ struct xhttp_prom_reply {
 };
 
 
-/** The context of the xhttp_prom request being processed.
+/**
+ * @brief The context of the xhttp_prom request being processed.
  *
  * This is the data structure that contains all data related to the xhttp_prom
  * request being processed, such as the reply code and reason, data to be sent
@@ -69,6 +71,15 @@ typedef struct prom_ctx {
 	int reply_sent;
 } prom_ctx_t;
 
+/**
+ * @brief string for beginning of metrics.
+ */
+extern str xhttp_prom_beginning;
+
+/**
+ * @brief timeout in minutes to delete old metrics.
+ */
+extern int timeout_minutes;
 
 #endif /* _XHTTP_PROM_H */