file_out: Module to log custom strings to files (GH #3741)

* add new module to log custom strings to files (GH #3741)
* support for multiple files and also file balancing
* smaller refactorings will be done afterwards (use string in function for names with internal matching, make worker sleep also configurable
* a round of tests has been done, but it will be certainly tested more in the next days

src/modules/file_out/Makefile

@@ -0,0 +1,8 @@
+# WARNING: do not run this directly, it should be run by the main Makefile
+
+include ../../Makefile.defs
+auto_gen=
+NAME=file_out.so
+LIBS=
+
+include ../../Makefile.modules

src/modules/file_out/doc/Makefile

@@ -0,0 +1,4 @@
+docs = file_out.xml
+
+docbook_dir = ../../../../doc/docbook
+include $(docbook_dir)/Makefile.module

src/modules/file_out/doc/file_out.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding='ISO-8859-1'?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.4//EN"
+"http://www.oasis-open.org/docbook/xml/4.4/docbookx.dtd" [
+
+<!-- Include general documentation entities -->
+<!ENTITY % docentities SYSTEM "../../../../doc/docbook/entities.xml">
+%docentities;
+
+]>
+
+<book xmlns:xi="http://www.w3.org/2001/XInclude">
+	<bookinfo>
+		<title>file_out Module</title>
+		<productname class="trade">&kamailioname;</productname>
+		<authorgroup>
+			<author>
+				<firstname>Xenofon</firstname>
+				<surname>Karamanos</surname>
+				<affiliation>
+					<orgname>GILAWA Ltd</orgname>
+				</affiliation>
+				<email>[email protected]</email>
+				<address>
+					<otheraddr>
+						<ulink url="http://www.gilawa.com">https://gilawa.com/</ulink>
+					</otheraddr>
+				</address>
+			</author>
+		</authorgroup>
+		<copyright>
+			<year>2024</year>
+		    <holder>GILAWA Ltd</holder>
+		</copyright>
+	</bookinfo>
+	<toc></toc>
+
+	<xi:include href="fileout_admin.xml"/>
+
+</book>

src/modules/file_out/doc/file_out_admin.xml

@@ -0,0 +1,191 @@
+<?xml version="1.0" encoding='ISO-8859-1'?>
+<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.4//EN"
+"http://www.oasis-open.org/docbook/xml/4.4/docbookx.dtd" [
+
+<!-- Include general documentation entities -->
+<!ENTITY % docentities SYSTEM "../../../../doc/docbook/entities.xml">
+%docentities;
+
+]>
+
+<!-- Module User's Guide -->
+
+<chapter>
+
+	<title>&adminguide;</title>
+
+	<section>
+		<title>Overview</title>
+		<para>
+		This is a simple module to support fast streaming output to file name
+		and handle that changes on an interval. It implements only one function
+		that streams a chunk of text to the current output file handle.
+		</para>
+		<para>
+		The module can be used to write logs for up to 10 different log files.
+		Each log file can be configured to have a different name and extension.
+		String can contain pseudo-variables. The module will replace the
+		pseudo-variables with the actual values. The module will also rotate
+		the log files at a specified interval. The interval is specified in seconds.
+		</para>
+	</section>
+	<section>
+		<title>Dependencies</title>
+		<section>
+			<title>&kamailio; Modules</title>
+			<para>
+		The following modules must be loaded before this module:
+				<itemizedlist>
+					<listitem>
+						<para>
+							<emphasis>none</emphasis>.
+						</para>
+					</listitem>
+				</itemizedlist>
+			</para>
+		</section>
+		<section>
+			<title>External Libraries or Applications</title>
+			<para>
+		The following libraries or applications must be installed before running
+		&kamailio; with this module loaded:
+				<itemizedlist>
+					<listitem>
+						<para>
+							<emphasis>none</emphasis>.
+						</para>
+					</listitem>
+				</itemizedlist>
+			</para>
+		</section>
+	</section>
+	<section>
+		<title>Parameters</title>
+		<section>
+			<title>
+				<varname>base_folder</varname> (string)</title>
+			<para>
+		Absolute path to the folder where log files should be saved.
+
+			</para>
+			<para>
+				<emphasis>
+		    Default value is <quote>/var/log/kamailio/file_out/</quote>.
+				</emphasis>
+			</para>
+			<example>
+				<title>Set <varname>base_folder</varname> parameter</title>
+				<programlisting format="linespecific">
+		...
+		modparam("file_out", "base_folder", "/tmp/file_out/")
+		...
+				</programlisting>
+			</example>
+
+			<title>
+				<varname>base_filename</varname> (string)</title>
+			<para>
+		The filename to be used for each file. Don't include the extension. Required.
+			</para>
+			<para>
+				<emphasis>
+		    Default value is <quote>null</quote>.
+				</emphasis>
+			</para>
+			<example>
+				<title>Set <varname>base_filename</varname> parameter</title>
+				<programlisting format="linespecific">
+			...
+			modparam("file_out", "base_filename", "accounting")
+			...
+				</programlisting>
+			</example>
+
+			<title>
+				<varname>extension</varname> (string)</title>
+			<para>
+		The extension to be used for each file.
+			</para>
+			<para>
+				<emphasis>
+		    Default value is <quote>.out</quote>.
+				</emphasis>
+			</para>
+			<example>
+				<title>Set <varname>extension</varname> parameter</title>
+				<programlisting format="linespecific">
+			...
+			modparam("file_out", "extension", ".txt")
+			...
+				</programlisting>
+			</example>
+
+			<title>
+				<varname>interval_seconds</varname> (int)</title>
+			<para>
+		The interval in seconds between file rotation.
+			</para>
+			<para>
+				<emphasis>
+		    Default value is <quote>600</quote> (10 minutes).
+				</emphasis>
+			</para>
+			<example>
+				<title>Set <varname>interval_seconds</varname> parameter</title>
+				<programlisting format="linespecific">
+			...
+			modparam("file_out", "interval_seconds", "300")
+			...
+				</programlisting>
+			</example>
+
+		</section>
+
+	</section>
+
+	<section>
+		<title>Functions</title>
+		<section>
+			<title>
+				<function moreinfo="none">file_out(index, string)</function>
+			</title>
+			<para>
+		This function is used to write a string to a file. The file is
+		determined by the index parameter. The string parameter is the
+		string to be written to the file.
+
+		Index order is the same as the order in which the log files are
+		defined in the configuration file starting from 0.
+			</para>
+			<example>
+				<title>
+					<function>file_out</function> usage</title>
+				<programlisting format="linespecific">
+			...
+			modparam("file_out", "base_filename", "accounting")
+			modparam("file_out", "base_filename", "missed_calls")
+			...
+			request_route {
+
+			file_out("0", "Writing  to accounting.out file $rm from $fu (IP:$si:$sp)");
+			file_out("1", "Writing  to missed_calls.out file $rm from $fu (IP:$si:$sp)");
+			...
+			}
+				</programlisting>
+			</example>
+		</section>
+
+	</section>
+
+	<section>
+		<title>Exported pseudo-variables</title>
+		<itemizedlist>
+			<listitem>
+				<para>
+					<emphasis>none</emphasis>.
+				</para>
+			</listitem>
+		</itemizedlist>
+	</section>
+
+</chapter>

src/modules/file_out/file_out.c

@@ -0,0 +1,380 @@
+/*
+ * Copyright (C) 2024 GILAWA Ltd
+ *
+ * This file is part of Kamailio, a free SIP server.
+ *
+ * Kamailio is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version
+ *
+ * Kamailio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include "../../core/sr_module.h"
+#include "../../core/route_struct.h"
+#include "../../core/str.h"
+#include "../../core/mod_fix.h"
+#include "../../core/locking.h"
+#include "../../core/cfg/cfg_struct.h"
+#include "types.h"
+
+#include <stdio.h>
+#include <time.h>
+#include <math.h>
+#include <errno.h>
+#include <unistd.h> /* usleep() */
+
+
+MODULE_VERSION
+
+
+#define FO_MAX_PATH_LEN 2048
+#define FO_MAX_FILES 10 /* Maximum number of files */
+
+static int mod_init(void);
+static int child_init(int rank);
+static void destroy(void);
+
+static int fo_write_to_file(sip_msg_t *msg, char *index, char *log_message);
+
+static FILE *fo_get_file_handle(const int index);
+static int fo_get_full_path(const int index, char *full_path);
+static int fo_init_file(const int index);
+static int fo_close_file(const int index);
+static int fo_check_interval();
+static int fo_fixup_int_pvar(void **param, int param_no);
+static int fo_count_assigned_files();
+static void fo_log_writer_process(int rank);
+static int fo_add_filename(modparam_t type, void *val);
+
+/* Default parameters */
+char *fo_base_folder = "/var/log/kamailio/file_out/";
+char *fo_base_filename[FO_MAX_FILES] = {""};
+char *fo_extension = ".out";
+int fo_interval_seconds = 10 * 60;
+
+/* Shared variables */
+fo_queue_t *fo_queue = NULL;
+int *fo_number_of_files = NULL;
+
+time_t fo_stored_timestamp = 0;
+time_t fo_current_timestamp = 0;
+FILE *fo_file_output[FO_MAX_FILES];
+
+static cmd_export_t cmds[] = {{"file_out", (cmd_function)fo_write_to_file, 2,
+									  fo_fixup_int_pvar, 0, ANY_ROUTE},
+		{0, 0, 0, 0, 0, 0}};
+
+static param_export_t params[] = {
+		{"base_folder", PARAM_STRING, &fo_base_folder},
+		{"base_filename", PARAM_STRING | PARAM_USE_FUNC, &fo_add_filename},
+		{"interval_seconds", PARAM_INT, &fo_interval_seconds},
+		{"extension", PARAM_STRING, &fo_extension}, {0, 0, 0}};
+
+struct module_exports exports = {
+		"file_out",		 /* module name */
+		DEFAULT_DLFLAGS, /* dlopen flags */
+		cmds,			 /* exported functions */
+		params,			 /* exported parameters */
+		0,				 /* RPC method exports */
+		0,				 /* exported pseudo-variables */
+		0,				 /* response handling function */
+		mod_init,		 /* module initialization function */
+		child_init,		 /* per-child init function */
+		destroy			 /* module destroy function */
+};
+
+
+static int mod_init(void)
+{
+	LM_DBG("initializing\n");
+	LM_DBG("base_folder = %s\n", fo_base_folder);
+	LM_DBG("base_filename_path= %s\n", fo_base_filename[0]);
+	LM_DBG("interval_seconds = %d\n", fo_interval_seconds);
+	LM_DBG("extension = %s\n", fo_extension);
+
+	//*  Create shared variables */
+	fo_queue = (fo_queue_t *)shm_malloc(sizeof(fo_queue_t));
+	if(!fo_queue) {
+		SHM_MEM_ERROR;
+		return -1;
+	}
+	fo_queue->front = NULL;
+	fo_queue->rear = NULL;
+	if(lock_init(&fo_queue->lock) == 0) {
+		/* error initializing the lock */
+		LM_ERR("error initializing the lock\n");
+		return -1;
+	}
+
+	/* Count the given files */
+	*fo_number_of_files = fo_count_assigned_files();
+
+	/* Initialize per process vars */
+	fo_stored_timestamp = time(NULL);
+
+	/* Register worker process */
+	register_procs(1);
+	cfg_register_child(1);
+	LM_DBG("Initialization done\n");
+	return 0;
+}
+
+/**
+ * per-child init function
+ */
+static int child_init(int rank)
+{
+	int pid;
+	if(rank != PROC_MAIN) {
+		return 0;
+	}
+
+	pid = fork_process(PROC_NOCHLDINIT, "log_writ", 1);
+	if(pid < 0) {
+		LM_ERR("fork failed\n");
+		return -1; /* error */
+	}
+	if(pid == 0) {
+		/* child */
+		/* initialize the config framework */
+		if(cfg_child_init())
+			return -1;
+
+		/* Initialize and open files  */
+		for(int i = 0; i < *fo_number_of_files; i++) {
+			fo_init_file(i);
+		}
+
+		for(;;) {
+			/* update the local config framework structures */
+			cfg_update();
+
+			usleep(10000);
+			fo_log_writer_process(rank);
+		}
+		// return 0;
+	}
+	return 0;
+}
+
+/**
+ * module destroy function
+ */
+static void destroy(void)
+{
+	int result = 0;
+	if(fo_file_output[0] != NULL) {
+		result = fclose(fo_file_output[0]);
+		if(result != 0) {
+			ERR("Failed to close output file");
+		}
+	}
+}
+
+static void fo_log_writer_process(int rank)
+{
+	fo_log_message_t log_message;
+	int result = 0;
+	while(!fo_is_queue_empty(fo_queue)) {
+		result = fo_dequeue(fo_queue, &log_message);
+		if(result < 0) {
+			LM_ERR("deque error\n");
+			return;
+		}
+		FILE *out = fo_get_file_handle(log_message.dest_file);
+		if(out == NULL) {
+			LM_ERR("out is NULL\n");
+			return;
+		}
+
+		fprintf(out, "%s\n", log_message.message);
+		fflush(out);
+	}
+}
+
+/*
+* fixup function for two parameters
+* 1st param: int
+* 2nd param: string containing PVs
+*/
+static int fo_fixup_int_pvar(void **param, int param_no)
+{
+	if(param_no == 1) {
+		return fixup_igp_null(param, param_no);
+	} else if(param_no == 2) {
+		return fixup_var_pve_str_12(param, param_no);
+	}
+	return 0;
+}
+
+static int fo_add_filename(modparam_t type, void *val)
+{
+	if(fo_number_of_files == NULL) {
+		LM_ERR("fo_number_of_files is NULL\n");
+		fo_number_of_files = (int *)shm_malloc(sizeof(int));
+		if(!fo_number_of_files) {
+			SHM_MEM_ERROR;
+			return -1;
+		}
+		*fo_number_of_files = 0;
+	}
+
+	if((type & PARAM_STRING) == 0) {
+		LM_ERR("bad parameter type %d\n", type);
+		return -1;
+	}
+
+	if(fo_number_of_files != NULL && *fo_number_of_files >= FO_MAX_FILES) {
+		LM_ERR("Maximum number of files [%d] reached. The rest will not be "
+			   "processed \n",
+				FO_MAX_FILES);
+		return 0;
+	}
+	fo_base_filename[*fo_number_of_files] = (char *)val;
+	LM_DBG("fo_base_filename[%d] = %s\n", *fo_number_of_files,
+			fo_base_filename[*fo_number_of_files]);
+	(*fo_number_of_files)++;
+	return 0;
+}
+
+/*
+* Count the number of files that are assigned
+* return the number of files (saved also in shared fo_number_of_files)
+*/
+static int fo_count_assigned_files()
+{
+	return *fo_number_of_files;
+}
+
+static int fo_init_file(const int index)
+{
+	char full_path[FO_MAX_PATH_LEN];
+	fo_get_full_path(index, full_path);
+	fo_file_output[index] = fopen(full_path, "a");
+	if(fo_file_output[index] == NULL) {
+		LM_ERR("Couldn't open file %s\n", strerror(errno));
+		return -1;
+	}
+	return 1;
+}
+
+static int fo_close_file(const int index)
+{
+	int result = 0;
+	if(fo_file_output[index] != NULL) {
+		result = fclose(fo_file_output[index]);
+		if(result != 0) {
+			ERR("Failed to close output file");
+			return -1;
+		}
+	}
+	return 1;
+}
+
+/*
+* Check if the interval has passed
+* return 1 if interval has passed
+* return 0 if interval has not passed
+*/
+static int fo_check_interval()
+{
+	fo_current_timestamp = time(NULL);
+
+	// Calculate the difference between the current timestamp and the stored timestamp
+	int difference = difftime(fo_current_timestamp, fo_stored_timestamp);
+	if(difference >= fo_interval_seconds) {
+		LM_ERR("interval has passed\n");
+		return 1;
+	}
+	// LM_ERR("interval has not passed\n");
+	return 0;
+}
+/**
+ * maintain file handle
+ */
+
+static FILE *fo_get_file_handle(const int index)
+{
+	int result = 0;
+	if(fo_check_interval()) {
+		/* Interval passed. Close all files and open new ones */
+		for(int i = 0; i < *fo_number_of_files; i++) {
+			result = fo_close_file(i);
+			if(result != 1) {
+				LM_ERR("Failed to close output file");
+				return NULL;
+			}
+		}
+		fo_stored_timestamp = fo_current_timestamp;
+
+		LM_DBG("Opening new files due to interval passed\n");
+		/* Make sure we know how many files we need */
+		if(fo_number_of_files == NULL) {
+			*fo_number_of_files = fo_count_assigned_files();
+		}
+		/* Initialize and open files  */
+		for(int i = 0; i < *fo_number_of_files; i++) {
+			result = fo_init_file(i);
+			if(result != 1) {
+				LM_ERR("Failed to initialize output file");
+				return NULL;
+			}
+		}
+		return fo_file_output[index];
+	} else {
+		/* Interval has not passed */
+		/* Assume files are correct */
+		return fo_file_output[index];
+	}
+}
+
+/**
+ * Determine full file path
+ */
+static int fo_get_full_path(const int index, char *full_path)
+{
+	snprintf(full_path, FO_MAX_PATH_LEN, "%s/%s_%.f%s", fo_base_folder,
+			fo_base_filename[index], difftime(fo_stored_timestamp, (time_t)0),
+			fo_extension);
+	LM_INFO("Path to write to: %s\n", full_path);
+	return 1;
+}
+
+static int fo_write_to_file(sip_msg_t *msg, char *index, char *log_message)
+{
+	int result, file_index;
+	if(index == NULL || log_message == NULL) {
+		LM_ERR("index or log_messsage is NULL\n");
+		return -1;
+	}
+
+	result = get_int_fparam(&file_index, msg, (fparam_t *)index);
+	if(result < 0) {
+		LM_ERR("Failed to get int from param 0: %d\n", result);
+		return -1;
+	}
+
+	str value;
+	result = get_str_fparam(&value, msg, (fparam_t *)log_message);
+	if(result < 0) {
+		LM_ERR("Failed to string from param 1: %d\n", result);
+		return -1;
+	}
+
+	/* Add the logging string to the global gueue */
+	fo_log_message_t logMessage;
+	logMessage.message = value.s;
+	logMessage.dest_file = file_index;
+	fo_enqueue(fo_queue, logMessage);
+
+	return 1;
+}

src/modules/file_out/types.c

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2024 GILAWA Ltd
+ *
+ * This file is part of Kamailio, a free SIP server.
+ *
+ * Kamailio is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version
+ *
+ * Kamailio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include <stddef.h>
+#include <stdlib.h>
+
+#include "types.h"
+
+static fo_node_t *fo_new_node(fo_log_message_t data)
+{
+	fo_node_t *temp = (fo_node_t *)shm_malloc(sizeof(fo_node_t));
+	temp->data = data;
+	temp->next = NULL;
+	return temp;
+}
+
+
+int fo_enqueue(fo_queue_t *q, fo_log_message_t data)
+{
+	/*
+	Copy the contents of data.message
+    */
+	char *message_copy = (char *)shm_malloc(strlen(data.message) + 1);
+	strcpy(message_copy, data.message);
+	data.message = message_copy;
+	fo_node_t *temp = fo_new_node(data);
+
+	lock_get(&(q->lock));
+
+	if(q->rear == NULL) {
+		q->front = q->rear = temp;
+		lock_release(&(q->lock));
+		return 1;
+	}
+
+	q->rear->next = temp;
+	q->rear = temp;
+
+	lock_release(&(q->lock));
+	return 1;
+}
+
+int fo_dequeue(fo_queue_t *q, fo_log_message_t *data)
+{
+	lock_get(&(q->lock));
+
+	if(q->front == NULL) {
+		lock_release(&(q->lock));
+		return -1;
+	}
+	fo_node_t *temp = q->front;
+	*data = temp->data;
+	q->front = q->front->next;
+
+	if(q->front == NULL)
+		q->rear = NULL;
+
+
+	if(temp != NULL) {
+		if(temp->data.message != NULL) {
+			shm_free(temp->data.message);
+			temp->data.message = NULL;
+		}
+		shm_free(temp);
+		temp = NULL;
+	}
+	lock_release(&(q->lock));
+
+	return 1;
+}
+
+int fo_is_queue_empty(fo_queue_t *q)
+{
+	lock_get(&(q->lock));
+	int result = (q->front == NULL);
+	lock_release(&(q->lock));
+	return result;
+}
+
+int fo_queue_size(fo_queue_t *q)
+{
+	lock_get(&(q->lock));
+	int count = 0;
+	fo_node_t *temp = q->front;
+	while(temp != NULL) {
+		count++;
+		temp = temp->next;
+	}
+	lock_release(&(q->lock));
+	return count;
+}

src/modules/file_out/types.h

@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2024 GILAWA Ltd
+ *
+ * This file is part of Kamailio, a free SIP server.
+ *
+ * Kamailio is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version
+ *
+ * Kamailio is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#include "../../core/locking.h"
+
+typedef struct log_message
+{
+	char *message;
+	int dest_file;
+} fo_log_message_t;
+
+typedef struct node
+{
+	struct log_message data;
+	struct node *next;
+} fo_node_t;
+
+typedef struct queue
+{
+	struct node *front;
+	struct node *rear;
+	gen_lock_t lock;
+} fo_queue_t;
+
+int fo_enqueue(fo_queue_t *q, fo_log_message_t data);
+int fo_dequeue(fo_queue_t *q, fo_log_message_t *data);
+int fo_is_queue_empty(fo_queue_t *q);
+int fo_queue_size(fo_queue_t *q);