pdb: new protocol between pdb_server and kama pdb module

Modified the communication protocol between the pdb clent <-> server such
that the server will give more feedback on scenarios like "pdb_id not found" or
"request number contains letters". New msg types or reply codes can be easily
added. Curent version of the protocol is 1 (0x01).
Also backwards compatibility is maintained when the first received byte is
different than the known versions (now, just 0x01).
Updated the http link for the get_carrier_germany script. Created a new perl
script to get german carrier id.
Updated doku (utils/pdbt/docs/network_protocol.txt).

added the perls script

modules/pdb/common.h

@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2009 1&1 Internet AG
+ *
+ * This file is part of sip-router, a free SIP server.
+ *
+ * sip-router is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version
+ *
+ * sip-router is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License 
+ * along with this program; if not, write to the Free Software 
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+#ifndef _COMMON_H_
+#define _COMMON_H_
+
+
+
+
+#include <stdint.h> 
+
+
+
+
+/*
+ 0 no carrier id defined.
+ 1..999 are regular carrier ids.
+ 1000 is used as fake carrier id when merging carriers we are not interested in.
+ -1000..-1 used in dtm to indicate a carrier id and that no more nodes will follow (leaf node compression).
+ -1001 used in dtm to mark a pointer to a child node as NULL.
+*/
+#define MIN_PDB_CARRIERID 1
+#define MAX_PDB_CARRIERID 999
+#define OTHER_CARRIERID 1000
+#define MAX_CARRIERID 1000
+#define NULL_CARRIERID -1001
+#define PAYLOADSIZE 256
+
+
+#define IS_VALID_PDB_CARRIERID(id) ((id>=MIN_PDB_CARRIERID) && (id<=MAX_PDB_CARRIERID))
+#define IS_VALID_CARRIERID(id) ((id>=MIN_PDB_CARRIERID) && (id<=MAX_CARRIERID))
+
+#define PDB_VERSION     1
+
+
+
+typedef int16_t carrier_t;
+
+enum __attribute__((packed)) pdb_versions {
+    PDB_VERSION_1 = 1,
+    PDB_VERSION_MAX
+};
+
+enum __attribute__((packed)) pdb_types {
+    PDB_TYPE_REQUEST_ID = 0,    /* request pdb type */
+    PDB_TYPE_REPLY_ID,          /* reply pdb type */
+    PDB_TYPE_MAX
+};
+
+enum __attribute__((packed)) pdb_codes {
+    PDB_CODE_DEFAULT = 0,   /* for request */
+    PDB_CODE_OK,            /* for response - OK */
+    PDB_CODE_NOT_NUMBER,    /* for response - letters found in the number */
+    PDB_CODE_NOT_FOUND,     /* for response - no pdb_id found for the number */
+    PDB_CODE_MAX
+};
+
+struct __attribute__((packed)) pdb_hdr {
+    uint8_t version;
+    uint8_t type;
+    uint8_t code;
+    uint8_t length;
+    uint16_t id;
+};
+
+struct __attribute__((packed)) pdb_bdy {
+    char payload[PAYLOADSIZE];
+};
+
+struct __attribute__((packed)) pdb_msg {
+    struct pdb_hdr hdr;
+    struct pdb_bdy bdy;
+};
+
+#endif

modules/pdb/doc/pdb_admin.xml

@@ -25,6 +25,10 @@
   	parallel and use the first answer, that comes back. A timeout for the
 	query can be defined in milliseconds. The queying can be activated and
   	deactivated using FIFO commands.
+	</para>
+	<para>
+    More about the new communication protocol between this module and pdb_server
+    can be found in utils/pdbt/docs/network_protocol.txt. The current version is 1.
 	</para>
 	</section>
 

modules/pdb/pdb.c

@@ -38,16 +38,15 @@
 #include <arpa/inet.h>
 #include <errno.h>
 
-MODULE_VERSION
-
-
-#define NETBUFSIZE 200
+#include "common.h"
 
+MODULE_VERSION
 
 static char* modp_server = NULL;  /*!< format: \<host\>:\<port\>,... */
 static int timeout = 50;  /*!< timeout for queries in milliseconds */
 static int timeoutlogs = -10;  /*!< for aggregating timeout logs */
 static int *active = NULL;
+static uint16_t *global_id = NULL;
 
 
 /*!
@@ -90,6 +89,14 @@ struct mi_root * mi_pdb_status(struct mi_root* cmd, void* param);  /* usage: kam
 struct mi_root * mi_pdb_activate(struct mi_root* cmd, void* param);  /* usage: kamctl fifo pdb_activate */
 struct mi_root * mi_pdb_deactivate(struct mi_root* cmd, void* param);  /* usage: kamctl fifo pdb_deactivate */
 
+/* debug function for the new client <-> server protocol */
+static void pdb_msg_dbg(struct pdb_msg msg, char *dbg_msg);
+
+/* build the new protocol message before transmission */
+static int pdb_msg_format_send(struct pdb_msg *msg,
+                               uint8_t version, uint8_t type,
+                               uint8_t code, uint16_t id,
+                               char *payload, uint16_t payload_len);
 
 static cmd_export_t cmds[]={
 	{ "pdb_query", (cmd_function)pdb_query, 2, pdb_query_fixup, 0, REQUEST_ROUTE | FAILURE_ROUTE },
@@ -150,20 +157,65 @@ struct server_list_t {
 static struct server_list_t *server_list;
 
 
+/* debug function for the new client <-> server protocol */
+static void pdb_msg_dbg(struct pdb_msg msg, char *dbg_msg) {
+    int i;
+    char buf[PAYLOADSIZE];
+    char *ptr = buf;
+
+    for (i = 0; i < msg.hdr.length - sizeof(msg.hdr); i++) {
+        ptr += sprintf(ptr,"%02X ", msg.bdy.payload[i]);
+    }
+
+    LM_DBG("%s\n"
+           "version = %d\ntype = %d\ncode = %d\nid = %d\nlen = %d\n"
+           "payload = %s\n",
+            dbg_msg,
+            msg.hdr.version, msg.hdr.type, msg.hdr.code, msg.hdr.id, msg.hdr.length,
+            buf);
+}
+
+/* build the message before send */
+static int pdb_msg_format_send(struct pdb_msg *msg,
+                               uint8_t version, uint8_t type,
+                               uint8_t code, uint16_t id,
+                               char *payload, uint16_t payload_len)
+{
+    msg->hdr.version    = version;
+    msg->hdr.type       = type;
+    msg->hdr.code       = code;
+    msg->hdr.id         = id;
+
+    if (payload == NULL) {
+        /* just ignore the NULL buff (called when just want to set the len) */
+        msg->hdr.length     = sizeof(struct pdb_hdr);
+        return 0;
+    } else {
+        msg->hdr.length     = sizeof(struct pdb_hdr) + payload_len;
+        memcpy(msg->bdy.payload, payload, payload_len);
+        return 0;
+    }
+
+    return 0;
+}
+
+
+
 /*!
  * \return 1 if query for the number succeded and the avp with the corresponding carrier id was set,
  * -1 otherwise
  */
 static int pdb_query(struct sip_msg *_msg, struct multiparam_t *_number, struct multiparam_t *_dstavp)
 {
+    struct pdb_msg msg;
 	struct timeval tstart, tnow;
 	struct server_item_t *server;
 	short int carrierid, *_id;
-	char buf[NETBUFSIZE+1+sizeof(carrierid)];
+    char buf[sizeof(struct pdb_msg)];
 	size_t reqlen;
 	int_str avp_val;
 	struct usr_avp *avp;
-	int i, ret, nflush;
+	int i, ret, nflush, bytes_received;
 	long int td;
 	str number = STR_NULL;
 
@@ -209,7 +261,7 @@ static int pdb_query(struct sip_msg *_msg, struct multiparam_t *_number, struct
 	server = server_list->head;
 	while (server) {
 		nflush = 0;
-		while (recv(server->sock, buf, NETBUFSIZE, MSG_DONTWAIT) > 0) {
+		while (recv(server->sock, buf, sizeof(struct pdb_msg), MSG_DONTWAIT) > 0) {
 			nflush++;
 			if (gettimeofday(&tnow, NULL) != 0) {
 				LM_ERR("gettimeofday() failed with errno=%d (%s)\n", errno, strerror(errno));
@@ -227,24 +279,46 @@ static int pdb_query(struct sip_msg *_msg, struct multiparam_t *_number, struct
 
 	/* prepare request */
 	reqlen = number.len + 1; /* include null termination */
-	if (reqlen > NETBUFSIZE) {
+	if (reqlen > sizeof(struct pdb_bdy)) {
 		LM_ERR("number too long '%.*s'.\n", number.len, number.s);
 		return -1;
 	}
 	strncpy(buf, number.s, number.len);
 	buf[number.len] = '\0';
 
-	/* send request to all servers */
-	server = server_list->head;
-	while (server) {
-		LM_DBG("sending request to '%s:%d'\n", server->host, server->port);
-		ret=sendto(server->sock, buf, reqlen, MSG_DONTWAIT, (struct sockaddr *)&(server->dstaddr), server->dstaddrlen);
-		if (ret < 0) {
-			LM_ERR("sendto() failed with errno=%d (%s)\n", errno, strerror(errno));
-		}
-		server = server->next;
-	}
-		
+    switch (PDB_VERSION) {
+        case PDB_VERSION_1:
+            pdb_msg_format_send(&msg, PDB_VERSION, PDB_TYPE_REQUEST_ID, PDB_CODE_DEFAULT, htons(*global_id), buf, reqlen);
+            pdb_msg_dbg(msg, "Kamailio pdb client sends:");
+
+            /* increment msg id for the next request */
+            *global_id = *global_id + 1;
+
+            /* send request to all servers */
+            server = server_list->head;
+            while (server) {
+                LM_DBG("sending request to '%s:%d'\n", server->host, server->port);
+                ret=sendto(server->sock, (struct pdb_msg*)&msg, msg.hdr.length, MSG_DONTWAIT, (struct sockaddr *)&(server->dstaddr), server->dstaddrlen);
+                if (ret < 0) {
+                    LM_ERR("sendto() failed with errno=%d (%s)\n", errno, strerror(errno));
+                }
+                server = server->next;
+            }
+            break;
+        default:
+            /* send request to all servers */
+            server = server_list->head;
+            while (server) {
+                LM_DBG("sending request to '%s:%d'\n", server->host, server->port);
+                ret=sendto(server->sock, buf, reqlen, MSG_DONTWAIT, (struct sockaddr *)&(server->dstaddr), server->dstaddrlen);
+                if (ret < 0) {
+                    LM_ERR("sendto() failed with errno=%d (%s)\n", errno, strerror(errno));
+                }
+                server = server->next;
+            }
+            break;
+    }
+
 	/* wait for response */
 	for (;;) {
 		if (gettimeofday(&tnow, NULL) != 0) {
@@ -267,13 +341,48 @@ static int pdb_query(struct sip_msg *_msg, struct multiparam_t *_number, struct
 		ret=poll(server_list->fds, server_list->nserver, timeout-td);
 		for (i=0; i<server_list->nserver; i++) {
 			if (server_list->fds[i].revents & POLLIN) {
-				if (recv(server_list->fds[i].fd, buf, NETBUFSIZE, MSG_DONTWAIT) > 0) { /* do not block - just in case select/poll was wrong */
-					buf[NETBUFSIZE] = '\0';
-					if (strncmp(buf, number.s, number.len) == 0) {
-						_id = (short int *)&(buf[reqlen]);
-						carrierid=ntohs(*_id); /* convert to host byte order */
-						goto found;
-					}
+				if ((bytes_received = recv(server_list->fds[i].fd, buf,  sizeof(struct pdb_msg), MSG_DONTWAIT)) > 0) { /* do not block - just in case select/poll was wrong */
+                    switch (PDB_VERSION) {
+                        case PDB_VERSION_1:
+                            memcpy(&msg, buf, bytes_received);
+                            pdb_msg_dbg(msg, "Kamailio pdb client receives:");
+
+                            _id = (short int *)&(msg.hdr.id); /* make gcc happy */
+                            msg.hdr.id = ntohs(*_id);
+
+                            switch (msg.hdr.code) {
+                                case PDB_CODE_OK:
+                                    msg.bdy.payload[sizeof(struct pdb_bdy) - 1] = '\0';
+                                    if (strcmp(msg.bdy.payload, number.s) == 0) {
+                                        _id = (short int *)&(msg.bdy.payload[reqlen]); /* make gcc happy */
+                                        carrierid=ntohs(*_id); /* convert to host byte order */
+                                        goto found;
+                                    }
+                                    break;
+                                case PDB_CODE_NOT_NUMBER:
+                                    LM_WARN("Number %s has letters in it\n", number.s);
+                                    carrierid = 0;
+                                    goto found;
+                                case PDB_CODE_NOT_FOUND:
+                                    LM_WARN("Number %s pdb_id not found\n", number.s);
+                                    carrierid = 0;
+                                    goto found;
+                                default:
+                                    LM_WARN("Invalid code %d received\n", msg.hdr.code);
+                                    carrierid = 0;
+                                    goto found;
+                            }
+
+                            break;
+                        default:
+                            buf[sizeof(struct pdb_msg) - 1] = '\0';
+                            if (strncmp(buf, number.s, number.len) == 0) {
+                                _id = (short int *)&(buf[reqlen]);
+                                carrierid=ntohs(*_id); /* convert to host byte order */
+                                goto found;
+                            }
+                            break;
+                    }
 				}
 			}
 			server_list->fds[i].revents = 0;
@@ -664,7 +773,10 @@ static int mod_init(void)
 		shm_free(active);
 		return -1;
 	}
-	return 0;
+
+    global_id = (uint16_t*)shm_malloc(sizeof(uint16_t));
+
+    return 0;
 }
 
 static int child_init (int rank)

utils/pdbt/Makefile

@@ -1,8 +1,8 @@
 .phony: all clean install
 
 header=common.h carrier.h dt.h dtm.h pdb_server_backend.h log.h
-obj=dt.o dtm.o carrier.o pdb_server_backend.o log.o
-pdb_server_obj=pdb_server_backend.o dtm.o log.o
+obj=dt.o dtm.o carrier.o pdb_server_backend.o log.o common.o
+pdb_server_obj=pdb_server_backend.o dtm.o log.o common.o
 cflags=-Wall -O2 -g
 # -march=x86-64
 extdep=Makefile

utils/pdbt/common.c

@@ -0,0 +1,43 @@
+#include "common.h"
+#include "log.h"
+#include <string.h>
+
+void pdb_msg_dbg(struct pdb_msg msg) {
+    int i;
+
+    LERR("version = %d\n", msg.hdr.version);
+    LERR("type = %d\n", msg.hdr.type);
+    LERR("code = %d\n", msg.hdr.code);
+    LERR("id = %d\n", msg.hdr.id);
+    LERR("len = %d\n", msg.hdr.length);
+    LERR("payload = ");
+    for (i = 0; i < msg.hdr.length - sizeof(msg.hdr); i++) {
+        LERR("%02X ", msg.bdy.payload[i]);
+    }
+    LERR("\n");
+
+    return ;
+}
+
+int pdb_msg_format_send(struct pdb_msg *msg,
+                        uint8_t version, uint8_t type,
+                        uint8_t code, uint16_t id,
+                        char *payload, uint16_t payload_len)
+{
+    msg->hdr.version    = version;
+    msg->hdr.type       = type;
+    msg->hdr.code       = code;
+    msg->hdr.id         = id;
+
+    if (payload == NULL) {
+        /* just ignore the NULL buff (called when just want to set the len) */
+        msg->hdr.length     = sizeof(struct pdb_hdr);
+        return 0;
+    } else {
+        msg->hdr.length     = sizeof(struct pdb_hdr) + payload_len;
+        memcpy(msg->bdy.payload, payload, payload_len);
+        return 0;
+    }
+
+    return 0;
+}

utils/pdbt/common.h

@@ -41,16 +41,60 @@
 #define OTHER_CARRIERID 1000
 #define MAX_CARRIERID 1000
 #define NULL_CARRIERID -1001
+#define PAYLOADSIZE 256
+
 
 #define IS_VALID_PDB_CARRIERID(id) ((id>=MIN_PDB_CARRIERID) && (id<=MAX_PDB_CARRIERID))
 #define IS_VALID_CARRIERID(id) ((id>=MIN_PDB_CARRIERID) && (id<=MAX_CARRIERID))
 
+#define PDB_VERSION     1
 
 
 
 typedef int16_t carrier_t;
 
-
-
+enum __attribute__((packed)) pdb_versions {
+    PDB_VERSION_1 = 1,
+    PDB_VERSION_MAX
+};
+
+enum __attribute__((packed)) pdb_types {
+    PDB_TYPE_REQUEST_ID = 0,    /* request pdb type */
+    PDB_TYPE_REPLY_ID,          /* reply pdb type */
+    PDB_TYPE_MAX
+};
+
+enum __attribute__((packed)) pdb_codes {
+    PDB_CODE_DEFAULT = 0,   /* for request */
+    PDB_CODE_OK,            /* for response - OK */
+    PDB_CODE_NOT_NUMBER,    /* for response - letters found in the number */
+    PDB_CODE_NOT_FOUND,     /* for response - no pdb_id found for the number */
+    PDB_CODE_MAX
+};
+
+struct __attribute__((packed)) pdb_hdr {
+    uint8_t version;
+    uint8_t type;
+    uint8_t code;
+    uint8_t length;
+    uint16_t id;
+};
+
+struct __attribute__((packed)) pdb_bdy {
+    char payload[PAYLOADSIZE];
+};
+
+struct __attribute__((packed)) pdb_msg {
+    struct pdb_hdr hdr;
+    struct pdb_bdy bdy;
+};
+
+
+
+void pdb_msg_dbg (struct pdb_msg msg);
+int pdb_msg_format_send(struct pdb_msg *msg,
+                            uint8_t version, uint8_t type,
+                            uint8_t code, uint16_t id,
+                            char *payload, uint16_t payload_len);
 
 #endif

utils/pdbt/docs/network_protocol.txt

@@ -14,3 +14,40 @@ Possible values for the search request:
   * 1-999 the number was found and the result represents its carrier ID
   * 1000: the number could be found, but its owned from a carriers which is
     not interesting for us and belongs to the "other carrier" group
+
+
+From PDB_VERSION_1 onwards the pdb request and reply looks like this:
+
+    +-------+----+----+------+--+-------+
+    |          header           |  body |
+    +-------+----+----+------+--+-------+
+    |version|type|code|length|id|payload|
+    +-------+----+----+------+--+-------+
+
+Version
+    Current version is 1 (0x01).
+
+Type
+    PDB_TYPE_REQUEST_ID = 0,    /* request pdb type */
+    PDB_TYPE_REPLY_ID,          /* reply pdb type */
+
+Code
+    PDB_CODE_DEFAULT = 0,   /* for request */
+    PDB_CODE_OK,            /* for response - OK */
+    PDB_CODE_NOT_NUMBER,    /* for response - letters found in the number */
+    PDB_CODE_NOT_FOUND,     /* for response - no pdb_id found for the number */
+
+Length
+    The length of the whole message
+
+Id
+    A pdb_msg id which might be used for asynchronous queries.
+
+Payload
+    Request number including '\0' for request.
+    Request number including '\0' and pdb_id for reply.
+
+Backwards compatibility with the old msg protocol is kept when the first byte
+received is different from the known versions ('0x01' for now)
+
+For more info about the data structures used, see common.h file.

utils/pdbt/log.c

@@ -55,10 +55,10 @@ void destroy_log(void) {
 
 
 
-void log_stdout(char * format, va_list ap)
+void log_stderr(char * format, va_list ap)
 {
-	vfprintf(stdout, format, ap);
-	fflush(stdout);
+	vfprintf(stderr, format, ap);
+	fflush(stderr);
 }
 
 
@@ -70,7 +70,7 @@ void pdb_log(int priority, char * format, ...) {
 	if (priority<=log_level) {
 		va_start(ap, format);
 		if (use_syslog) vsyslog(priority, format, ap);
-		else log_stdout(format, ap);
+		else log_stderr(format, ap);
 		va_end(ap);
 	}
 }

utils/pdbt/log.h

@@ -43,7 +43,12 @@ void pdb_log(int priority, char * format, ...);
 #define LINFO(fmt, args...) pdb_log(LOG_INFO, fmt, ## args)
 #define LDEBUG(fmt, args...) pdb_log(LOG_DEBUG, fmt, ## args)
 
+/* several shell exit codes for the application pdbt */
 
-
+#define PDB_OK 0              /* Everything ok */
+#define PDB_USE_ERROR 1       /* Wrong usage of application (unknown command, file not found, etc.) */
+#define PDB_NOT_IN_PDB 2      /* A queried number is not in the pdb */
+#define PDB_TIMEOUT 3         /* A timeout (server not responding) occured */
+#define PDB_OTHER 4           /* Another application error occured */
 
 #endif

utils/pdbt/pdb_server.c

@@ -65,7 +65,29 @@ void print_usage(char *program) {
 	LINFO("    -h: Print this help.\n");
 }
 
+int pdb_msg_server_send(int so, char *buf, size_t answerlen, struct sockaddr *fromaddr, socklen_t fromaddrlen)
+{
+	ssize_t bytes_sent;
+	int try = 0;
+	again:
+		bytes_sent = sendto(so, buf, answerlen, 0, fromaddr, fromaddrlen);
+		if (bytes_sent < 3) {
+			if ((errno == EINTR) && (try < 3)) {
+				try++;
+				LERR("sendto() failed - trying again. errno=%d (%s)\n", errno, strerror(errno));
+				goto again;
+			}
+			LERR("sendto() failed with errno=%d (%s)\n", errno, strerror(errno));
+			if ((errno==EAGAIN)||(errno==EINTR)||(errno==EWOULDBLOCK)) return 0;
+			return -1;
+		}
+		if (bytes_sent != answerlen) {
+			LERR("cannot send the whole answer (%ld/%ld).\n", (long int)bytes_sent, (long int)answerlen);
+			return 0;
+		}
 
+    return 0;
+}
 
 
 /*
@@ -77,60 +99,101 @@ void print_usage(char *program) {
 */
 int udp_server(int so)
 {
+    struct pdb_msg msg;
 	struct sockaddr fromaddr;
 	socklen_t fromaddrlen;
-	size_t answerlen;
+	size_t answerlen = 0;
 	ssize_t bytes_received;
-	ssize_t bytes_sent;
 	carrier_t carrierid;
-	char buf[NETBUFSIZE+1+sizeof(carrierid)]; /* additional space for '\0' termination and carrier */
+	char buf[sizeof(struct pdb_msg)];
 	int i;
-	int try;
 
 	for (;;) {
 		fromaddrlen = sizeof(fromaddr);
-		bytes_received = recvfrom(so, buf, NETBUFSIZE, 0, &fromaddr, &fromaddrlen);
+		bytes_received = recvfrom(so, buf, sizeof(struct pdb_msg), 0, &fromaddr, &fromaddrlen);
 		if (bytes_received<0) {
-      LERR("recvfrom() failed with errno=%d (%s)\n", errno, strerror(errno));
+            LERR("recvfrom() failed with errno=%d (%s)\n", errno, strerror(errno));
 			if ((errno==EAGAIN)||(errno==EINTR)||(errno==EWOULDBLOCK)) continue;
 			return -1;
 		}
-		
-		/* take only digits */
-		i=0;
-		while ((i<bytes_received) && (buf[i]>='0') && (buf[i]<='9')) i++;
-		buf[i]=0; /* terminate string */
-		i++;
 
-		carrierid=lookup_number(buf);
-		
-		/* convert to network byte order*/
-		carrierid=htons(carrierid);
+        switch (buf[0]) {
+            case PDB_VERSION_1:
+                /* get received bytes */
+                memcpy(&msg, buf, bytes_received);
+//                pdb_msg_dbg(msg);
+                short int *_id = (short int *)&(msg.hdr.id); /* make gcc happy */
+                msg.hdr.id = ntohs(*_id);
 
-		/* append carrier id to answer */
-		memcpy(&(buf[i]), &carrierid, sizeof(carrierid));
-		answerlen=i+sizeof(carrierid);
-		
-		try=0;
-	again:
-		bytes_sent = sendto(so, buf, answerlen, 0, &fromaddr, fromaddrlen);
-		if (bytes_sent < 3) {
-			if ((errno==EINTR) && (try<3)) {
-				try++;
-				LERR("sendto() failed - trying again. errno=%d (%s)\n", errno, strerror(errno));
-				goto again;
-			}
-			LERR("sendto() failed with errno=%d (%s)\n", errno, strerror(errno));
-			if ((errno==EAGAIN)||(errno==EINTR)||(errno==EWOULDBLOCK)) continue;
-			return -1;
-		}
-		if (bytes_sent != answerlen) {
-			LERR("cannot send the whole answer (%ld/%ld).\n", (long int)bytes_sent, (long int)answerlen);
-			continue;
-		}
-	}
+                i = 0;
+                while (i < strlen(msg.bdy.payload)) {
+                    if (msg.bdy.payload[i] < '0' || msg.bdy.payload[i] > '9') {
+                        pdb_msg_format_send(&msg, PDB_VERSION_1, PDB_TYPE_REPLY_ID, PDB_CODE_NOT_NUMBER, htons(msg.hdr.id), NULL, 0);
+                        goto msg_send;
+                    }
+                    i++;
+                }
+                /* lookup pdb_id */
+                carrierid=lookup_number(msg.bdy.payload);
 
-	return 0;
+                /* check if not found pdb_id */
+                if (carrierid == 0) {
+                    pdb_msg_format_send(&msg, PDB_VERSION_1, PDB_TYPE_REPLY_ID, PDB_CODE_NOT_FOUND, htons(msg.hdr.id), NULL, 0);
+                    goto msg_send;
+                }
+
+                /* convert to network byte order*/
+                carrierid = htons(carrierid);
+
+                /* prepare the message payload to be sent
+                 * add the number string and append the carrier id
+                 */
+                memcpy(buf, msg.bdy.payload, msg.hdr.length - sizeof(msg.hdr));
+                memcpy(buf + msg.hdr.length - sizeof(msg.hdr), &carrierid, sizeof(carrierid));
+
+                /* all ok, send pdb_msg with pdb_id in payload */
+                pdb_msg_format_send(&msg, PDB_VERSION_1, PDB_TYPE_REPLY_ID, PDB_CODE_OK, htons(msg.hdr.id), buf, msg.hdr.length - sizeof(msg.hdr) + sizeof(carrierid));
+                goto msg_send;
+
+                break;
+
+            /* old pdb version; no pdb_msg used */
+            default:
+                /* take only digits */
+                i=0;
+                while ((i<bytes_received) && (buf[i]>='0') && (buf[i]<='9')) i++;
+                buf[i]=0; /* terminate string */
+                i++;
+
+                /* lookup pdb_id */
+                carrierid=lookup_number(buf);
+
+                /* convert to network byte order*/
+                carrierid=htons(carrierid);
+
+                /* append carrier id to answer */
+                memcpy(&(buf[i]), &carrierid, sizeof(carrierid));
+                answerlen=i+sizeof(carrierid);
+                goto buf_send;
+
+                break;
+        }
+
+msg_send:
+//        pdb_msg_dbg(msg);
+        if (pdb_msg_server_send(so, (char*)&msg, msg.hdr.length, &fromaddr, fromaddrlen) < 0) {
+            return -1;
+        }
+        continue;
+
+buf_send:
+        if (pdb_msg_server_send(so, buf, answerlen, &fromaddr, fromaddrlen)) {
+            return -1;
+        }
+        continue;
+    }
+
+	return -1;
 }
 
 

utils/pdbt/pdbt.c

@@ -39,14 +39,10 @@
 #include "log.h"
 
 
+/* incremented after request sent to the server */
+uint16_t id = 0;
 
-
-#define NETBUFSIZE 200
-
-
-
-
-typedef void (*query_func_t)(char *number, char *comment, void *data);
+typedef int (*query_func_t)(char *number, char *comment, void *data);
 
 
 
@@ -136,13 +132,11 @@ int file_query(char *filename, query_func_t query_func, void *data) {
 	else fp = fopen(filename, "r");
 	if (fp == NULL) {
 		LERR("cannot open file '%s'\n", filename);
-		return -1;
+		return -PDB_USE_ERROR;
 	}
 	while ((read = getline(&line, &len, fp)) != -1) {
 		p=line;
 		while ((*p >= '0') && (*p <= '9') && (p < line+len)) p++;
-		*p='\0';
-		p++;
 		comment=p;
 		while ((*p >= 32) && (p < line+len)) p++;
 		*p='\0';
@@ -150,7 +144,7 @@ int file_query(char *filename, query_func_t query_func, void *data) {
 	}
 	if (line) free(line);
 	fclose(fp);
-	return 0;
+	return -PDB_OK;
 }
 
 
@@ -184,6 +178,11 @@ int import_csv(struct dt_node_t *root, char *filename) {
 	while ((read = getline(&line, &len, fp)) != -1) {
 		carrier_str=line;
 		prefix=strsep(&carrier_str, ";");
+		if ( carrier_str == NULL ) {
+			LWARNING("line %ld: no delimiter `;' found, ignoring line.\n", n); 
+			n++;
+			continue;
+		}
 		ret=strtol(carrier_str, NULL, 10);
 		if (!IS_VALID_PDB_CARRIERID(ret)) {
 			LWARNING("invalid carrier '%s' in line %ld.\n", carrier_str, (long int)n);
@@ -451,12 +450,16 @@ int merge_carrier(struct dt_node_t *root, int keep_carriers_num, carrier_t keep_
  */
 int query_udp(char *number, int timeout, struct pollfd *pfds, struct sockaddr_in *dstaddr, socklen_t dstaddrlen)
 {
+    struct pdb_msg msg;
 	struct timeval tstart, tnow;
 	short int carrierid;
-	char buf[NETBUFSIZE+1+sizeof(carrierid)];
+	char buf[sizeof(struct pdb_msg)];
 	size_t reqlen;
 	int ret, nflush;
 	long int td;
+    ssize_t bytes_received;
+    short int * idptr;
+
 
 	if (gettimeofday(&tstart, NULL) != 0) {
 		LERR("gettimeofday() failed with errno=%d (%s)\n", errno, strerror(errno));
@@ -465,54 +468,108 @@ int query_udp(char *number, int timeout, struct pollfd *pfds, struct sockaddr_in
 
 	/* clear recv buffer */
 	nflush = 0;
-	while (recv(pfds->fd, buf, NETBUFSIZE, MSG_DONTWAIT) > 0) {
+	while (recv(pfds->fd, buf, sizeof(struct pdb_msg), MSG_DONTWAIT) > 0) {
 		nflush++;
 		if (gettimeofday(&tnow, NULL) != 0) {
 			LERR("gettimeofday() failed with errno=%d (%s)\n", errno, strerror(errno));
-			return -1;
+			return -PDB_OTHER;
 		}
 		td=(tnow.tv_usec-tstart.tv_usec+(tnow.tv_sec-tstart.tv_sec)*1000000) / 1000;
 		if (td > timeout) {
 			LWARNING("exceeded timeout while flushing recv buffer.\n");
-			return -1;
+			return -PDB_TIMEOUT;
 		}
 	}
 	
 	/* prepare request */
 	reqlen = strlen(number) + 1; /* include null termination */
-	if (reqlen > NETBUFSIZE) {
+	if (reqlen > sizeof(struct pdb_bdy)) {
 		LERR("number too long '%s'.\n", number);
-		return -1;
+		return -PDB_USE_ERROR;
 	}
 	strcpy(buf, number);
 
-	/* send request to all servers */
-	ret=sendto(pfds->fd, buf, reqlen, MSG_DONTWAIT, (struct sockaddr *)dstaddr, dstaddrlen);
-	if (ret < 0) {
-		LERR("sendto() failed with errno=%d (%s)\n", errno, strerror(errno));
-		return -1;
-	}
-		
+    switch (PDB_VERSION) {
+        case PDB_VERSION_1:
+            pdb_msg_format_send(&msg, PDB_VERSION, PDB_TYPE_REQUEST_ID, PDB_CODE_DEFAULT, htons(id), buf, reqlen);
+//            pdb_msg_dbg(msg);
+
+            /* increment msg id for the next request */
+            id++;
+
+            /* send request to all servers */
+            ret=sendto(pfds->fd, (struct pdb_msg*)&msg, msg.hdr.length, MSG_DONTWAIT, (struct sockaddr *)dstaddr, dstaddrlen);
+            if (ret < 0) {
+                LERR("sendto() failed with errno=%d (%s)\n", errno, strerror(errno));
+                return -1;
+            }
+            break;
+        default:
+            /* send request to all servers */
+            ret=sendto(pfds->fd, buf, reqlen, MSG_DONTWAIT, (struct sockaddr *)dstaddr, dstaddrlen);
+            if (ret < 0) {
+                LERR("sendto() failed with errno=%d (%s)\n", errno, strerror(errno));
+                return -PDB_OTHER;
+            }
+            break;
+    }
+
 	/* wait for response */
 	for (;;) {
 		if (gettimeofday(&tnow, NULL) != 0) {
 			LERR("gettimeofday() failed with errno=%d (%s)\n", errno, strerror(errno));
-			return -1;
+			return -PDB_OTHER;
 		}
 		td=(tnow.tv_usec-tstart.tv_usec+(tnow.tv_sec-tstart.tv_sec)*1000000) / 1000;
 		if (td > timeout) {
 			LWARNING("exceeded timeout while waiting for response.\n");
-			return -1;
+			return -PDB_TIMEOUT;
 		}
-		
+
 		ret=poll(pfds, 1, timeout-td);
 		if (pfds->revents & POLLIN) {
-			if (recv(pfds->fd, buf, NETBUFSIZE, MSG_DONTWAIT) > 0) { /* do not block - just in case select/poll was wrong */
-				buf[NETBUFSIZE] = '\0';
-				if (strcmp(buf, number) == 0) {
-					carrierid=ntohs(*((short int *)&(buf[reqlen]))); /* convert to host byte order */
-					goto found;
-				}
+			if ((bytes_received = recv(pfds->fd, buf, sizeof(struct pdb_msg), MSG_DONTWAIT)) > 0) { /* do not block - just in case select/poll was wrong */
+                switch (PDB_VERSION) {
+                    case PDB_VERSION_1:
+                        memcpy(&msg, buf, bytes_received);
+//                        pdb_msg_dbg(msg);
+                          idptr = (short int *)&(msg.hdr.id); /* make gcc happy */
+                          msg.hdr.id = ntohs(*idptr);
+
+
+                        switch (msg.hdr.code) {
+                            case PDB_CODE_OK:
+                                msg.bdy.payload[sizeof(struct pdb_bdy) - 1] = '\0';
+                                if (strcmp(msg.bdy.payload, number) == 0) {
+                                    idptr = (short int *)&(msg.bdy.payload[reqlen]); /* make gcc happy */
+                                    carrierid=ntohs(*idptr); /* convert to host byte order */
+                                    goto found;
+                                }
+                                break;
+                            case PDB_CODE_NOT_NUMBER:
+			                    LERR("Number %s has letters in it\n", number);
+                                carrierid = 0;
+                                goto found;
+                            case PDB_CODE_NOT_FOUND:
+			                    LERR("Number %s pdb_id not found\n", number);
+                                carrierid = 0;
+                                goto found;
+                            default:
+			                    LERR("Invalid code %d received\n", msg.hdr.code);
+                                carrierid = 0;
+                                goto found;
+                        }
+
+                        break;
+                    default:
+                        buf[sizeof(struct pdb_msg) - 1] = '\0';
+                        if (strcmp(buf, number) == 0) {
+                            idptr = (short int *)&(buf[reqlen]); /* make gcc happy */
+                            carrierid=ntohs(*idptr); /* convert to host byte order */
+                            goto found;
+                        }
+                        break;
+                }
 			}
 		}
 		pfds->revents = 0;
@@ -538,7 +595,7 @@ struct server_query_data_t {
 
 
 
-void query_mmap(char *number, char *comment, void *data) {
+int query_mmap(char *number, char *comment, void *data) {
 	int nmatch;
 	carrier_t carrierid;
 	struct dtm_node_t *mroot = (struct dtm_node_t *)data;
@@ -547,31 +604,39 @@ void query_mmap(char *number, char *comment, void *data) {
 
 	if (nmatch<=0) {
 		LINFO("%s:%s:%ld:%s\n", number, comment, (long int)carrierid, "not allocated, probably old");
+		return -PDB_NOT_IN_PDB;
 	}
 	else {
 		LINFO("%s:%s:%ld:%s\n", number, comment, (long int)carrierid, carrierid2name(carrierid));
 		/* LINFO("%s: found: carrier_id=%ld, carrier_name='%s', nmatch=%ld, comment='%s'\n", number, (long int)carrierid, carrierid2name(carrierid), (long int)nmatch, comment);
 		*/
+		return -PDB_OK;
 	}
 }
 
 
 
 
-void query_server(char *number, char *comment, void *data) {
+int query_server(char *number, char *comment, void *data) {
 	carrier_t carrierid;
 	struct server_query_data_t *sdata = (struct server_query_data_t *)data;
+	int result = 0;
 
 	carrierid = query_udp(number, sdata->timeout, &(sdata->pfds), &(sdata->dstaddr), sdata->dstaddrlen);
 
 	if (carrierid<=0) {
-		LINFO("%s: not_found: comment='%s'\n", number, comment);
+		LINFO("%s: not_found: comment='%s', result=%d\n", number, comment, carrierid);
+		if (carrierid < 0) {
+			result = carrierid; 
+		} else {
+			result = PDB_NOT_IN_PDB; 
+		}
 	}
 	else {
 		LINFO("%s:%ld:%s\n", number, (long int)carrierid, carrierid2name(carrierid));
-		/* LINFO("%s: found: carrier_id=%ld, carrier_name='%s', comment='%s'\n", number, (long int)carrierid, carrierid2name(carrierid), comment);
-		*/
+		result = PDB_OK;
 	}
+	return result; 
 }
 
 
@@ -604,6 +669,7 @@ int main(int argc, char *argv[]) {
 
 	char *id_str;
 	long int ret;
+	int exit_status = PDB_OK;
 
 	sdata.timeout=500;
 
@@ -782,7 +848,7 @@ int main(int argc, char *argv[]) {
 			if (hp == NULL) {
 				LERR("gethostbyname(%s) failed with h_errno=%d.\n", host_str, h_errno);
 				close(sockfd);
-				return -1;
+				exit (PDB_USE_ERROR);
 			}
 			memcpy(&sdata.dstaddr.sin_addr.s_addr, hp->h_addr, hp->h_length);
 			sdata.dstaddrlen=sizeof(sdata.dstaddr);
@@ -792,19 +858,30 @@ int main(int argc, char *argv[]) {
 
 			if (query_file==NULL) {
 				LINFO("\nprocessing command line parameters...\n");
+				if ( optind+1 >= argc) {
+					exit_status = PDB_USE_ERROR;
+				}
 				for (n=optind+1; n<argc; n++) {
-					query_server(argv[n], "", &sdata);
+					int result; 
+					result = query_server(argv[n], "", &sdata); 
+					if ( result != 0) {
+						exit_status = -result; 
+					}
 				}
 			}
 			else {
-				file_query(query_file, query_server, &sdata);
+				int result; 
+				result = file_query(query_file, query_server, &sdata);
+				if ( result != 0) {
+					exit_status = -result; 
+				}
 			}
 		}
 		else {
 			mroot=dtm_load(mmap_file);
 			if (mroot == NULL) {
 				LERR("cannot load '%s'.\n", mmap_file);
-				return -1;
+				exit(PDB_USE_ERROR);
 			}
 			
 			if (query_file==NULL) {
@@ -814,14 +891,18 @@ int main(int argc, char *argv[]) {
 				}
 			}
 			else {
-				file_query(query_file, query_mmap, mroot);
+				int result; 
+				result = file_query(query_file, query_mmap, mroot);
+				if ( result != 0) {
+					exit_status = -result; 
+				}
 			}
 		}
 	}
 	else {
 		LERR("invalid command '%s'.\n", argv[optind]);
-		return 1;
+		exit(PDB_USE_ERROR);
 	}
 
-	return 0;
+	exit (exit_status);
 }

utils/pdbt/scripts/get_carrier_names_germany.pl

@@ -0,0 +1,60 @@
+#!/usr/bin/perl
+use utf8;
+use LWP::UserAgent;
+use HTTP::Cookies;
+
+sub main
+{
+    # Create the fake browser (user agent).
+    my $ua = LWP::UserAgent->new();
+
+    # Pretend to be Internet Explorer.
+    $ua->agent("Windows IE 7");
+    # or maybe .... $ua->agent("Mozilla/8.0");
+
+    # Get some HTML.
+    my $response = $ua->get('http://www.bundesnetzagentur.de/cln_1421/DE/Sachgebiete/Telekommunikation/Unternehmen_Institutionen/Nummerierung/Technische%20Nummern/Portierungskennungen/VerzeichnisPortKenn_Basepage.html?nn=268376');
+
+    unless($response->is_success) {
+        print "Error: " . $response->status_line;
+    }
+
+    # Let's save the output.
+    my $file = $response->decoded_content;
+    utf8::encode($file);
+    @pieces=split('\<tbody\>', $file);
+    @pieces2=split('\</tbody\>', $pieces[1]);
+    @linii=split('\</tr\>', $pieces2[0]);
+    foreach(@linii)
+    {
+        my($first, $rest) = split(/>/, $_, 2);
+        @tds=split('/td><td', $rest);
+        @names=split('>', $tds[0], 2);
+        my $name=$names[1];
+        $name =~ s/<p>//;
+        $name =~ s/<p>//;
+        $name =~ s/<\/p>//;
+        $name =~ s/<\/p>//;
+        $name =~ s/<//;
+        $name =~ s/\n//;
+        $name =~ s/br\/>//;
+        $name =~ s/br\/>//;
+        $name =~ s/<//;
+        $name =~ s/<//;
+        $name =~ s/\n//;
+        chomp($name);
+        @tds2=split('>', $tds[1], 2);
+        @tds3=split('</', $tds2[1]);
+        @tds4=split('<br/>', $tds3[0]);
+        foreach(@tds4)
+        {
+            $_ =~ s/^\n//;
+            if ($_ =~ /^D/)
+            {
+                    $number=substr($_,0,4);
+                    print "$number $name\n";
+            }
+        }
+    }
+}
+main();

utils/pdbt/scripts/get_carrier_names_germany.sh

@@ -23,14 +23,15 @@
 # the 'Bundesnetzagentur' and convert this into the format which the pdbt tool
 # understands.
 
-url="http://www.bundesnetzagentur.de/cln_1912/DE/Sachgebiete/Telekommunikation/RegulierungTelekommunikation/Nummernverwaltung/TechnischeNummern/Portierungskennung/VerzeichnisPortKenn_Basepage.html"
+#old_url="http://www.bundesnetzagentur.de/cln_1912/DE/Sachgebiete/Telekommunikation/RegulierungTelekommunikation/Nummernverwaltung/TechnischeNummern/Portierungskennung/VerzeichnisPortKenn_Basepage.html"
+url="http://www.bundesnetzagentur.de/cln_1421/DE/Sachgebiete/Telekommunikation/Unternehmen_Institutionen/Nummerierung/Technische%20Nummern/Portierungskennungen/VerzeichnisPortKenn_Basepage.html?nn=268376#Inhalt"
 
-# fix LOCALE problem during filtering 
+# fix LOCALE problem during filtering
 export LANG="C"
 
-wget -O - "$url" | recode latin1..utf8 | sed 's/^*.Verzeichnis der Portierungskennungen//' | awk '/<tbody>/, /<\/tbody>/' | tr -d '\r' | tr '\n' '@' | sed 's/<\/table>.*$//' | sed 's/<\/tbody>.*$//'
+wget -O - "$url" | recode latin1..utf8 | sed 's/^*.Verzeichnis der Portierungskennungen//' | awk '/<tbody>/, /<\/tbody>/' | tr -d '\r' | tr '\n' '@' | sed 's/<\/table>.*$//' | sed 's/<\/tbody>.*$//' | awk -F "</td" -v RS="</tr" '{ gsub(/.*>/,"",$1) gsub(/.*>/,"",$2); if ( $1 != "") { printf "%s %s\n",$2,$1 } }'
 
 # probably also possible to use this:
 # http://www.bundesnetzagentur.de/cae/servlet/contentblob/156772/publicationFile/8492/KonsolidiertesVerzPortierungsk.zip
 # main page (for reference):
-# http://www.bundesnetzagentur.de/cln_1932/DE/Sachgebiete/Telekommunikation/RegulierungTelekommunikation/Nummernverwaltung/TechnischeNummern/Portierungskennung/KonsolidiertesVerzPortKenn_Basepage.html?nn=120380
+# http://www.bundesnetzagentur.de/cln_1932/DE/Sachgebiete/Telekommunikation/RegulierungTelekommunikation/Nummernverwaltung/TechnischeNummern/Portierungskennung/KonsolidiertesVerzPortKenn_Basepage.html?nn=120380