utils: http_query() now handles large replies properly.

The callback function passed to CURL (CURLOPT_WRITEFUNCTION) malloc'd one buffer on the presumption that it would be called once. This is not necessarily true for large replies, so the callback was modified to realloc() the buffer upward as needed until all data chunks were retrieved.

modules/utils/functions.c

@@ -46,24 +46,30 @@
 /* 
  * curl write function that saves received data as zero terminated
  * to stream. Returns the amount of data taken care of.
+ *
+ * This function may be called multiple times for larger responses, 
+ * so it reallocs + concatenates the buffer as needed.
  */
-size_t write_function( void *ptr, size_t size, size_t nmemb, void *stream)
+size_t write_function( void *ptr, size_t size, size_t nmemb, void *stream_ptr)
 {
-    /* Allocate memory and copy */
-    char* data;
+    http_res_stream_t *stream = (http_res_stream_t *) stream_ptr;
+
+    stream->buf = (char *) pkg_realloc(stream->buf, stream->curr_size + 
+				(size * nmemb) + 1);
 
-    data = (char*)pkg_malloc((size* nmemb) + 1);
-    if (data == NULL) {
+    if (stream->buf == NULL) {
 	LM_ERR("cannot allocate memory for stream\n");
 	return CURLE_WRITE_ERROR;
     }
 
-    memcpy(data, (char*)ptr, size* nmemb);
-    data[nmemb] = '\0';
-        
-    *((char**) stream) = data;
-    
-    return size* nmemb;
+    memcpy(&stream->buf[stream->pos], (char *) ptr, (size * nmemb));
+
+    stream->curr_size += ((size * nmemb) + 1);
+    stream->pos += (size * nmemb);
+
+    stream->buf[stream->pos + 1] = '\0';
+
+    return size * nmemb;
  }
 
 
@@ -77,12 +83,14 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
     CURLcode res;  
     str value, post_value;
     char *url, *at, *post;
-    char* stream;
+    http_res_stream_t stream;
     long stat;
     pv_spec_t *dst;
     pv_value_t val;
     double download_size;
 
+    memset(&stream, 0, sizeof(http_res_stream_t));
+
     if (fixup_get_svalue(_m, (gparam_p)_url, &value) != 0) {
 	LM_ERR("cannot get page value\n");
 	return -1;
@@ -129,7 +137,6 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
     curl_easy_setopt(curl, CURLOPT_NOSIGNAL, (long)1);
     curl_easy_setopt(curl, CURLOPT_TIMEOUT, (long)http_query_timeout);
 
-    stream = NULL;
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_function);
     curl_easy_setopt(curl, CURLOPT_WRITEDATA, &stream);
 
@@ -150,8 +157,8 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
 		}
 	
 		curl_easy_cleanup(curl);
-		if(stream)
-			pkg_free(stream);
+		if(stream.buf)
+			pkg_free(stream.buf);
 		return -1;
     }
 
@@ -159,14 +166,15 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
     if ((stat >= 200) && (stat < 500)) {
 	curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD, &download_size);
 	LM_DBG("http_query download size: %u\n", (unsigned int)download_size);
+
 	/* search for line feed */
-	at = memchr(stream, (char)10, download_size);
+	at = memchr(stream.buf, (char)10, download_size);
 	if (at == NULL) {
 	    /* not found: use whole stream */
-	    at = stream + (unsigned int)download_size;
+	    at = stream.buf + (unsigned int)download_size;
 	}
-	val.rs.s = stream;
-	val.rs.len = at - stream;
+	val.rs.s = stream.buf;
+	val.rs.len = at - stream.buf;
 	LM_DBG("http_query result: %.*s\n", val.rs.len, val.rs.s);
 	val.flags = PV_VAL_STR;
 	dst = (pv_spec_t *)_dst;
@@ -174,6 +182,6 @@ int http_query(struct sip_msg* _m, char* _url, char* _dst, char* _post)
     }
 	
     curl_easy_cleanup(curl);
-    pkg_free(stream);
+    pkg_free(stream.buf);
     return stat;
 }

modules/utils/utils.h

@@ -40,4 +40,10 @@ extern int http_query_timeout;
 extern db1_con_t *pres_dbh;
 extern db_func_t pres_dbf;
 
+typedef struct {
+	char		*buf;
+	size_t		curr_size;
+	size_t		pos;
+} http_res_stream_t;
+
 #endif /* UTILS_H */