try select

panda/src/downloader/downloader.cxx

@@ -0,0 +1,1096 @@
+// Filename: downloader.cxx
+// Created by:  mike (09Jan97)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+
+#include "downloader.h"
+#include "config_downloader.h"
+#include "urlSpec.h"
+#include "error_utils.h"
+#include "filename.h"
+
+#include <errno.h>
+#include <math.h>
+
+#if !defined(WIN32_VC)
+  #include <sys/time.h>
+  #include <netinet/in.h>
+  #include <arpa/inet.h>
+  #include <netdb.h>
+  #define SOCKET_ERROR -1
+#endif
+
+////////////////////////////////////////////////////////////////////
+// Defines
+////////////////////////////////////////////////////////////////////
+const int MAX_RECEIVE_BYTES = 16384;
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::Constructor
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+Downloader::
+Downloader(void) {
+  _frequency = downloader_frequency;
+  _byte_rate = (float) downloader_byte_rate;
+  _disk_write_frequency = downloader_disk_write_frequency;
+  nassertv(_frequency > 0 && _byte_rate > 0 && _disk_write_frequency > 0);
+  _receive_size = (ulong)(_byte_rate * _frequency);
+  _disk_buffer_size = _disk_write_frequency * _receive_size;
+  _buffer = new Buffer(_disk_buffer_size);
+
+  _connected = false;
+  _use_proxy = false;
+  // We need to flush after every write in case we're interrupted
+  _dest_stream.setf(ios::unitbuf, 0);
+  _current_status = NULL;
+  _recompute_buffer = false;
+
+  _tfirst = 0.0;
+  _tlast = 0.0;
+  _got_any_data = false;
+  _initiated = false;
+  _ever_initiated = false;
+  _TCP_stack_initialized = false;
+  _total_bytes_written = 0;
+  _total_bytes_requested = 0;
+  _total_bytes_requested = 0;
+
+  _file_size = 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::Destructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+Downloader::
+~Downloader() {
+  if (_connected)
+    disconnect_from_server();
+  _buffer.clear();
+  if (_initiated == true)
+    cleanup();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::connect_to_server_by_proxy
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+int Downloader::
+connect_to_server_by_proxy(const URLSpec &proxy, const string &server_name) {
+  if (connect_to_server(proxy.get_server(), proxy.get_port()) != EU_success) {
+    downloader_cat.error()
+      << "Downloader::connect_to_server_by_proxy() - could not connect to: "
+      << proxy << endl;
+    return EU_error_abort;
+  }
+  
+  _proxy_string = "http://";
+  _proxy_string += server_name;
+  _use_proxy = true;
+
+  return EU_success;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::connect_to_server
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+int Downloader::
+connect_to_server(const string &name, uint port) {
+
+#if defined(WIN32)
+  if (_TCP_stack_initialized == false) {
+    WSAData mydata;
+    int answer1 = WSAStartup(0x0101, &mydata);
+    if(answer1 != 0) {
+      downloader_cat.error()
+        << "Downloader::connect_to_server() - WSAStartup() - error: "
+        << handle_socket_error() << endl;
+      return EU_error_abort;
+    }
+    _TCP_stack_initialized = true;
+  }
+#endif
+
+  if (downloader_cat.is_debug())
+    downloader_cat.debug()
+      << "Downloader connecting to server: " << name << " on port: "
+      << port << endl;
+
+  _server_name = name;
+
+  _sin.sin_family = PF_INET;
+  _sin.sin_port = htons(port);
+  ulong addr = (ulong)inet_addr(name.c_str());
+  struct hostent *hp = NULL;
+
+  if (addr == INADDR_NONE) {
+    hp = gethostbyname(name.c_str());
+    if (hp != NULL)
+      (void)memcpy(&_sin.sin_addr, hp->h_addr, (uint)hp->h_length);
+    else {
+      downloader_cat.error()
+        << "Downloader::connect_to_server() - gethostbyname() failed on: "
+        << name.c_str() << " with error: "
+        << handle_socket_error() << endl;
+      return get_network_error();
+    }
+  } else
+    (void)memcpy(&_sin.sin_addr, &addr, sizeof(addr));
+
+  return connect_to_server();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::connect_to_server
+//       Access: Private
+//  Description:
+////////////////////////////////////////////////////////////////////
+int Downloader::
+connect_to_server(void) {
+  if (_connected == true)
+    return EU_success;
+
+  _socket = 0xffffffff;
+  _socket = socket(PF_INET, SOCK_STREAM, 0);
+  if (_socket == (int)0xffffffff) {
+    downloader_cat.error()
+      << "Downloader::connect_to_server() - socket failed: "
+      << handle_socket_error() << endl;
+    return get_network_error();
+  }
+
+  if (connect(_socket, (struct sockaddr *)&_sin, sizeof(_sin)) ==
+                                                        SOCKET_ERROR) {
+    downloader_cat.error()
+      << "Downloader::connect_to_server() - connect() failed: "
+      << handle_socket_error() << endl;
+    disconnect_from_server();
+    return get_network_error();
+  }
+
+  _connected = true;
+  return EU_success;
+}
+
+///////////////////////////////////////////////////////////////////
+//     Function: Downloader::disconnect_from_server
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+void Downloader::
+disconnect_from_server(void) {
+  if (downloader_cat.is_debug())
+    downloader_cat.debug()
+      << "Downloader disconnecting from server..." << endl;
+#if defined(WIN32)
+  (void)closesocket(_socket);
+#else
+  (void)close(_socket);
+#endif
+  _connected = false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::safe_send
+//       Access: Private
+//  Description:
+////////////////////////////////////////////////////////////////////
+int Downloader::
+safe_send(int socket, const char *data, int length, long timeout) {
+  if (length == 0) {
+    downloader_cat.error()
+      << "Downloader::safe_send() - requested 0 length send!" << endl;
+    return EU_error_abort;
+  }
+  int bytes = 0;
+  struct timeval tv;
+  tv.tv_sec = timeout;
+  tv.tv_usec = 0;
+  fd_set wset;
+  FD_ZERO(&wset);
+  while (bytes < length) {
+    FD_SET(socket, &wset);
+    int sret = select(socket + 1, NULL, &wset, NULL, &tv);
+    if (sret == 0) {
+      downloader_cat.error()
+        << "Downloader::safe_send() - select timed out after: "
+        << timeout << " seconds" << endl;
+      return EU_error_network_timeout;
+    } else if (sret == -1) {
+      downloader_cat.error()
+        << "Downloader::safe_send() - error: " << handle_socket_error()
+        << endl;
+      return get_network_error();
+    }
+    int ret = send(socket, data, length, 0);
+    if (ret > 0)
+      bytes += ret;
+    else {
+      downloader_cat.error()
+        << "Downloader::safe_send() - error: " << handle_socket_error()
+        << endl;
+      return get_network_error();
+    }
+  }
+  return EU_success;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::fast_receive
+//       Access: Private
+//  Description:
+////////////////////////////////////////////////////////////////////
+int Downloader::
+fast_receive(int socket, DownloadStatus *status, int rec_size) {
+  nassertr(status != NULL, EU_error_abort);
+  if (rec_size <= 0) {
+    downloader_cat.error()
+      << "Downloader::fast_receive() - Invalid receive size: " << rec_size
+      << endl;
+    return EU_error_abort;
+  }
+
+  // Poll the socket with select() to see if there is any data
+  struct timeval tv;
+  tv.tv_sec = 0;
+  tv.tv_usec = 0;
+  fd_set rset;
+  FD_ZERO(&rset);
+  FD_SET(socket, &rset);
+  int sret = select(socket, &rset, NULL, NULL, &tv);
+  if (sret == 0) {
+    return EU_network_no_data;
+  } else if (sret == -1) {
+    downloader_cat.error()
+      << "Downloader::fast_receive() - select() error: "
+      << handle_socket_error() << endl;
+    return get_network_error();
+  }
+  int ret = recv(socket, status->_next_in, rec_size, 0);
+  if (ret == 0) {
+    return EU_eof;
+  } else if (ret == -1) {
+    downloader_cat.error()
+      << "Downloader::fast_receive() - recv() error: "
+      << handle_socket_error() << endl;
+    return get_network_error();
+  }
+  if (downloader_cat.is_spam())
+    downloader_cat.spam()
+      << "Downloader::fast_receive() - recv() requested: " << rec_size
+      << " got: " << ret << " bytes" << endl;
+  status->_next_in += ret;
+  status->_bytes_in_buffer += ret;
+  status->_total_bytes += ret;
+  return EU_success;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::initiate
+//       Access: Published
+//  Description: Initiate the download of a complete file from the server.
+////////////////////////////////////////////////////////////////////
+int Downloader::
+initiate(const string &file_name, Filename file_dest) {
+  return initiate(file_name, file_dest, 0, 0, 0, false);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::initiate
+//       Access: Published
+//  Description: Initiate the download of a file from a server.
+////////////////////////////////////////////////////////////////////
+int Downloader::
+initiate(const string &file_name, Filename file_dest,
+        int first_byte, int last_byte, int total_bytes,
+        bool partial_content) {
+
+  if (_initiated == true) {
+    downloader_cat.error()
+      << "Downloader::initiate() - Download has already been initiated"
+      << endl;
+    return EU_error_abort;
+  }
+
+  // The file size is zero until we hear otherwise from the server
+  _file_size = 0;
+
+  // Connect to the server
+  int connect_ret = connect_to_server();
+  if (connect_ret < 0)
+    return connect_ret;
+
+  // Attempt to open the destination file
+  file_dest.set_binary();
+  _dest_stream.setf(ios::unitbuf, 0);
+  bool result;
+  if (partial_content == true && first_byte > 0)
+    result = file_dest.open_append(_dest_stream);
+  else
+    result = file_dest.open_write(_dest_stream);
+  if (result == false) {
+    downloader_cat.error()
+      << "Downloader::initiate() - Error opening file: " << file_dest
+      << " for writing: " << strerror(errno) << endl;
+    return get_write_error();
+  }
+
+  // Send an HTTP request for the file to the server
+  string request = "GET ";
+  if (_use_proxy == true)
+    request += _proxy_string;
+  request += file_name;
+  request += " HTTP/1.1\012Host: ";
+  request += _server_name;
+  request += "\012Connection: close";
+  if (partial_content == true) {
+    if (downloader_cat.is_debug())
+      downloader_cat.debug()
+        << "Downloader::initiate() - Requesting byte range: " << first_byte
+        << "-" << last_byte << endl;
+    request += "\012Range: bytes=";
+    stringstream start_stream;
+    start_stream << first_byte << "-" << last_byte;
+    request += start_stream.str();
+  }
+  request += "\012\012";
+  int outlen = request.size();
+  if (downloader_cat.is_debug())
+    downloader_cat.debug()
+      << "Downloader::initiate() - Sending request:\n" << request << endl;
+  int send_ret = safe_send(_socket, request.c_str(), outlen,
+                        (long)downloader_timeout);
+  if (send_ret < 0)
+    return send_ret;
+
+  // Create a download status to maintain download progress information
+  _current_status = new DownloadStatus(_buffer->_buffer,
+                                first_byte, last_byte, total_bytes,
+                                partial_content);
+
+  _tfirst = 0.0;
+  _tlast = 0.0;
+  _got_any_data = false;
+  _initiated = true;
+  _ever_initiated = true;
+  _download_to_ram = false;
+  return EU_success;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::initiate
+//       Access: Published
+//  Description: Initiate the download of a file from a server.
+////////////////////////////////////////////////////////////////////
+int Downloader::
+initiate(const string &file_name) {
+  if (_initiated == true) {
+    downloader_cat.error()
+      << "Downloader::initiate() - Download has already been initiated"
+      << endl;
+    return EU_error_abort;
+  }
+
+  // The file size is zero until we hear otherwise from the server
+  _file_size = 0;
+
+  // Connect to the server
+  int connect_ret = connect_to_server();
+  if (connect_ret < 0)
+    return connect_ret;
+
+  // Send an HTTP request for the file to the server
+  string request = "GET ";
+  if (_use_proxy == true)
+    request += _proxy_string;
+  request += file_name;
+  request += " HTTP/1.1\012Host: ";
+  request += _server_name;
+  request += "\012Connection: close";
+  request += "\012\012";
+  int outlen = request.size();
+  if (downloader_cat.is_debug())
+    downloader_cat.debug()
+      << "Downloader::initiate() - Sending request:\n" << request << endl;
+  int send_ret = safe_send(_socket, request.c_str(), outlen,
+                        (long)downloader_timeout);
+  if (send_ret < 0)
+    return send_ret;
+
+  // Create a download status to maintain download progress information
+  _current_status = new DownloadStatus(_buffer->_buffer, 0, 0, 0, false);
+
+  _tfirst = 0.0;
+  _tlast = 0.0;
+  _got_any_data = false;
+  _initiated = true;
+  _ever_initiated = true;
+  _download_to_ram = true;
+  _dest_string_stream = new ostringstream();
+  return EU_success;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::cleanup
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+void Downloader::
+cleanup(void) {
+  if (_initiated == false) {
+    downloader_cat.error()
+      << "Downloader::cleanup() - Download has not been initiated"
+      << endl;
+    return;
+  }
+
+  // The "Connection: close" line tells the server to close the
+  // connection when the download is complete
+  disconnect_from_server();
+  _dest_stream.close();
+  _total_bytes_written = _current_status->_total_bytes_written;
+  _total_bytes_requested = _current_status->_total_bytes_requested;
+  delete _current_status;
+  // We must set this to NULL otherwise there is a bad pointer floating around
+  _current_status = NULL;
+  _initiated = false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::run
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+int Downloader::
+run(void) {
+  if (_initiated == false) {
+    downloader_cat.error()
+      << "Downloader::run() - Download has not been initiated"
+      << endl;
+    return EU_error_abort;
+  }
+
+  nassertr(_current_status != NULL, EU_error_abort);
+
+  int connect_ret = connect_to_server();
+  if (connect_ret < 0)
+    return connect_ret;
+
+  if (_download_to_ram == true)
+    return run_to_ram();
+
+  int ret = EU_ok;
+  int write_ret;
+  double t0 = _clock.get_real_time();
+  if (_tfirst == 0.0) {
+    _tfirst = t0;
+  }
+  if (t0 - _tlast < _frequency)
+    return EU_ok;
+
+  _tlast = _clock.get_real_time();
+
+  // Recompute the buffer size if necessary
+  if (_recompute_buffer == true) {
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run() - Recomputing the buffer" << endl;
+
+    // Flush the current buffer if it holds any data
+    if (_current_status->_bytes_in_buffer > 0) {
+      write_ret = write_to_disk(_current_status);
+      if (write_ret < 0)
+        return write_ret;
+
+      ret = EU_write;
+    }
+
+    // Allocate a new buffer
+    _buffer.clear();
+    _receive_size = (ulong)(_frequency * _byte_rate);
+    _disk_buffer_size = _receive_size * _disk_write_frequency;
+    _buffer = new Buffer(_disk_buffer_size);
+    _current_status->_buffer = _buffer->_buffer;
+    _current_status->reset();
+    // Reset the flag
+    _recompute_buffer = false;
+    // Reset the statistics
+    _tfirst = t0;
+    _current_status->_total_bytes = 0;
+
+  } else if (_current_status->_bytes_in_buffer + _receive_size > ((unsigned int)_disk_buffer_size)) {
+
+    // Flush the current buffer if the next request would overflow it
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run() - Flushing buffer" << endl;
+    write_ret = write_to_disk(_current_status);
+    if (write_ret < 0)
+      return write_ret;
+    ret = EU_write;
+  }
+
+  // Attempt to receive the bytes from the socket
+  int fret = 0;
+  // Handle the case of a fast connection
+  if (_receive_size > (ulong)MAX_RECEIVE_BYTES) {
+    int repeat = (int)(_receive_size / MAX_RECEIVE_BYTES);
+    int remain = (int)fmod((double)_receive_size, (double)MAX_RECEIVE_BYTES);
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run() - fast connection - repeat: " << repeat
+        << " remain: " << remain << endl;
+    // Make multiple requests at once but do not exceed MAX_RECEIVE_BYTES
+    // for any single request
+    for (int i = 0; i <= repeat; i++) {
+      if (i < repeat)
+        fret = fast_receive(_socket, _current_status, MAX_RECEIVE_BYTES);
+      else if (remain > 0)
+        fret = fast_receive(_socket, _current_status, remain);
+      if (fret == EU_eof || fret < 0) {
+        break;
+      } else if (fret == EU_success) {
+        _got_any_data = true;
+      }
+    }
+  } else { // Handle the normal speed connection case
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run() - normal connection" << endl;
+    fret = fast_receive(_socket, _current_status, _receive_size);
+  }
+
+  _current_status->_total_bytes_requested += _receive_size;
+
+  // Check for end of file
+  if (fret == EU_eof) {
+    if (_got_any_data == true) {
+      if (_current_status->_bytes_in_buffer > 0) {
+        write_ret = write_to_disk(_current_status);
+        if (write_ret < 0)
+          return write_ret;
+      }
+      if (downloader_cat.is_spam())
+        downloader_cat.spam()
+          << "Downloader::run() - Got eof" << endl;
+      cleanup();
+      return EU_success;
+    } else {
+      if (downloader_cat.is_spam())
+        downloader_cat.spam()
+          << "Downloader::run() - Got 0 bytes" << endl;
+      return ret;
+    }
+  } else if (fret == EU_network_no_data) {
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run() - No data" << endl;
+      return ret;
+  } else if (fret < 0) {
+    return fret;
+  }
+
+  _got_any_data = true;
+  return ret;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::run_to_ram
+//       Access: Private
+//  Description:
+////////////////////////////////////////////////////////////////////
+int Downloader::
+run_to_ram(void) {
+  int ret = EU_ok;
+  int write_ret;
+
+  double t0 = _clock.get_real_time();
+  if (_tfirst == 0.0) {
+    _tfirst = t0;
+  }
+  if (t0 - _tlast < _frequency)
+    return EU_ok;
+
+  _tlast = _clock.get_real_time();
+
+  // Recompute the buffer size if necessary
+  if (_recompute_buffer == true) {
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run_to_ram() - Recomputing the buffer" << endl;
+
+    // Flush the current buffer if it holds any data
+    if (_current_status->_bytes_in_buffer > 0) {
+      write_ret = write_to_ram(_current_status);
+      if (write_ret < 0)
+        return write_ret;
+      ret = EU_write_ram;
+    }
+
+    // Allocate a new buffer
+    _buffer.clear();
+    _receive_size = (ulong)(_frequency * _byte_rate);
+    _disk_buffer_size = _receive_size * _disk_write_frequency;
+    _buffer = new Buffer(_disk_buffer_size);
+    _current_status->_buffer = _buffer->_buffer;
+    _current_status->reset();
+    // Reset the flag
+    _recompute_buffer = false;
+    // Reset the statistics
+    _tfirst = t0;
+    _current_status->_total_bytes = 0;
+
+  } else if (_current_status->_bytes_in_buffer + _receive_size >
+                                                ((unsigned int)_disk_buffer_size)) {
+
+    // Flush the current buffer if the next request would overflow it
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run_to_ram() - Flushing buffer" << endl;
+    write_ret = write_to_ram(_current_status);
+    if (write_ret < 0)
+      return write_ret;
+    ret = EU_write_ram;
+  }
+
+  // Attempt to receive the bytes from the socket
+  int fret = 0;
+  // Handle the case of a fast connection
+  if (_receive_size > (ulong)MAX_RECEIVE_BYTES) {
+    int repeat = (int)(_receive_size / MAX_RECEIVE_BYTES);
+    int remain = (int)fmod((double)_receive_size, (double)MAX_RECEIVE_BYTES);
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run_to_ram() - fast connection - repeat: " << repeat
+        << " remain: " << remain << endl;
+    // Make multiple requests at once but do not exceed MAX_RECEIVE_BYTES
+    // for any single request
+    for (int i = 0; i <= repeat; i++) {
+      if (i < repeat)
+        fret = fast_receive(_socket, _current_status, MAX_RECEIVE_BYTES);
+      else if (remain > 0)
+        fret = fast_receive(_socket, _current_status, remain);
+      if (fret == EU_eof || fret < 0) {
+        break;
+      } else if (fret == EU_success) {
+        _got_any_data = true;
+      }
+    }
+  } else { // Handle the normal speed connection case
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run_to_ram() - normal connection" << endl;
+    fret = fast_receive(_socket, _current_status, _receive_size);
+  }
+
+  _current_status->_total_bytes_requested += _receive_size;
+
+  // Check for end of file
+  if (fret == EU_eof) {
+    if (_got_any_data == true) {
+      if (_current_status->_bytes_in_buffer > 0) {
+        write_ret = write_to_ram(_current_status);
+        if (write_ret < 0)
+          return write_ret;
+      }
+      if (downloader_cat.is_spam())
+        downloader_cat.spam()
+          << "Downloader::run_to_ram() - Got eof" << endl;
+      cleanup();
+      return EU_success;
+    } else {
+      if (downloader_cat.is_spam())
+        downloader_cat.spam()
+          << "Downloader::run_to_ram() - Got 0 bytes" << endl;
+      return ret;
+    }
+  } else if (fret == EU_network_no_data) {
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::run_to_ram() - No data" << endl;
+      return ret;
+  } else if (fret < 0) {
+    return fret;
+  }
+
+  _got_any_data = true;
+  return ret;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::parse_http_response
+//       Access: Private
+//  Description: Check the HTTP response from the server
+////////////////////////////////////////////////////////////////////
+int Downloader::
+parse_http_response(const string &resp) {
+  size_t ws = resp.find(" ", 0);
+  string httpstr = resp.substr(0, ws);
+#if 0
+  if (!(httpstr == "HTTP/1.1")) {
+    downloader_cat.error()
+      << "Downloader::parse_http_response() - not HTTP/1.1 - got: "
+      << httpstr << endl;
+    return EU_error_abort;
+  }
+#endif
+  size_t ws2 = resp.find(" ", ws);
+  string numstr = resp.substr(ws, ws2);
+  nassertr(numstr.length() > 0, false);
+  int num = atoi(numstr.c_str());
+  switch (num) {
+    case 200:
+    case 206:
+      return EU_success;
+    case 202:
+      // Accepted - server may not honor request, though
+      if (downloader_cat.is_debug())
+        downloader_cat.debug()
+          << "Downloader::parse_http_response() - got a 202 Accepted - "
+          << "server does not guarantee to honor this request" << endl;
+      return EU_success;
+    case 201:
+    case 203:
+    case 204:
+    case 205:
+      break;
+    case 302:
+      if (downloader_cat.is_debug())
+        downloader_cat.debug()
+          << "Downloader::parse_http_response() - got a 302 redirect"
+          << endl;
+      return EU_error_abort;
+      break;
+    case 407:
+      return EU_error_http_proxy_authentication;
+    case 408:
+      return EU_error_http_server_timeout;
+    case 503:
+      return EU_error_http_service_unavailable;
+    case 504:
+      return EU_error_http_gateway_timeout;
+    default:
+      break;
+  }
+
+  downloader_cat.error()
+    << "Downloader::parse_http_response() - Invalid response: "
+    << resp << endl;
+  return EU_error_abort;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::parse_header
+//       Access: Private
+//  Description: Looks for a valid header.  If it finds one, it
+//               calculates the header length and strips it from
+//               the download status structure.  Function returns false
+//               on an error condition, otherwise true.
+////////////////////////////////////////////////////////////////////
+int Downloader::
+parse_header(DownloadStatus *status) {
+  nassertr(status != NULL, EU_error_abort);
+
+  if (status->_header_is_complete == true)
+    return EU_success;
+
+  if (status->_bytes_in_buffer == 0) {
+    downloader_cat.error()
+      << "Downloader::parse_header() - Empty buffer!" << endl;
+    return EU_error_abort;
+  }
+
+  string bufstr((char *)status->_start, status->_bytes_in_buffer);
+  size_t p  = 0;
+  bool redirect = false;
+  bool authenticate = false;
+  while (p < bufstr.length()) {
+    // Server sends out CR LF (\r\n) as newline delimiter
+    size_t nl = bufstr.find("\015\012", p);
+    if (nl == string::npos) {
+      downloader_cat.error()
+        << "Downloader::parse_header() - No newlines in buffer of "
+        << "length: " << status->_bytes_in_buffer << endl;
+      return EU_error_abort;
+    } else if (p == 0 && nl == p) {
+      downloader_cat.error()
+        << "Downloader::parse_header() - Buffer begins with newline!"
+        << endl;
+        return EU_error_abort;
+    }
+
+    string component = bufstr.substr(p, nl - p);
+
+    // The first line of the response should say whether
+    // got an error or not
+    if (status->_first_line_complete == false) {
+      status->_first_line_complete = true;
+      int parse_ret = parse_http_response(component);
+      if (parse_ret == EU_success) {
+        if (downloader_cat.is_spam())
+          downloader_cat.spam()
+            << "Downloader::parse_header() - Header is valid: "
+            << component << endl;
+        status->_header_is_valid = true;
+      } else if (parse_ret == EU_error_http_proxy_authentication) {
+        authenticate = true;
+        status->_header_is_valid = true;
+      } else if (parse_ret == EU_http_redirect) {
+        redirect = true;
+        status->_header_is_valid = true;
+      } else {
+        return parse_ret;
+      }
+    }
+
+    // Look for content length and location
+    size_t cpos = component.find(":");
+    string tline = component.substr(0, cpos);
+    if (tline == "Content-Length") {
+      tline = component.substr(cpos + 2, string::npos);
+      int server_download_bytes = atoi(tline.c_str());
+      _file_size = server_download_bytes;
+
+      if (status->_partial_content) {
+        // Ensure that our expected content length matches what the
+        // server gives us.
+        int client_download_bytes = status->_last_byte - status->_first_byte;
+        if (status->_first_byte == 0)
+          client_download_bytes += 1;
+        if (client_download_bytes != server_download_bytes) {
+          downloader_cat.error()
+            << "Downloader::parse_header() - server size = "
+            << server_download_bytes << ", client size = "
+            << client_download_bytes << " ("
+            << status->_last_byte << "-" << status->_first_byte << ")" << endl;
+          return EU_error_abort;
+        }
+      }
+
+    } else if (tline == "Transfer-Encoding") {
+      // This code isn't designed to handle transfer encodings other
+      // than identity.  It will certainly fail if we ever get this.
+      string encoding = component.substr(cpos + 2);
+      if (encoding == "identity") {
+        // No problem.
+
+      } else {
+        downloader_cat.error()
+          << "Non-identity transfer encoding specified by server!\n";
+        downloader_cat.error()
+          << component << "\n";
+        return EU_error_abort;
+      }
+
+    } else if (authenticate && tline == "Proxy-Authenticate") {
+      // We don't presently support authentication-demanding proxies.
+      downloader_cat.error()
+        << component << "\n";
+      return EU_error_http_proxy_authentication;
+
+    } else if (redirect == true && tline == "Location") {
+      tline = component.substr(cpos + 1, string::npos);
+      downloader_cat.error()
+        << "Got redirect to " << tline << endl;
+      return EU_error_abort;
+    }
+
+    // Two consecutive (CR LF)s indicates end of HTTP header
+    if (nl == p) {
+      // Make sure we didn't get a redirect
+      if (redirect == true) {
+        downloader_cat.error()
+          << "Downloader::parse_header() - Got a 302 redirect but no "
+          << "Location directive" << endl;
+        return EU_error_abort;
+      }
+      if (authenticate) {
+        return EU_error_http_proxy_authentication;
+      }
+      if (downloader_cat.is_spam())
+        downloader_cat.spam()
+          << "Downloader::parse_header() - Header is complete" << endl;
+      status->_header_is_complete = true;
+
+      // Strip the header out of the status buffer
+      int header_length = nl + 2;
+      status->_start += header_length;
+      status->_bytes_in_buffer -= header_length;
+
+      if (downloader_cat.is_spam())
+        downloader_cat.spam()
+          << "Downloader::parse_header() - Stripping out header of size: "
+          << header_length << endl;
+
+      return EU_success;
+    }
+
+    p = nl + 2;
+  }
+
+  if (status->_header_is_complete == false) {
+    if (downloader_cat.is_debug())
+      downloader_cat.debug()
+        << "Downloader::parse_header() - Reached end of buffer without "
+        << "successfully parsing the header - buffer size: "
+        << status->_bytes_in_buffer << endl;
+    return EU_error_abort;
+  }
+
+  return EU_success;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::write_to_disk
+//       Access: Private
+//  Description: Writes a download to disk.  If there is a header,
+//               the pointer and size are adjusted so the header
+//               is excluded.  Function returns false on error
+//               condition.
+////////////////////////////////////////////////////////////////////
+int Downloader::
+write_to_disk(DownloadStatus *status) {
+  nassertr(status != NULL, EU_error_abort);
+
+  // Ensure the header has been parsed successfully first
+  int parse_ret = parse_header(status);
+  if (parse_ret < 0)
+    return parse_ret;
+
+  if (status->_header_is_complete == false) {
+    downloader_cat.error()
+      << "Downloader::write_to_disk() - Incomplete HTTP header - "
+      << "(or header was larger than download buffer) - "
+      << "try increasing download-buffer-size" << endl;
+    return EU_error_abort;
+  }
+
+  // Write what we have so far to disk
+  if (status->_bytes_in_buffer > 0) {
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::write_to_disk() - Writing "
+        << status->_bytes_in_buffer << " to disk" << endl;
+
+    _dest_stream.write(status->_start, status->_bytes_in_buffer);
+    _dest_stream.flush();
+    status->_total_bytes_written += status->_bytes_in_buffer;
+  }
+
+  status->reset();
+
+  return EU_success;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::write_to_ram
+//       Access: Private
+//  Description: Writes a download to memory.  If there is a header,
+//               the pointer and size are adjusted so the header
+//               is excluded.  Function returns false on error
+//               condition.
+////////////////////////////////////////////////////////////////////
+int Downloader::
+write_to_ram(DownloadStatus *status) {
+  nassertr(status != NULL, EU_error_abort);
+
+  // Ensure the header has been parsed successfully first
+  int parse_ret = parse_header(status);
+  if (parse_ret < 0)
+    return parse_ret;
+
+  if (status->_header_is_complete == false) {
+    downloader_cat.error()
+      << "Downloader::write_to_ram() - Incomplete HTTP header - "
+      << "(or header was larger than download buffer) - "
+      << "try increasing download-buffer-size" << endl;
+    return EU_error_abort;
+  }
+
+  // Write what we have so far to memory
+  if (status->_bytes_in_buffer > 0) {
+    if (downloader_cat.is_spam())
+      downloader_cat.spam()
+        << "Downloader::write_to_ram() - Writing "
+        << status->_bytes_in_buffer << " to memory" << endl;
+
+    _dest_string_stream->write(status->_start, status->_bytes_in_buffer);
+    status->_total_bytes_written += status->_bytes_in_buffer;
+  }
+
+  status->reset();
+
+  return EU_success;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::get_ramfile
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+bool Downloader::
+get_ramfile(Ramfile &rfile) {
+  rfile._data = _dest_string_stream->str();
+  delete _dest_string_stream;
+  _dest_string_stream = NULL;
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::DownloadStatus::constructor
+//       Access: Private
+//  Description:
+////////////////////////////////////////////////////////////////////
+Downloader::DownloadStatus::
+DownloadStatus(char *buffer, int first_byte, int last_byte,
+                                int total_bytes, bool partial_content) {
+  _first_line_complete = false;
+  _header_is_complete = false;
+  _header_is_valid = false;
+  _buffer = buffer;
+  _first_byte = first_byte;
+  _last_byte = last_byte;
+  _total_bytes = total_bytes;
+  // Initialize the total bytes written to include all
+  // the bytes from previous partial downloads. This will
+  // ensure that when somebody calls get_bytes_written they
+  // will get the total size of the file, not just the number
+  // of bytes for this partial download
+  _total_bytes_written = first_byte;
+  _total_bytes_requested = first_byte;
+  _partial_content = partial_content;
+  reset();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Downloader::DownloadStatus::reset
+//       Access: Public
+//  Description: Resets the status buffer for more downloading after
+//               a write.
+////////////////////////////////////////////////////////////////////
+void Downloader::DownloadStatus::
+reset(void) {
+  _start = _buffer;
+  _next_in = _start;
+  _bytes_in_buffer = 0;
+}

panda/src/downloader/downloader.h

@@ -0,0 +1,157 @@
+// Filename: downloader.h
+// Created by:  mike (09Jan97)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+#ifndef DOWNLOADER_H
+#define DOWNLOADER_H
+//
+////////////////////////////////////////////////////////////////////
+// Includes
+////////////////////////////////////////////////////////////////////
+#include <pandabase.h>
+#include <notify.h>
+#include <filename.h>
+#include <buffer.h>
+#include <pointerTo.h>
+#include <clockObject.h>
+
+#if defined(WIN32_VC)
+  #include <winsock.h>
+#else
+  #include <netinet/in.h>  // Irix seems to require this one for resolv.h.
+  #include <sys/types.h>
+  #include <sys/socket.h>
+  #include <resolv.h>
+#endif
+
+class URLSpec;
+
+////////////////////////////////////////////////////////////////////
+//       Class : Downloader
+// Description : This object is used to manage downloading of data
+//               from an HTTP server as a background task within a
+//               single-threaded network application.  The class can
+//               download a small piece at a time when run() is called
+//               from time to time; it is designed to both limit time
+//               spent in the run() call as well as limiting network
+//               bandwidth utilized by the download, so that CPU
+//               cycles and bandwidth are still available to the
+//               application for other purposes.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDAEXPRESS Downloader {
+PUBLISHED:
+  Downloader(void);
+  virtual ~Downloader(void);
+
+  int connect_to_server_by_proxy(const URLSpec &proxy, const string &server_name);
+  int connect_to_server(const string &name, uint port=80);
+  void disconnect_from_server(void);
+
+  int initiate(const string &file_name, Filename file_dest);
+  int initiate(const string &file_name, Filename file_dest,
+                int first_byte, int last_byte, int total_bytes,
+                bool partial_content = true);
+  int initiate(const string &file_name);
+  INLINE int get_file_size() const;
+
+  int run(void);
+
+  bool get_ramfile(Ramfile &rfile);
+
+  INLINE void set_frequency(float frequency);
+  INLINE float get_frequency(void) const;
+  INLINE void set_byte_rate(float bytes);
+  INLINE float get_byte_rate(void) const;
+  INLINE void set_disk_write_frequency(int frequency);
+  INLINE int get_disk_write_frequency(void) const;
+  INLINE int get_bytes_written(void) const;
+  INLINE int get_bytes_requested(void) const;
+  INLINE float get_bytes_per_second(void) const;
+
+  void cleanup(void);
+
+
+private:
+  class DownloadStatus {
+  public:
+    DownloadStatus(char *buffer, int first_byte, int last_byte,
+                        int total_bytes, bool partial_content);
+    void reset(void);
+
+  public:
+    bool _first_line_complete;
+    bool _header_is_complete;
+    bool _header_is_valid;
+    char *_start;
+    char *_next_in;
+    int _bytes_in_buffer;
+    int _total_bytes_written;
+    int _total_bytes_requested;
+    int _first_byte;
+    int _last_byte;
+    int _total_bytes;
+    bool _partial_content;
+    char *_buffer;
+  };
+
+  INLINE void recompute_buffer(void);
+
+  int connect_to_server(void);
+  int safe_send(int socket, const char *data, int length, long timeout);
+  int fast_receive(int socket, DownloadStatus *status, int rec_size);
+  int parse_http_response(const string &resp);
+  int parse_header(DownloadStatus *status);
+  int write_to_disk(DownloadStatus *status);
+  int run_to_ram(void);
+  int write_to_ram(DownloadStatus *status);
+
+private:
+  bool _connected;
+  int _socket;
+  string _server_name;
+  bool _use_proxy;
+  string _proxy_string;
+  struct sockaddr_in _sin;
+  bool _TCP_stack_initialized;
+
+  bool _initiated;
+  bool _ever_initiated;
+  PT(Buffer) _buffer;
+  int _disk_write_frequency;
+  float _frequency;
+  float _byte_rate;
+  ulong _receive_size;
+  int _disk_buffer_size;
+  ofstream _dest_stream;
+  ostringstream *_dest_string_stream;
+  bool _recompute_buffer;
+
+  DownloadStatus *_current_status;
+  bool _got_any_data;
+  int _total_bytes_written;
+  int _total_bytes_requested;
+  bool _download_to_ram;
+
+  int _file_size;
+
+  double _tlast;
+  double _tfirst;
+  ClockObject _clock;
+};
+
+#include "downloader.I"
+
+#endif

panda/src/downloader/downloader_composite3.cxx

@@ -1,2 +1,3 @@
+#include "downloader.cxx"
 #include "downloadDb.cxx"
 

panda/src/downloader/httpChannel.cxx

@@ -593,6 +593,22 @@ run_connecting() {
   _status_string = string();
   if (BIO_do_connect(*_bio) <= 0) {
     if (BIO_should_retry(*_bio)) {
+
+      /* Put a block here for now. */
+      int fd = -1;
+      BIO_get_fd(*_bio, &fd);
+      if (fd < 0) {
+        downloader_cat.warning()
+          << "nonblocking socket BIO has no file descriptor.\n";
+      } else {
+        downloader_cat.spam()
+          << "waiting to connect.\n";
+        fd_set wset;
+        FD_ZERO(&wset);
+        FD_SET(fd, &wset);
+        select(fd + 1, NULL, &wset, NULL, NULL);
+      }        
+
       return true;
     }
     downloader_cat.info()