downloader is now superceded by HTTPClient

panda/src/downloader/Sources.pp

@@ -25,7 +25,7 @@
     multiplexStreamBuf.I multiplexStreamBuf.h \
     socketStream.h socketStream.I \
     urlSpec.I urlSpec.h \
-    $[if $[HAVE_NET], downloadDb.I downloadDb.h downloader.I downloader.h] \
+    $[if $[HAVE_NET], downloadDb.I downloadDb.h] \
     $[if $[HAVE_ZLIB], decompressor.h download_utils.h] \
     $[if $[HAVE_CRYPTO], patcher.cxx patcher.h patcher.I]
     
@@ -43,7 +43,7 @@
     multiplexStream.cxx multiplexStreamBuf.cxx \
     socketStream.cxx \
     urlSpec.cxx \
-    $[if $[HAVE_NET], downloadDb.cxx downloader.cxx] \
+    $[if $[HAVE_NET], downloadDb.cxx] \
     $[if $[HAVE_ZLIB], decompressor.cxx download_utils.cxx]
 
   #define INSTALL_HEADERS \
@@ -55,7 +55,6 @@
     config_downloader.h \
     decompressor.h \
     download_utils.h downloadDb.h downloadDb.I \
-    downloader.h downloader.I \
     extractor.h \
     httpClient.I httpClient.h \
     httpChannel.I httpChannel.h \

panda/src/downloader/downloader.I

@@ -1,168 +0,0 @@
-// Filename: downloader.I
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-
-#include "config_downloader.h"
-
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_file_size
-//       Access: Published
-//  Description: Returns the size of the file currently being
-//               downloaded, in bytes, if it is known.  Returns 0 if
-//               the file size is not known.
-//
-//               After a call to initiate(), this will return 0; at
-//               some later point, when the first response from the
-//               server is read, it may return the actual size of the
-//               file if the server told us.  It is possible the
-//               server will not tell us the size of the file, in
-//               which case this will always return 0.
-////////////////////////////////////////////////////////////////////
-INLINE int Downloader::
-get_file_size() const {
-  return _file_size;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::set_frequency
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE void Downloader::
-set_frequency(float frequency) {
-  nassertv(frequency > 0.0);
-  if (_frequency != frequency) {
-    _frequency = frequency;
-    _recompute_buffer = true;
-  }
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_frequency
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE float Downloader::
-get_frequency(void) const {
-  return _frequency;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::set_byte_rate
-//       Access: Public
-//  Description: Note: modem speeds are reported in bits, so you
-//               need to convert!
-////////////////////////////////////////////////////////////////////
-INLINE void Downloader::
-set_byte_rate(float bytes) {
-  nassertv(bytes > 0.0);
-  if (_byte_rate != bytes) {
-    _byte_rate = bytes;
-    _recompute_buffer = true;
-  }
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_byte_rate
-//       Access: Public
-//  Description: Returns byte rate in bytes.
-////////////////////////////////////////////////////////////////////
-INLINE float Downloader::
-get_byte_rate(void) const {
-  return _byte_rate;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::set_disk_write_frequency
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE void Downloader::
-set_disk_write_frequency(int frequency) {
-  nassertv(frequency > 0);
-  if (_disk_write_frequency != frequency) {
-    _disk_write_frequency = frequency;
-    _recompute_buffer = true;
-  }
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_disk_write_frequency
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE int Downloader::
-get_disk_write_frequency(void) const {
-  return _disk_write_frequency;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_bytes_written
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE int Downloader::
-get_bytes_written(void) const {
-  if (_ever_initiated == false) {
-    downloader_cat.warning()
-      << "Downloader::get_bytes_written() - Download has not been "
-      << "initiated" << endl;
-    return 0;
-  }
-  if (_current_status != NULL)
-    return _current_status->_total_bytes_written;
-  else
-    return _total_bytes_written;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_bytes_requested
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE int Downloader::
-get_bytes_requested(void) const {
-  if (_ever_initiated == false) {
-    downloader_cat.warning()
-      << "Downloader::get_bytes_requested() - Download has not been "
-      << "initiated" << endl;
-    return 0;
-  }
-  if (_current_status != NULL)
-    return _current_status->_total_bytes_requested;
-  else
-    return _total_bytes_requested;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_bytes_per_second
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE float Downloader::
-get_bytes_per_second(void) const {
-  if (_ever_initiated == false) {
-    downloader_cat.warning()
-      << "Downloader::get_bytes_per_second() - Download has not been "
-      << "initiated" << endl;
-    return 0.0;
-  }
-  nassertr(_tlast - _tfirst > 0.0, 0.0);
-  nassertr(_current_status != NULL, 0.0);
-  return (float)((double)_current_status->_total_bytes / (_tlast - _tfirst));
-}

panda/src/downloader/downloader.cxx

@@ -1,1096 +0,0 @@
-// Filename: downloader.cxx
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-
-
-#include "downloader.h"
-#include "config_downloader.h"
-#include "urlSpec.h"
-#include "error_utils.h"
-#include "filename.h"
-
-#include <errno.h>
-#include <math.h>
-
-#if !defined(WIN32_VC)
-  #include <sys/time.h>
-  #include <netinet/in.h>
-  #include <arpa/inet.h>
-  #include <netdb.h>
-  #define SOCKET_ERROR -1
-#endif
-
-////////////////////////////////////////////////////////////////////
-// Defines
-////////////////////////////////////////////////////////////////////
-const int MAX_RECEIVE_BYTES = 16384;
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::Constructor
-//       Access: Published
-//  Description:
-////////////////////////////////////////////////////////////////////
-Downloader::
-Downloader(void) {
-  _frequency = downloader_frequency;
-  _byte_rate = (float) downloader_byte_rate;
-  _disk_write_frequency = downloader_disk_write_frequency;
-  nassertv(_frequency > 0 && _byte_rate > 0 && _disk_write_frequency > 0);
-  _receive_size = (ulong)(_byte_rate * _frequency);
-  _disk_buffer_size = _disk_write_frequency * _receive_size;
-  _buffer = new Buffer(_disk_buffer_size);
-
-  _connected = false;
-  _use_proxy = false;
-  // We need to flush after every write in case we're interrupted
-  _dest_stream.setf(ios::unitbuf, 0);
-  _current_status = NULL;
-  _recompute_buffer = false;
-
-  _tfirst = 0.0;
-  _tlast = 0.0;
-  _got_any_data = false;
-  _initiated = false;
-  _ever_initiated = false;
-  _TCP_stack_initialized = false;
-  _total_bytes_written = 0;
-  _total_bytes_requested = 0;
-  _total_bytes_requested = 0;
-
-  _file_size = 0;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::Destructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Downloader::
-~Downloader() {
-  if (_connected)
-    disconnect_from_server();
-  _buffer.clear();
-  if (_initiated == true)
-    cleanup();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::connect_to_server_by_proxy
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-connect_to_server_by_proxy(const URLSpec &proxy, const string &server_name) {
-  if (connect_to_server(proxy.get_server(), proxy.get_port()) != EU_success) {
-    downloader_cat.error()
-      << "Downloader::connect_to_server_by_proxy() - could not connect to: "
-      << proxy << endl;
-    return EU_error_abort;
-  }
-  
-  _proxy_string = "http://";
-  _proxy_string += server_name;
-  _use_proxy = true;
-
-  return EU_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::connect_to_server
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-connect_to_server(const string &name, uint port) {
-
-#if defined(WIN32)
-  if (_TCP_stack_initialized == false) {
-    WSAData mydata;
-    int answer1 = WSAStartup(0x0101, &mydata);
-    if(answer1 != 0) {
-      downloader_cat.error()
-        << "Downloader::connect_to_server() - WSAStartup() - error: "
-        << handle_socket_error() << endl;
-      return EU_error_abort;
-    }
-    _TCP_stack_initialized = true;
-  }
-#endif
-
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Downloader connecting to server: " << name << " on port: "
-      << port << endl;
-
-  _server_name = name;
-
-  _sin.sin_family = PF_INET;
-  _sin.sin_port = htons(port);
-  ulong addr = (ulong)inet_addr(name.c_str());
-  struct hostent *hp = NULL;
-
-  if (addr == INADDR_NONE) {
-    hp = gethostbyname(name.c_str());
-    if (hp != NULL)
-      (void)memcpy(&_sin.sin_addr, hp->h_addr, (uint)hp->h_length);
-    else {
-      downloader_cat.error()
-        << "Downloader::connect_to_server() - gethostbyname() failed on: "
-        << name.c_str() << " with error: "
-        << handle_socket_error() << endl;
-      return get_network_error();
-    }
-  } else
-    (void)memcpy(&_sin.sin_addr, &addr, sizeof(addr));
-
-  return connect_to_server();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::connect_to_server
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-connect_to_server(void) {
-  if (_connected == true)
-    return EU_success;
-
-  _socket = 0xffffffff;
-  _socket = socket(PF_INET, SOCK_STREAM, 0);
-  if (_socket == (int)0xffffffff) {
-    downloader_cat.error()
-      << "Downloader::connect_to_server() - socket failed: "
-      << handle_socket_error() << endl;
-    return get_network_error();
-  }
-
-  if (connect(_socket, (struct sockaddr *)&_sin, sizeof(_sin)) ==
-                                                        SOCKET_ERROR) {
-    downloader_cat.error()
-      << "Downloader::connect_to_server() - connect() failed: "
-      << handle_socket_error() << endl;
-    disconnect_from_server();
-    return get_network_error();
-  }
-
-  _connected = true;
-  return EU_success;
-}
-
-///////////////////////////////////////////////////////////////////
-//     Function: Downloader::disconnect_from_server
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-void Downloader::
-disconnect_from_server(void) {
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Downloader disconnecting from server..." << endl;
-#if defined(WIN32)
-  (void)closesocket(_socket);
-#else
-  (void)close(_socket);
-#endif
-  _connected = false;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::safe_send
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-safe_send(int socket, const char *data, int length, long timeout) {
-  if (length == 0) {
-    downloader_cat.error()
-      << "Downloader::safe_send() - requested 0 length send!" << endl;
-    return EU_error_abort;
-  }
-  int bytes = 0;
-  struct timeval tv;
-  tv.tv_sec = timeout;
-  tv.tv_usec = 0;
-  fd_set wset;
-  FD_ZERO(&wset);
-  while (bytes < length) {
-    FD_SET(socket, &wset);
-    int sret = select(socket + 1, NULL, &wset, NULL, &tv);
-    if (sret == 0) {
-      downloader_cat.error()
-        << "Downloader::safe_send() - select timed out after: "
-        << timeout << " seconds" << endl;
-      return EU_error_network_timeout;
-    } else if (sret == -1) {
-      downloader_cat.error()
-        << "Downloader::safe_send() - error: " << handle_socket_error()
-        << endl;
-      return get_network_error();
-    }
-    int ret = send(socket, data, length, 0);
-    if (ret > 0)
-      bytes += ret;
-    else {
-      downloader_cat.error()
-        << "Downloader::safe_send() - error: " << handle_socket_error()
-        << endl;
-      return get_network_error();
-    }
-  }
-  return EU_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::fast_receive
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-fast_receive(int socket, DownloadStatus *status, int rec_size) {
-  nassertr(status != NULL, EU_error_abort);
-  if (rec_size <= 0) {
-    downloader_cat.error()
-      << "Downloader::fast_receive() - Invalid receive size: " << rec_size
-      << endl;
-    return EU_error_abort;
-  }
-
-  // Poll the socket with select() to see if there is any data
-  struct timeval tv;
-  tv.tv_sec = 0;
-  tv.tv_usec = 0;
-  fd_set rset;
-  FD_ZERO(&rset);
-  FD_SET(socket, &rset);
-  int sret = select(socket, &rset, NULL, NULL, &tv);
-  if (sret == 0) {
-    return EU_network_no_data;
-  } else if (sret == -1) {
-    downloader_cat.error()
-      << "Downloader::fast_receive() - select() error: "
-      << handle_socket_error() << endl;
-    return get_network_error();
-  }
-  int ret = recv(socket, status->_next_in, rec_size, 0);
-  if (ret == 0) {
-    return EU_eof;
-  } else if (ret == -1) {
-    downloader_cat.error()
-      << "Downloader::fast_receive() - recv() error: "
-      << handle_socket_error() << endl;
-    return get_network_error();
-  }
-  if (downloader_cat.is_spam())
-    downloader_cat.spam()
-      << "Downloader::fast_receive() - recv() requested: " << rec_size
-      << " got: " << ret << " bytes" << endl;
-  status->_next_in += ret;
-  status->_bytes_in_buffer += ret;
-  status->_total_bytes += ret;
-  return EU_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::initiate
-//       Access: Published
-//  Description: Initiate the download of a complete file from the server.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-initiate(const string &file_name, Filename file_dest) {
-  return initiate(file_name, file_dest, 0, 0, 0, false);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::initiate
-//       Access: Published
-//  Description: Initiate the download of a file from a server.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-initiate(const string &file_name, Filename file_dest,
-        int first_byte, int last_byte, int total_bytes,
-        bool partial_content) {
-
-  if (_initiated == true) {
-    downloader_cat.error()
-      << "Downloader::initiate() - Download has already been initiated"
-      << endl;
-    return EU_error_abort;
-  }
-
-  // The file size is zero until we hear otherwise from the server
-  _file_size = 0;
-
-  // Connect to the server
-  int connect_ret = connect_to_server();
-  if (connect_ret < 0)
-    return connect_ret;
-
-  // Attempt to open the destination file
-  file_dest.set_binary();
-  _dest_stream.setf(ios::unitbuf, 0);
-  bool result;
-  if (partial_content == true && first_byte > 0)
-    result = file_dest.open_append(_dest_stream);
-  else
-    result = file_dest.open_write(_dest_stream);
-  if (result == false) {
-    downloader_cat.error()
-      << "Downloader::initiate() - Error opening file: " << file_dest
-      << " for writing: " << strerror(errno) << endl;
-    return get_write_error();
-  }
-
-  // Send an HTTP request for the file to the server
-  string request = "GET ";
-  if (_use_proxy == true)
-    request += _proxy_string;
-  request += file_name;
-  request += " HTTP/1.1\012Host: ";
-  request += _server_name;
-  request += "\012Connection: close";
-  if (partial_content == true) {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Downloader::initiate() - Requesting byte range: " << first_byte
-        << "-" << last_byte << endl;
-    request += "\012Range: bytes=";
-    stringstream start_stream;
-    start_stream << first_byte << "-" << last_byte;
-    request += start_stream.str();
-  }
-  request += "\012\012";
-  int outlen = request.size();
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Downloader::initiate() - Sending request:\n" << request << endl;
-  int send_ret = safe_send(_socket, request.c_str(), outlen,
-                        (long)downloader_timeout);
-  if (send_ret < 0)
-    return send_ret;
-
-  // Create a download status to maintain download progress information
-  _current_status = new DownloadStatus(_buffer->_buffer,
-                                first_byte, last_byte, total_bytes,
-                                partial_content);
-
-  _tfirst = 0.0;
-  _tlast = 0.0;
-  _got_any_data = false;
-  _initiated = true;
-  _ever_initiated = true;
-  _download_to_ram = false;
-  return EU_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::initiate
-//       Access: Published
-//  Description: Initiate the download of a file from a server.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-initiate(const string &file_name) {
-  if (_initiated == true) {
-    downloader_cat.error()
-      << "Downloader::initiate() - Download has already been initiated"
-      << endl;
-    return EU_error_abort;
-  }
-
-  // The file size is zero until we hear otherwise from the server
-  _file_size = 0;
-
-  // Connect to the server
-  int connect_ret = connect_to_server();
-  if (connect_ret < 0)
-    return connect_ret;
-
-  // Send an HTTP request for the file to the server
-  string request = "GET ";
-  if (_use_proxy == true)
-    request += _proxy_string;
-  request += file_name;
-  request += " HTTP/1.1\012Host: ";
-  request += _server_name;
-  request += "\012Connection: close";
-  request += "\012\012";
-  int outlen = request.size();
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Downloader::initiate() - Sending request:\n" << request << endl;
-  int send_ret = safe_send(_socket, request.c_str(), outlen,
-                        (long)downloader_timeout);
-  if (send_ret < 0)
-    return send_ret;
-
-  // Create a download status to maintain download progress information
-  _current_status = new DownloadStatus(_buffer->_buffer, 0, 0, 0, false);
-
-  _tfirst = 0.0;
-  _tlast = 0.0;
-  _got_any_data = false;
-  _initiated = true;
-  _ever_initiated = true;
-  _download_to_ram = true;
-  _dest_string_stream = new ostringstream();
-  return EU_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::cleanup
-//       Access: Published
-//  Description:
-////////////////////////////////////////////////////////////////////
-void Downloader::
-cleanup(void) {
-  if (_initiated == false) {
-    downloader_cat.error()
-      << "Downloader::cleanup() - Download has not been initiated"
-      << endl;
-    return;
-  }
-
-  // The "Connection: close" line tells the server to close the
-  // connection when the download is complete
-  disconnect_from_server();
-  _dest_stream.close();
-  _total_bytes_written = _current_status->_total_bytes_written;
-  _total_bytes_requested = _current_status->_total_bytes_requested;
-  delete _current_status;
-  // We must set this to NULL otherwise there is a bad pointer floating around
-  _current_status = NULL;
-  _initiated = false;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::run
-//       Access: Published
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-run(void) {
-  if (_initiated == false) {
-    downloader_cat.error()
-      << "Downloader::run() - Download has not been initiated"
-      << endl;
-    return EU_error_abort;
-  }
-
-  nassertr(_current_status != NULL, EU_error_abort);
-
-  int connect_ret = connect_to_server();
-  if (connect_ret < 0)
-    return connect_ret;
-
-  if (_download_to_ram == true)
-    return run_to_ram();
-
-  int ret = EU_ok;
-  int write_ret;
-  double t0 = _clock.get_real_time();
-  if (_tfirst == 0.0) {
-    _tfirst = t0;
-  }
-  if (t0 - _tlast < _frequency)
-    return EU_ok;
-
-  _tlast = _clock.get_real_time();
-
-  // Recompute the buffer size if necessary
-  if (_recompute_buffer == true) {
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run() - Recomputing the buffer" << endl;
-
-    // Flush the current buffer if it holds any data
-    if (_current_status->_bytes_in_buffer > 0) {
-      write_ret = write_to_disk(_current_status);
-      if (write_ret < 0)
-        return write_ret;
-
-      ret = EU_write;
-    }
-
-    // Allocate a new buffer
-    _buffer.clear();
-    _receive_size = (ulong)(_frequency * _byte_rate);
-    _disk_buffer_size = _receive_size * _disk_write_frequency;
-    _buffer = new Buffer(_disk_buffer_size);
-    _current_status->_buffer = _buffer->_buffer;
-    _current_status->reset();
-    // Reset the flag
-    _recompute_buffer = false;
-    // Reset the statistics
-    _tfirst = t0;
-    _current_status->_total_bytes = 0;
-
-  } else if (_current_status->_bytes_in_buffer + _receive_size > ((unsigned int)_disk_buffer_size)) {
-
-    // Flush the current buffer if the next request would overflow it
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run() - Flushing buffer" << endl;
-    write_ret = write_to_disk(_current_status);
-    if (write_ret < 0)
-      return write_ret;
-    ret = EU_write;
-  }
-
-  // Attempt to receive the bytes from the socket
-  int fret = 0;
-  // Handle the case of a fast connection
-  if (_receive_size > (ulong)MAX_RECEIVE_BYTES) {
-    int repeat = (int)(_receive_size / MAX_RECEIVE_BYTES);
-    int remain = (int)fmod((double)_receive_size, (double)MAX_RECEIVE_BYTES);
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run() - fast connection - repeat: " << repeat
-        << " remain: " << remain << endl;
-    // Make multiple requests at once but do not exceed MAX_RECEIVE_BYTES
-    // for any single request
-    for (int i = 0; i <= repeat; i++) {
-      if (i < repeat)
-        fret = fast_receive(_socket, _current_status, MAX_RECEIVE_BYTES);
-      else if (remain > 0)
-        fret = fast_receive(_socket, _current_status, remain);
-      if (fret == EU_eof || fret < 0) {
-        break;
-      } else if (fret == EU_success) {
-        _got_any_data = true;
-      }
-    }
-  } else { // Handle the normal speed connection case
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run() - normal connection" << endl;
-    fret = fast_receive(_socket, _current_status, _receive_size);
-  }
-
-  _current_status->_total_bytes_requested += _receive_size;
-
-  // Check for end of file
-  if (fret == EU_eof) {
-    if (_got_any_data == true) {
-      if (_current_status->_bytes_in_buffer > 0) {
-        write_ret = write_to_disk(_current_status);
-        if (write_ret < 0)
-          return write_ret;
-      }
-      if (downloader_cat.is_spam())
-        downloader_cat.spam()
-          << "Downloader::run() - Got eof" << endl;
-      cleanup();
-      return EU_success;
-    } else {
-      if (downloader_cat.is_spam())
-        downloader_cat.spam()
-          << "Downloader::run() - Got 0 bytes" << endl;
-      return ret;
-    }
-  } else if (fret == EU_network_no_data) {
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run() - No data" << endl;
-      return ret;
-  } else if (fret < 0) {
-    return fret;
-  }
-
-  _got_any_data = true;
-  return ret;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::run_to_ram
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-run_to_ram(void) {
-  int ret = EU_ok;
-  int write_ret;
-
-  double t0 = _clock.get_real_time();
-  if (_tfirst == 0.0) {
-    _tfirst = t0;
-  }
-  if (t0 - _tlast < _frequency)
-    return EU_ok;
-
-  _tlast = _clock.get_real_time();
-
-  // Recompute the buffer size if necessary
-  if (_recompute_buffer == true) {
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run_to_ram() - Recomputing the buffer" << endl;
-
-    // Flush the current buffer if it holds any data
-    if (_current_status->_bytes_in_buffer > 0) {
-      write_ret = write_to_ram(_current_status);
-      if (write_ret < 0)
-        return write_ret;
-      ret = EU_write_ram;
-    }
-
-    // Allocate a new buffer
-    _buffer.clear();
-    _receive_size = (ulong)(_frequency * _byte_rate);
-    _disk_buffer_size = _receive_size * _disk_write_frequency;
-    _buffer = new Buffer(_disk_buffer_size);
-    _current_status->_buffer = _buffer->_buffer;
-    _current_status->reset();
-    // Reset the flag
-    _recompute_buffer = false;
-    // Reset the statistics
-    _tfirst = t0;
-    _current_status->_total_bytes = 0;
-
-  } else if (_current_status->_bytes_in_buffer + _receive_size >
-                                                ((unsigned int)_disk_buffer_size)) {
-
-    // Flush the current buffer if the next request would overflow it
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run_to_ram() - Flushing buffer" << endl;
-    write_ret = write_to_ram(_current_status);
-    if (write_ret < 0)
-      return write_ret;
-    ret = EU_write_ram;
-  }
-
-  // Attempt to receive the bytes from the socket
-  int fret = 0;
-  // Handle the case of a fast connection
-  if (_receive_size > (ulong)MAX_RECEIVE_BYTES) {
-    int repeat = (int)(_receive_size / MAX_RECEIVE_BYTES);
-    int remain = (int)fmod((double)_receive_size, (double)MAX_RECEIVE_BYTES);
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run_to_ram() - fast connection - repeat: " << repeat
-        << " remain: " << remain << endl;
-    // Make multiple requests at once but do not exceed MAX_RECEIVE_BYTES
-    // for any single request
-    for (int i = 0; i <= repeat; i++) {
-      if (i < repeat)
-        fret = fast_receive(_socket, _current_status, MAX_RECEIVE_BYTES);
-      else if (remain > 0)
-        fret = fast_receive(_socket, _current_status, remain);
-      if (fret == EU_eof || fret < 0) {
-        break;
-      } else if (fret == EU_success) {
-        _got_any_data = true;
-      }
-    }
-  } else { // Handle the normal speed connection case
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run_to_ram() - normal connection" << endl;
-    fret = fast_receive(_socket, _current_status, _receive_size);
-  }
-
-  _current_status->_total_bytes_requested += _receive_size;
-
-  // Check for end of file
-  if (fret == EU_eof) {
-    if (_got_any_data == true) {
-      if (_current_status->_bytes_in_buffer > 0) {
-        write_ret = write_to_ram(_current_status);
-        if (write_ret < 0)
-          return write_ret;
-      }
-      if (downloader_cat.is_spam())
-        downloader_cat.spam()
-          << "Downloader::run_to_ram() - Got eof" << endl;
-      cleanup();
-      return EU_success;
-    } else {
-      if (downloader_cat.is_spam())
-        downloader_cat.spam()
-          << "Downloader::run_to_ram() - Got 0 bytes" << endl;
-      return ret;
-    }
-  } else if (fret == EU_network_no_data) {
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::run_to_ram() - No data" << endl;
-      return ret;
-  } else if (fret < 0) {
-    return fret;
-  }
-
-  _got_any_data = true;
-  return ret;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::parse_http_response
-//       Access: Private
-//  Description: Check the HTTP response from the server
-////////////////////////////////////////////////////////////////////
-int Downloader::
-parse_http_response(const string &resp) {
-  size_t ws = resp.find(" ", 0);
-  string httpstr = resp.substr(0, ws);
-#if 0
-  if (!(httpstr == "HTTP/1.1")) {
-    downloader_cat.error()
-      << "Downloader::parse_http_response() - not HTTP/1.1 - got: "
-      << httpstr << endl;
-    return EU_error_abort;
-  }
-#endif
-  size_t ws2 = resp.find(" ", ws);
-  string numstr = resp.substr(ws, ws2);
-  nassertr(numstr.length() > 0, false);
-  int num = atoi(numstr.c_str());
-  switch (num) {
-    case 200:
-    case 206:
-      return EU_success;
-    case 202:
-      // Accepted - server may not honor request, though
-      if (downloader_cat.is_debug())
-        downloader_cat.debug()
-          << "Downloader::parse_http_response() - got a 202 Accepted - "
-          << "server does not guarantee to honor this request" << endl;
-      return EU_success;
-    case 201:
-    case 203:
-    case 204:
-    case 205:
-      break;
-    case 302:
-      if (downloader_cat.is_debug())
-        downloader_cat.debug()
-          << "Downloader::parse_http_response() - got a 302 redirect"
-          << endl;
-      return EU_error_abort;
-      break;
-    case 407:
-      return EU_error_http_proxy_authentication;
-    case 408:
-      return EU_error_http_server_timeout;
-    case 503:
-      return EU_error_http_service_unavailable;
-    case 504:
-      return EU_error_http_gateway_timeout;
-    default:
-      break;
-  }
-
-  downloader_cat.error()
-    << "Downloader::parse_http_response() - Invalid response: "
-    << resp << endl;
-  return EU_error_abort;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::parse_header
-//       Access: Private
-//  Description: Looks for a valid header.  If it finds one, it
-//               calculates the header length and strips it from
-//               the download status structure.  Function returns false
-//               on an error condition, otherwise true.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-parse_header(DownloadStatus *status) {
-  nassertr(status != NULL, EU_error_abort);
-
-  if (status->_header_is_complete == true)
-    return EU_success;
-
-  if (status->_bytes_in_buffer == 0) {
-    downloader_cat.error()
-      << "Downloader::parse_header() - Empty buffer!" << endl;
-    return EU_error_abort;
-  }
-
-  string bufstr((char *)status->_start, status->_bytes_in_buffer);
-  size_t p  = 0;
-  bool redirect = false;
-  bool authenticate = false;
-  while (p < bufstr.length()) {
-    // Server sends out CR LF (\r\n) as newline delimiter
-    size_t nl = bufstr.find("\015\012", p);
-    if (nl == string::npos) {
-      downloader_cat.error()
-        << "Downloader::parse_header() - No newlines in buffer of "
-        << "length: " << status->_bytes_in_buffer << endl;
-      return EU_error_abort;
-    } else if (p == 0 && nl == p) {
-      downloader_cat.error()
-        << "Downloader::parse_header() - Buffer begins with newline!"
-        << endl;
-        return EU_error_abort;
-    }
-
-    string component = bufstr.substr(p, nl - p);
-
-    // The first line of the response should say whether
-    // got an error or not
-    if (status->_first_line_complete == false) {
-      status->_first_line_complete = true;
-      int parse_ret = parse_http_response(component);
-      if (parse_ret == EU_success) {
-        if (downloader_cat.is_spam())
-          downloader_cat.spam()
-            << "Downloader::parse_header() - Header is valid: "
-            << component << endl;
-        status->_header_is_valid = true;
-      } else if (parse_ret == EU_error_http_proxy_authentication) {
-        authenticate = true;
-        status->_header_is_valid = true;
-      } else if (parse_ret == EU_http_redirect) {
-        redirect = true;
-        status->_header_is_valid = true;
-      } else {
-        return parse_ret;
-      }
-    }
-
-    // Look for content length and location
-    size_t cpos = component.find(":");
-    string tline = component.substr(0, cpos);
-    if (tline == "Content-Length") {
-      tline = component.substr(cpos + 2, string::npos);
-      int server_download_bytes = atoi(tline.c_str());
-      _file_size = server_download_bytes;
-
-      if (status->_partial_content) {
-        // Ensure that our expected content length matches what the
-        // server gives us.
-        int client_download_bytes = status->_last_byte - status->_first_byte;
-        if (status->_first_byte == 0)
-          client_download_bytes += 1;
-        if (client_download_bytes != server_download_bytes) {
-          downloader_cat.error()
-            << "Downloader::parse_header() - server size = "
-            << server_download_bytes << ", client size = "
-            << client_download_bytes << " ("
-            << status->_last_byte << "-" << status->_first_byte << ")" << endl;
-          return EU_error_abort;
-        }
-      }
-
-    } else if (tline == "Transfer-Encoding") {
-      // This code isn't designed to handle transfer encodings other
-      // than identity.  It will certainly fail if we ever get this.
-      string encoding = component.substr(cpos + 2);
-      if (encoding == "identity") {
-        // No problem.
-
-      } else {
-        downloader_cat.error()
-          << "Non-identity transfer encoding specified by server!\n";
-        downloader_cat.error()
-          << component << "\n";
-        return EU_error_abort;
-      }
-
-    } else if (authenticate && tline == "Proxy-Authenticate") {
-      // We don't presently support authentication-demanding proxies.
-      downloader_cat.error()
-        << component << "\n";
-      return EU_error_http_proxy_authentication;
-
-    } else if (redirect == true && tline == "Location") {
-      tline = component.substr(cpos + 1, string::npos);
-      downloader_cat.error()
-        << "Got redirect to " << tline << endl;
-      return EU_error_abort;
-    }
-
-    // Two consecutive (CR LF)s indicates end of HTTP header
-    if (nl == p) {
-      // Make sure we didn't get a redirect
-      if (redirect == true) {
-        downloader_cat.error()
-          << "Downloader::parse_header() - Got a 302 redirect but no "
-          << "Location directive" << endl;
-        return EU_error_abort;
-      }
-      if (authenticate) {
-        return EU_error_http_proxy_authentication;
-      }
-      if (downloader_cat.is_spam())
-        downloader_cat.spam()
-          << "Downloader::parse_header() - Header is complete" << endl;
-      status->_header_is_complete = true;
-
-      // Strip the header out of the status buffer
-      int header_length = nl + 2;
-      status->_start += header_length;
-      status->_bytes_in_buffer -= header_length;
-
-      if (downloader_cat.is_spam())
-        downloader_cat.spam()
-          << "Downloader::parse_header() - Stripping out header of size: "
-          << header_length << endl;
-
-      return EU_success;
-    }
-
-    p = nl + 2;
-  }
-
-  if (status->_header_is_complete == false) {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Downloader::parse_header() - Reached end of buffer without "
-        << "successfully parsing the header - buffer size: "
-        << status->_bytes_in_buffer << endl;
-    return EU_error_abort;
-  }
-
-  return EU_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::write_to_disk
-//       Access: Private
-//  Description: Writes a download to disk.  If there is a header,
-//               the pointer and size are adjusted so the header
-//               is excluded.  Function returns false on error
-//               condition.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-write_to_disk(DownloadStatus *status) {
-  nassertr(status != NULL, EU_error_abort);
-
-  // Ensure the header has been parsed successfully first
-  int parse_ret = parse_header(status);
-  if (parse_ret < 0)
-    return parse_ret;
-
-  if (status->_header_is_complete == false) {
-    downloader_cat.error()
-      << "Downloader::write_to_disk() - Incomplete HTTP header - "
-      << "(or header was larger than download buffer) - "
-      << "try increasing download-buffer-size" << endl;
-    return EU_error_abort;
-  }
-
-  // Write what we have so far to disk
-  if (status->_bytes_in_buffer > 0) {
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::write_to_disk() - Writing "
-        << status->_bytes_in_buffer << " to disk" << endl;
-
-    _dest_stream.write(status->_start, status->_bytes_in_buffer);
-    _dest_stream.flush();
-    status->_total_bytes_written += status->_bytes_in_buffer;
-  }
-
-  status->reset();
-
-  return EU_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::write_to_ram
-//       Access: Private
-//  Description: Writes a download to memory.  If there is a header,
-//               the pointer and size are adjusted so the header
-//               is excluded.  Function returns false on error
-//               condition.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-write_to_ram(DownloadStatus *status) {
-  nassertr(status != NULL, EU_error_abort);
-
-  // Ensure the header has been parsed successfully first
-  int parse_ret = parse_header(status);
-  if (parse_ret < 0)
-    return parse_ret;
-
-  if (status->_header_is_complete == false) {
-    downloader_cat.error()
-      << "Downloader::write_to_ram() - Incomplete HTTP header - "
-      << "(or header was larger than download buffer) - "
-      << "try increasing download-buffer-size" << endl;
-    return EU_error_abort;
-  }
-
-  // Write what we have so far to memory
-  if (status->_bytes_in_buffer > 0) {
-    if (downloader_cat.is_spam())
-      downloader_cat.spam()
-        << "Downloader::write_to_ram() - Writing "
-        << status->_bytes_in_buffer << " to memory" << endl;
-
-    _dest_string_stream->write(status->_start, status->_bytes_in_buffer);
-    status->_total_bytes_written += status->_bytes_in_buffer;
-  }
-
-  status->reset();
-
-  return EU_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_ramfile
-//       Access: Published
-//  Description:
-////////////////////////////////////////////////////////////////////
-bool Downloader::
-get_ramfile(Ramfile &rfile) {
-  rfile._data = _dest_string_stream->str();
-  delete _dest_string_stream;
-  _dest_string_stream = NULL;
-  return true;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::DownloadStatus::constructor
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-Downloader::DownloadStatus::
-DownloadStatus(char *buffer, int first_byte, int last_byte,
-                                int total_bytes, bool partial_content) {
-  _first_line_complete = false;
-  _header_is_complete = false;
-  _header_is_valid = false;
-  _buffer = buffer;
-  _first_byte = first_byte;
-  _last_byte = last_byte;
-  _total_bytes = total_bytes;
-  // Initialize the total bytes written to include all
-  // the bytes from previous partial downloads. This will
-  // ensure that when somebody calls get_bytes_written they
-  // will get the total size of the file, not just the number
-  // of bytes for this partial download
-  _total_bytes_written = first_byte;
-  _total_bytes_requested = first_byte;
-  _partial_content = partial_content;
-  reset();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::DownloadStatus::reset
-//       Access: Public
-//  Description: Resets the status buffer for more downloading after
-//               a write.
-////////////////////////////////////////////////////////////////////
-void Downloader::DownloadStatus::
-reset(void) {
-  _start = _buffer;
-  _next_in = _start;
-  _bytes_in_buffer = 0;
-}

panda/src/downloader/downloader.h

@@ -1,157 +0,0 @@
-// Filename: downloader.h
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-#ifndef DOWNLOADER_H
-#define DOWNLOADER_H
-//
-////////////////////////////////////////////////////////////////////
-// Includes
-////////////////////////////////////////////////////////////////////
-#include <pandabase.h>
-#include <notify.h>
-#include <filename.h>
-#include <buffer.h>
-#include <pointerTo.h>
-#include <clockObject.h>
-
-#if defined(WIN32_VC)
-  #include <winsock.h>
-#else
-  #include <netinet/in.h>  // Irix seems to require this one for resolv.h.
-  #include <sys/types.h>
-  #include <sys/socket.h>
-  #include <resolv.h>
-#endif
-
-class URLSpec;
-
-////////////////////////////////////////////////////////////////////
-//       Class : Downloader
-// Description : This object is used to manage downloading of data
-//               from an HTTP server as a background task within a
-//               single-threaded network application.  The class can
-//               download a small piece at a time when run() is called
-//               from time to time; it is designed to both limit time
-//               spent in the run() call as well as limiting network
-//               bandwidth utilized by the download, so that CPU
-//               cycles and bandwidth are still available to the
-//               application for other purposes.
-////////////////////////////////////////////////////////////////////
-class EXPCL_PANDAEXPRESS Downloader {
-PUBLISHED:
-  Downloader(void);
-  virtual ~Downloader(void);
-
-  int connect_to_server_by_proxy(const URLSpec &proxy, const string &server_name);
-  int connect_to_server(const string &name, uint port=80);
-  void disconnect_from_server(void);
-
-  int initiate(const string &file_name, Filename file_dest);
-  int initiate(const string &file_name, Filename file_dest,
-                int first_byte, int last_byte, int total_bytes,
-                bool partial_content = true);
-  int initiate(const string &file_name);
-  INLINE int get_file_size() const;
-
-  int run(void);
-
-  bool get_ramfile(Ramfile &rfile);
-
-  INLINE void set_frequency(float frequency);
-  INLINE float get_frequency(void) const;
-  INLINE void set_byte_rate(float bytes);
-  INLINE float get_byte_rate(void) const;
-  INLINE void set_disk_write_frequency(int frequency);
-  INLINE int get_disk_write_frequency(void) const;
-  INLINE int get_bytes_written(void) const;
-  INLINE int get_bytes_requested(void) const;
-  INLINE float get_bytes_per_second(void) const;
-
-  void cleanup(void);
-
-
-private:
-  class DownloadStatus {
-  public:
-    DownloadStatus(char *buffer, int first_byte, int last_byte,
-                        int total_bytes, bool partial_content);
-    void reset(void);
-
-  public:
-    bool _first_line_complete;
-    bool _header_is_complete;
-    bool _header_is_valid;
-    char *_start;
-    char *_next_in;
-    int _bytes_in_buffer;
-    int _total_bytes_written;
-    int _total_bytes_requested;
-    int _first_byte;
-    int _last_byte;
-    int _total_bytes;
-    bool _partial_content;
-    char *_buffer;
-  };
-
-  INLINE void recompute_buffer(void);
-
-  int connect_to_server(void);
-  int safe_send(int socket, const char *data, int length, long timeout);
-  int fast_receive(int socket, DownloadStatus *status, int rec_size);
-  int parse_http_response(const string &resp);
-  int parse_header(DownloadStatus *status);
-  int write_to_disk(DownloadStatus *status);
-  int run_to_ram(void);
-  int write_to_ram(DownloadStatus *status);
-
-private:
-  bool _connected;
-  int _socket;
-  string _server_name;
-  bool _use_proxy;
-  string _proxy_string;
-  struct sockaddr_in _sin;
-  bool _TCP_stack_initialized;
-
-  bool _initiated;
-  bool _ever_initiated;
-  PT(Buffer) _buffer;
-  int _disk_write_frequency;
-  float _frequency;
-  float _byte_rate;
-  ulong _receive_size;
-  int _disk_buffer_size;
-  ofstream _dest_stream;
-  ostringstream *_dest_string_stream;
-  bool _recompute_buffer;
-
-  DownloadStatus *_current_status;
-  bool _got_any_data;
-  int _total_bytes_written;
-  int _total_bytes_requested;
-  bool _download_to_ram;
-
-  int _file_size;
-
-  double _tlast;
-  double _tfirst;
-  ClockObject _clock;
-};
-
-#include "downloader.I"
-
-#endif

panda/src/downloader/downloader_composite3.cxx

@@ -1,3 +1,2 @@
-#include "downloader.cxx"
 #include "downloadDb.cxx"