robustify http handling

panda/src/downloader/Sources.pp

@@ -18,6 +18,7 @@
     extractor.h \
     httpClient.I httpClient.h \
     httpDocument.I httpDocument.h \
+    identityStream.I identityStream.h identityStreamBuf.h \
     multiplexStream.I multiplexStream.h \
     multiplexStreamBuf.I multiplexStreamBuf.h \
     urlSpec.I urlSpec.h \
@@ -33,6 +34,7 @@
     extractor.cxx \
     httpClient.cxx \
     httpDocument.cxx \
+    identityStream.cxx identityStreamBuf.cxx \
     multiplexStream.cxx multiplexStreamBuf.cxx \
     urlSpec.cxx \
     $[if $[HAVE_NET], downloadDb.cxx downloader.cxx] \
@@ -49,6 +51,7 @@
     extractor.h \
     httpClient.I httpClient.h \
     httpDocument.I httpDocument.h \
+    identityStream.I identityStream.h identityStreamBuf.h \
     multiplexStream.I multiplexStream.h \
     multiplexStreamBuf.I multiplexStreamBuf.I \
     patcher.h patcher.I \

panda/src/downloader/chunkedStream.h

@@ -33,14 +33,15 @@ class HTTPDocument;
 //
 //               Seeking is not supported.
 ////////////////////////////////////////////////////////////////////
-class EXPCL_PANDAEXPRESS IChunkedStream : public istream {
+// No need to export from DLL.
+class IChunkedStream : public istream {
 public:
   INLINE IChunkedStream();
   INLINE IChunkedStream(istream *source, bool owns_source, 
-                        HTTPDocument *doc = NULL);
+                        HTTPDocument *doc);
 
   INLINE IChunkedStream &open(istream *source, bool owns_source, 
-                              HTTPDocument *doc = NULL);
+                              HTTPDocument *doc);
   INLINE IChunkedStream &close();
 
 private:
@@ -52,3 +53,4 @@ private:
 #endif
 
 
+

panda/src/downloader/chunkedStreamBuf.cxx

@@ -77,6 +77,7 @@ open_read(istream *source, bool owns_source, HTTPDocument *doc) {
   _doc = doc;
 
   if (_doc != (HTTPDocument *)NULL) {
+    _read_index = doc->_read_index;
     _doc->_file_size = 0;
 
     // Read a little bit from the file to get the first chunk (and
@@ -162,17 +163,20 @@ read_chars(char *start, size_t length) {
   if (!line.empty() && line[line.length() - 1] == '\r') {
     line = line.substr(0, line.length() - 1);
   }
-  int chunk_size = strtol(line.c_str(), NULL, 16);
-  if (chunk_size <= 0) {
+  size_t chunk_size = (size_t)strtol(line.c_str(), NULL, 16);
+  if (chunk_size == 0) {
     // Last chunk; we're done.
     _done = true;
+    if (_doc != (HTTPDocument *)NULL && _read_index == _doc->_read_index) {
+      _doc->_state = HTTPDocument::S_read_body;
+    }
     return 0;
   }
 
-  if (_doc != (HTTPDocument *)NULL) {
-    _doc->_file_size += (size_t)chunk_size;
+  if (_doc != (HTTPDocument *)NULL && _read_index == _doc->_read_index) {
+    _doc->_file_size += chunk_size;
   }
 
-  _chunk_remaining = (size_t)chunk_size;
+  _chunk_remaining = chunk_size;
   return read_chars(start, length);
 }

panda/src/downloader/chunkedStreamBuf.h

@@ -28,7 +28,8 @@
 // Description : The streambuf object that implements
 //               IChunkedStream.
 ////////////////////////////////////////////////////////////////////
-class EXPCL_PANDAEXPRESS ChunkedStreamBuf : public streambuf {
+// No need to export from DLL.
+class ChunkedStreamBuf : public streambuf {
 public:
   ChunkedStreamBuf();
   virtual ~ChunkedStreamBuf();
@@ -48,6 +49,7 @@ private:
   bool _done;
 
   PT(HTTPDocument) _doc;
+  int _read_index;
 };
 
 #endif

panda/src/downloader/downloader_composite1.cxx

@@ -7,6 +7,8 @@
 #include "extractor.cxx"
 #include "httpClient.cxx"
 #include "httpDocument.cxx"
+#include "identityStream.cxx"
+#include "identityStreamBuf.cxx"
 #include "multiplexStream.cxx"
 #include "multiplexStreamBuf.cxx"
 #include "urlSpec.cxx"

panda/src/downloader/httpClient.I

@@ -92,37 +92,3 @@ INLINE bool HTTPClient::
 get_verify_ssl() const {
   return _verify_ssl;
 }
-
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPClient::get_document
-//       Access: Published
-//  Description: Opens the named document for reading, or if body is
-//               nonempty, posts data for a particular URL and
-//               retrieves the response.  Returns a new HTTPDocument
-//               object whether the document is successfully read or
-//               not; you can test is_valid() and get_return_code() to
-//               determine whether the document was retrieved.
-////////////////////////////////////////////////////////////////////
-INLINE PT(HTTPDocument) HTTPClient::
-get_document(const URLSpec &url, const string &body) {
-  const char *method = "GET";
-  if (!body.empty()) {
-    method = "POST";
-  }
-
-  return make_request(method, url, body);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPClient::get_header
-//       Access: Published
-//  Description: Like get_document(), except only the header
-//               associated with the file is retrieved.  This may be
-//               used to test for existence of the file; it might also
-//               return the size of the file (if the server gives us
-//               this information).
-////////////////////////////////////////////////////////////////////
-INLINE PT(HTTPDocument) HTTPClient::
-get_header(const URLSpec &url) {
-  return make_request("HEAD", url, string());
-}

panda/src/downloader/httpClient.cxx

@@ -17,6 +17,7 @@
 ////////////////////////////////////////////////////////////////////
 
 #include "httpClient.h"
+#include "httpDocument.h"
 #include "config_downloader.h"
 #include "filename.h"
 #include "config_express.h"
@@ -106,6 +107,48 @@ HTTPClient::
   clear_expected_servers();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPClient::set_username
+//       Access: Published
+//  Description: Specifies the username:password string corresponding
+//               to a particular server and/or realm, when demanded by
+//               the server.  Either or both of the server or realm
+//               may be empty; if so, they match anything.  Also, the
+//               server may be set to the special string "*proxy",
+//               which will match any proxy server.
+//
+//               If the username is set to the empty string, this
+//               clears the password for the particular server/realm
+//               pair.
+////////////////////////////////////////////////////////////////////
+void HTTPClient::
+set_username(const string &server, const string &realm, const string &username) {
+  string key = server + ":" + realm;
+  if (username.empty()) {
+    _usernames.erase(key);
+  } else {
+    _usernames[key] = username;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPClient::get_username
+//       Access: Published
+//  Description: Returns the username:password string set for this
+//               server/realm pair, or empty string if nothing has
+//               been set.  See set_username().
+////////////////////////////////////////////////////////////////////
+string HTTPClient::
+get_username(const string &server, const string &realm) const {
+  string key = server + ":" + realm;
+  Usernames::const_iterator ui;
+  ui = _usernames.find(key);
+  if (ui != _usernames.end()) {
+    return (*ui).second;
+  }
+  return string();
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPClient::get_http_version_string
 //       Access: Published
@@ -120,11 +163,34 @@ get_http_version_string() const {
 
   case HV_11:
     return "HTTP/1.1";
+
+  case HV_other:
+    // Report the best we can do.
+    return "HTTP/1.1";
   }
 
   return "unknown";
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPClient::parse_http_version_string
+//       Access: Published
+//  Description: Matches the string representing a particular HTTP
+//               version against any of the known versions and returns
+//               the appropriate enumerated value, or HV_other if the
+//               version is unknown.
+////////////////////////////////////////////////////////////////////
+HTTPClient::HTTPVersion HTTPClient::
+parse_http_version_string(const string &version) {
+  if (version == "HTTP/1.0") {
+    return HV_10;
+  } else if (version == "HTTP/1.1") {
+    return HV_11;
+  } else {
+    return HV_other;
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPClient::load_certificates
 //       Access: Published
@@ -209,6 +275,55 @@ clear_expected_servers() {
   _expected_servers.clear();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPClient::get_documents
+//       Access: Published
+//  Description: Returns a new HTTPDocument object that may be used
+//               for reading multiple documents using the same
+//               connection, for greater network efficiency than
+//               calling HTTPClient::get_document() repeatedly (and
+//               thus forcing a new connection for each document).
+////////////////////////////////////////////////////////////////////
+PT(HTTPDocument) HTTPClient::
+get_documents() {
+  PT(HTTPDocument) doc = new HTTPDocument(this);
+  doc->set_persistent_connection(true);
+  return doc;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPClient::get_document
+//       Access: Published
+//  Description: Opens the named document for reading, or if body is
+//               nonempty, posts data for a particular URL and
+//               retrieves the response.  Returns a new HTTPDocument
+//               object whether the document is successfully read or
+//               not; you can test is_valid() and get_return_code() to
+//               determine whether the document was retrieved.
+////////////////////////////////////////////////////////////////////
+PT(HTTPDocument) HTTPClient::
+get_document(const URLSpec &url, const string &body) {
+  PT(HTTPDocument) doc = new HTTPDocument(this);
+  doc->get_document(url, body);
+  return doc;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPClient::get_header
+//       Access: Published
+//  Description: Like get_document(), except only the header
+//               associated with the file is retrieved.  This may be
+//               used to test for existence of the file; it might also
+//               return the size of the file (if the server gives us
+//               this information).
+////////////////////////////////////////////////////////////////////
+PT(HTTPDocument) HTTPClient::
+get_header(const URLSpec &url) {
+  PT(HTTPDocument) doc = new HTTPDocument(this);
+  doc->get_header(url);
+  return doc;
+}
+
 
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPClient::make_ctx
@@ -376,43 +491,44 @@ load_verify_locations(SSL_CTX *ctx, const Filename &ca_file) {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPClient::make_request
+//     Function: HTTPClient::establish_connection
 //       Access: Private
-//  Description: Chooses a suitable mechanism to handle this request,
-//               based on whether it is an http or https request, and
-//               according to whether we have a proxy in place.
-//               Issues the request and returns an HTTPDocument that
-//               represents the results.
+//  Description: Establishes a connection to the server, using the
+//               appropriate means.  Returns the newly allocated BIO
+//               representing the connection, or NULL if a connection
+//               cannot be established.
 ////////////////////////////////////////////////////////////////////
-PT(HTTPDocument) HTTPClient::
-make_request(const string &method, const URLSpec &url, const string &body) {
+BIO *HTTPClient::
+establish_connection(const URLSpec &url) {
   BIO *bio;
 
   if (_proxy.empty()) {
     if (url.get_scheme() == "https") {
-      bio = get_https(method, url, body);
+      bio = establish_https(url);
     } else {
-      bio = get_http(method, url, body);
+      bio = establish_http(url);
     }
   } else {
     if (url.get_scheme() == "https") {
-      bio = get_https_proxy(method, url, body);
+      bio = establish_https_proxy(url);
     } else {
-      bio = get_http_proxy(method, url, body);
+      bio = establish_http_proxy(url);
     }
   }    
 
-  return new HTTPDocument(bio, true);
+  return bio;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPClient::get_https
+//     Function: HTTPClient::establish_http
 //       Access: Private
-//  Description: Opens the indicated URL directly as an ordinary http
-//               document.
+//  Description: Establishes a connection to the server directly,
+//               without using a proxy.  Returns the newly allocated
+//               BIO representing the connection, or NULL if a
+//               connection cannot be established.
 ////////////////////////////////////////////////////////////////////
 BIO *HTTPClient::
-get_http(const string &method, const URLSpec &url, const string &body) {
+establish_http(const URLSpec &url) {
   ostringstream server;
   server << url.get_server() << ":" << url.get_port();
   string server_str = server.str();
@@ -431,18 +547,19 @@ get_http(const string &method, const URLSpec &url, const string &body) {
     return NULL;
   }
 
-  send_request(bio, method, url.get_path(), url.get_server(), body);
   return bio;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPClient::get_https
+//     Function: HTTPClient::establish_https
 //       Access: Private
-//  Description: Opens the indicated URL directly as an https
-//               document.
+//  Description: Establishes a connection to the secure server
+//               directly, without using a proxy.  Returns the newly
+//               allocated BIO representing the connection, or NULL if
+//               a connection cannot be established.
 ////////////////////////////////////////////////////////////////////
 BIO *HTTPClient::
-get_https(const string &method, const URLSpec &url, const string &body) {
+establish_https(const URLSpec &url) {
   ostringstream server;
   server << url.get_server() << ":" << url.get_port();
   string server_str = server.str();
@@ -461,22 +578,19 @@ get_https(const string &method, const URLSpec &url, const string &body) {
     return NULL;
   }
 
-  BIO *sbio = make_https_connection(bio, url);
-
-  if (sbio != (BIO *)NULL) {
-    send_request(sbio, method, url.get_path(), url.get_server(), body);
-  }
-  return sbio;
+  return make_https_connection(bio, url);
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPClient::get_http_proxy
+//     Function: HTTPClient::establish_http_proxy
 //       Access: Private
-//  Description: Opens the indicated URL via the proxy as an ordinary
-//               http document.
+//  Description: Establishes a connection to the server through a
+//               proxy.  Returns the newly allocated BIO representing
+//               the connection, or NULL if a connection cannot be
+//               established.
 ////////////////////////////////////////////////////////////////////
 BIO *HTTPClient::
-get_http_proxy(const string &method, const URLSpec &url, const string &body) {
+establish_http_proxy(const URLSpec &url) {
   ostringstream proxy_server;
   proxy_server << _proxy.get_server() << ":" << _proxy.get_port();
   string proxy_server_str = proxy_server.str();
@@ -496,18 +610,19 @@ get_http_proxy(const string &method, const URLSpec &url, const string &body) {
     return NULL;
   }
 
-  send_request(bio, method, url, url.get_server(), body);
   return bio;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPClient::get_https_proxy
+//     Function: HTTPClient::establish_https_proxy
 //       Access: Private
-//  Description: Opens the indicated URL via the proxy as an https
-//               document.
+//  Description: Establishes a connection to the secure server through
+//               a proxy.  Returns the newly allocated BIO
+//               representing the connection, or NULL if a connection
+//               cannot be established.
 ////////////////////////////////////////////////////////////////////
 BIO *HTTPClient::
-get_https_proxy(const string &method, const URLSpec &url, const string &body) {
+establish_https_proxy(const URLSpec &url) {
   // First, ask the proxy to open a connection for us.
   ostringstream proxy_server;
   proxy_server << _proxy.get_server() << ":" << _proxy.get_port();
@@ -528,35 +643,21 @@ get_https_proxy(const string &method, const URLSpec &url, const string &body) {
     return NULL;
   }
 
-  {
-    ostringstream request;
-    request 
-      << "CONNECT " << url.get_server() << ":" << url.get_port()
-      << " " << get_http_version_string() << "\r\n"
-      << "\r\n";
-    string request_str = request.str();
-
-#ifndef NDEBUG
-    if (downloader_cat.is_debug()) {
-      show_send(request_str);
-    }
-#endif
-    BIO_puts(bio, request_str.c_str());
-  }
+  ostringstream request;
+  request 
+    << "CONNECT " << url.get_server() << ":" << url.get_port()
+    << " " << get_http_version_string() << "\r\n";
+  string connect_header = request.str();
 
-  // Create a temporary HTTPDocument to read the response from the
-  // proxy.
+  // Create a temporary HTTPDocument to issue the request and read the
+  // response from the proxy.
   {
-    PT(HTTPDocument) doc = new HTTPDocument(bio, false);
-    if (!doc->is_valid()) {
+    PT(HTTPDocument) doc = new HTTPDocument(this, bio);
+    if (!doc->send_request(connect_header, string())) {
       downloader_cat.info()
         << "proxy would not open connection to " << url.get_authority()
         << ": " << doc->get_status_code() << " "
         << doc->get_status_string() << "\n";
-
-      if (downloader_cat.is_debug()) {
-        doc->write_headers(downloader_cat.debug(false));
-      }
       
       if (!get_verify_ssl()) {
         // If the proxy refused to open a raw connection for us, see
@@ -567,7 +668,7 @@ get_https_proxy(const string &method, const URLSpec &url, const string &body) {
         // us.)
         if ((doc->get_status_code() / 100) == 4) {
           BIO_free_all(bio);
-          return get_http_proxy(method, url, body);
+          return establish_http_proxy(url);
         }
       }
       return NULL;
@@ -581,12 +682,7 @@ get_https_proxy(const string &method, const URLSpec &url, const string &body) {
 
   // Ok, we now have a connection to our actual server, so start
   // speaking SSL and then ask for the document we really want.
-  BIO *sbio = make_https_connection(bio, url);
-
-  if (sbio != (BIO *)NULL) {
-    send_request(sbio, method, url.get_path(), url.get_server(), body);
-  }
-  return sbio;
+  return make_https_connection(bio, url);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -850,46 +946,6 @@ certificate signing
 */
 
 
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPClient::send_request
-//       Access: Private
-//  Description: Sends the appropriate GET or POST (or whatever)
-//               request to the server on the indicated connection.
-////////////////////////////////////////////////////////////////////
-void HTTPClient::
-send_request(BIO *bio, const string &method, 
-             const string &path, const string &server,
-             const string &body) const {
-  ostringstream request;
-
-  request 
-    << method << " " << path << " " << get_http_version_string() << "\r\n";
-
-  if (_http_version > HV_10) {
-    request 
-      << "Host: " << server << "\r\n"
-      << "Connection: close\r\n";
-  }
-
-  if (!body.empty()) {
-    request
-      << "Content-type: application/x-www-form-urlencoded\r\n"
-      << "Content-Length: " << body.length() << "\r\n";
-  }
-
-  request
-    << "\r\n"
-    << body;
-
-  string request_str = request.str();
-#ifndef NDEBUG
-  if (downloader_cat.is_debug()) {
-    show_send(request_str);
-  }
-#endif
-  BIO_puts(bio, request_str.c_str());
-}
-
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPClient::verify_server
 //       Access: Private
@@ -1089,31 +1145,6 @@ x509_name_subset(X509_NAME *name_a, X509_NAME *name_b) {
   return true;
 }
 
-#ifndef NDEBUG
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPClient::show_send
-//       Access: Private, Static
-//  Description: Writes the outgoing message, one line at a time, to
-//               the debugging log.
-////////////////////////////////////////////////////////////////////
-void HTTPClient::
-show_send(const string &message) {
-  size_t start = 0;
-  size_t newline = message.find('\n', start);
-  while (newline != string::npos) {
-    downloader_cat.debug()
-      << "send: " << message.substr(start, newline - start + 1);
-    start = newline + 1;
-    newline = message.find('\n', start);
-  }
-
-  if (start < message.length()) {
-    downloader_cat.debug()
-      << "send: " << message.substr(start) << " (no newline)\n";
-  }
-}
-#endif   // NDEBUG
-
 #if defined(SSL_097) && !defined(NDEBUG)
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPClient::ssl_msg_callback

panda/src/downloader/httpClient.h

@@ -29,17 +29,17 @@
 #ifdef HAVE_SSL
 
 #include "urlSpec.h"
-#include "httpDocument.h"
 #include "pointerTo.h"
 
 #include <openssl/ssl.h>
 
 // Windows may define this macro inappropriately.
-#ifdef WIN32
+#ifdef X509_NAME
 #undef X509_NAME
 #endif
 
 class Filename;
+class HTTPDocument;
 
 ////////////////////////////////////////////////////////////////////
 //       Class : HTTPClient
@@ -60,14 +60,19 @@ PUBLISHED:
   INLINE void set_proxy(const URLSpec &proxy);
   INLINE const URLSpec &get_proxy() const;
 
+  void set_username(const string &server, const string &realm, const string &username);
+  string get_username(const string &server, const string &realm) const;
+
   enum HTTPVersion {
     HV_10,  // HTTP 1.0
     HV_11,  // HTTP 1.1
+    HV_other,
   };
 
   INLINE void set_http_version(HTTPVersion version);
   INLINE HTTPVersion get_http_version() const;
   string get_http_version_string() const;
+  static HTTPVersion parse_http_version_string(const string &version);
 
   bool load_certificates(const Filename &filename);
 
@@ -77,37 +82,29 @@ PUBLISHED:
   bool add_expected_server(const string &server_attributes);
   void clear_expected_servers();
 
-  INLINE PT(HTTPDocument) get_document(const URLSpec &url,
-                                       const string &body = string());
-  INLINE PT(HTTPDocument) get_header(const URLSpec &url);
+  PT(HTTPDocument) get_documents();
+  PT(HTTPDocument) get_document(const URLSpec &url,
+                                const string &body = string());
+  PT(HTTPDocument) get_header(const URLSpec &url);
 
 private:
   void make_ctx();
   static void initialize_ssl();
   static int load_verify_locations(SSL_CTX *ctx, const Filename &ca_file);
 
-  PT(HTTPDocument) make_request(const string &method, const URLSpec &url, 
-                                const string &body);
-
-  BIO *get_http(const string &method, const URLSpec &url, const string &body);
-  BIO *get_https(const string &method, const URLSpec &url, const string &body);
-  BIO *get_http_proxy(const string &method, const URLSpec &url, const string &body);
-  BIO *get_https_proxy(const string &method, const URLSpec &url, const string &body);
+  BIO *establish_connection(const URLSpec &url);
+  BIO *establish_http(const URLSpec &url);
+  BIO *establish_https(const URLSpec &url);
+  BIO *establish_http_proxy(const URLSpec &url);
+  BIO *establish_https_proxy(const URLSpec &url);
 
   BIO *make_https_connection(BIO *bio, const URLSpec &url) const;
-  void send_request(BIO *bio, const string &method,
-                    const string &path, const string &server, 
-                    const string &body) const;
   bool verify_server(X509_NAME *subject) const;
 
   static X509_NAME *parse_x509_name(const string &source);
   static string get_x509_name_component(X509_NAME *name, int nid);
   static bool x509_name_subset(X509_NAME *name_a, X509_NAME *name_b);
 
-#ifndef NDEBUG
-  static void show_send(const string &message);
-#endif
-
 #if defined(SSL_097) && !defined(NDEBUG)
   static void ssl_msg_callback(int write_p, int version, int content_type,
                                const void *buf, size_t len, SSL *ssl,
@@ -118,6 +115,9 @@ private:
   HTTPVersion _http_version;
   bool _verify_ssl;
 
+  typedef pmap<string, string> Usernames;
+  Usernames _usernames;
+
   // List of allowable SSL servers to connect to.  If the list is
   // empty, any server is acceptable.
   typedef pvector<X509_NAME *> ExpectedServers;
@@ -127,6 +127,7 @@ private:
 
   static bool _ssl_initialized;
   static X509_STORE *_x509_store;
+  friend class HTTPDocument;
 };
 
 #include "httpClient.I"

panda/src/downloader/httpDocument.I

@@ -29,17 +29,41 @@ is_valid() const {
           (_status_code / 100) == 2);
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::get_url
+//       Access: Published
+//  Description: Returns the URL that was used to retrieve this
+//               document: whatever URL was last passed to
+//               get_document() or get_header().
+////////////////////////////////////////////////////////////////////
+INLINE const URLSpec &HTTPDocument::
+get_url() const {
+  return _url;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPDocument::get_http_version
 //       Access: Published
 //  Description: Returns the HTTP version number returned by the
-//               server, formatted as a string, e.g. "HTTP/1.1".
+//               server, as one of the HTTPClient enumerated types,
+//               e.g. HTTPClient::HV_11.
 ////////////////////////////////////////////////////////////////////
-INLINE const string &HTTPDocument::
+INLINE HTTPClient::HTTPVersion HTTPDocument::
 get_http_version() const {
   return _http_version;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::get_http_version_string
+//       Access: Published
+//  Description: Returns the HTTP version number returned by the
+//               server, formatted as a string, e.g. "HTTP/1.1".
+////////////////////////////////////////////////////////////////////
+INLINE const string &HTTPDocument::
+get_http_version_string() const {
+  return _http_version_string;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPDocument::get_status_code
 //       Access: Published
@@ -65,6 +89,80 @@ get_status_string() const {
   return _status_string;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::get_realm
+//       Access: Published
+//  Description: If the document failed to connect because of a 401
+//               (Authorization required) or 407 (Proxy authorization
+//               required) error, this method should return the
+//               "realm" returned by the server in which the requested
+//               document must be authenticated.  This string may be
+//               presented to the user to request an associated
+//               username and password (which then should be stored in
+//               HTTPClient::set_username()).
+//
+//               If the document was retrieved successfully, or failed
+//               to connect for some other reason, this string will be
+//               empty.  If it is empty even in the presence of a 401
+//               or 407 error, then the client code and the server do
+//               not have a common scheme for exchanging passwords.
+////////////////////////////////////////////////////////////////////
+INLINE const string &HTTPDocument::
+get_realm() const {
+  return _realm;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::get_redirect
+//       Access: Published
+//  Description: If the document failed with a redirect code (300
+//               series), this will generally contain the new URL the
+//               server wants us to try.  In many cases, the client
+//               will automatically follow redirects; if these are
+//               succesful the client will return a successful code
+//               and get_redirect() will return empty, but get_url()
+//               will return the new, redirected URL.
+////////////////////////////////////////////////////////////////////
+INLINE const URLSpec &HTTPDocument::
+get_redirect() const {
+  return _redirect;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::set_persistent_connection
+//       Access: Published
+//  Description: Indicates whether the HTTPDocument should try to keep
+//               the connection to the server open and reuse that
+//               connection for multiple documents, or whether it
+//               should close the connection and open a new one for
+//               each request.  Set this true to keep the connections
+//               around when possible, false to recycle them.
+//
+//               It makes most sense to set this false when the
+//               HTTPDocument will be used only once to retrieve a
+//               single document, true when you will be using the
+//               HTTPDocument interface to retrieve multiple
+//               documents.
+////////////////////////////////////////////////////////////////////
+INLINE void HTTPDocument::
+set_persistent_connection(bool persistent_connection) {
+  _persistent_connection = persistent_connection;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::get_persistent_connection
+//       Access: Published
+//  Description: Returns whether the HTTPDocument should try to keep
+//               the connection to the server open and reuse that
+//               connection for multiple documents, or whether it
+//               should close the connection and open a new one for
+//               each request.  See set_persistent_connection().
+////////////////////////////////////////////////////////////////////
+INLINE bool HTTPDocument::
+get_persistent_connection() const {
+  return _persistent_connection;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPDocument::get_file_size
 //       Access: Published
@@ -83,3 +181,34 @@ INLINE size_t HTTPDocument::
 get_file_size() const {
   return _file_size;
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::get_document
+//       Access: Published
+//  Description: Opens the named document for reading, or if body is
+//               nonempty, posts data for a particular URL and
+//               retrieves the response.
+////////////////////////////////////////////////////////////////////
+INLINE bool HTTPDocument::
+get_document(const URLSpec &url, const string &body) {
+  const char *method = "GET";
+  if (!body.empty()) {
+    method = "POST";
+  }
+
+  return send_request(method, url, body);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::get_header
+//       Access: Published
+//  Description: Like get_document(), except only the header
+//               associated with the file is retrieved.  This may be
+//               used to test for existence of the file; it might also
+//               return the size of the file (if the server gives us
+//               this information).
+////////////////////////////////////////////////////////////////////
+INLINE bool HTTPDocument::
+get_header(const URLSpec &url) {
+  return send_request("HEAD", url, string());
+}

panda/src/downloader/httpDocument.cxx

@@ -17,32 +17,148 @@
 ////////////////////////////////////////////////////////////////////
 
 #include "httpDocument.h"
+#include "httpClient.h"
 #include "bioStream.h"
 #include "chunkedStream.h"
+#include "identityStream.h"
 
 #ifdef HAVE_SSL
 
 TypeHandle HTTPDocument::_type_handle;
 
+static const char base64_table[64] = {
+  'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
+  'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 
+  'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
+  'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
+  'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
+  'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
+  'w', 'x', 'y', 'z', '0', '1', '2', '3',
+  '4', '5', '6', '7', '8', '9', '+', '/',
+};
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPDocument::Constructor
-//       Access: Public
+//       Access: Private
 //  Description: 
 ////////////////////////////////////////////////////////////////////
 HTTPDocument::
-HTTPDocument(BIO *bio, bool owns_bio) {
+HTTPDocument(HTTPClient *client, BIO *bio) :
+  _client(client),
+  _bio(bio)
+{
+  _owns_bio = false;
+  _source = (IBioStream *)NULL;
+  _persistent_connection = false;
+  _state = S_new;
+  _read_index = 0;
   _file_size = 0;
+  _status_code = 0;
+  _status_string = "No connection";
+  _proxy = _client->get_proxy();
+  _http_version = _client->get_http_version();
+  _http_version_string = _client->get_http_version_string();
+}
 
-  if (bio != (BIO *)NULL) {
-    _source = new IBioStream(bio, owns_bio);
-    read_headers();
-    determine_content_length();
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::send_request
+//       Access: Private
+//  Description: This is normally called immediately after
+//               construction to send the request to the server and
+//               read the response.  It can't be called as part of the
+//               constructor because it may involve an up-and-down
+//               change in the reference count of the HTTPDocument
+//               object, which would inadvertently cause the object to
+//               be deleted if it hasn't returned from its constructor
+//               yet!
+////////////////////////////////////////////////////////////////////
+bool HTTPDocument::
+send_request(const string &method, const URLSpec &url, const string &body) {
+  // Let's call this before we call make_header(), so we'll get the
+  // right HTTP version and proxy information etc.
+  set_url(url);
+  prepare_for_next();
 
-  } else {
-    _source = (IBioStream *)NULL;
-    _status_code = 0;
-    _status_string = "No connection";
+  string header;
+  make_header(header, method, url, body);
+  send_request(header, body);
+
+  if ((get_status_code() / 100) == 3 && get_status_code() != 305) {
+    // Redirect.  Should we handle it automatically?
+    if (!get_redirect().empty() && (method == "GET" || method == "HEAD")) {
+      // Sure!
+      pset<URLSpec> already_seen;
+      bool keep_going;
+      do {
+        keep_going = false;
+        if (downloader_cat.is_debug()) {
+          downloader_cat.debug()
+            << "following redirect to " << get_redirect() << "\n";
+        }
+        URLSpec new_url = get_redirect();
+        if (already_seen.insert(new_url).second) {
+          if (url.has_username()) {
+            new_url.set_username(url.get_username());
+          }
+          set_url(new_url);
+          if (prepare_for_next()) {
+            make_header(header, method, new_url, body);
+            send_request(header, body);
+            keep_going =
+              ((get_status_code() / 100) == 3 && get_status_code() != 305);
+          }
+        }
+      } while (keep_going);
+    }
   }
+
+  return is_valid();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::send_request
+//       Access: Private
+//  Description: This is a lower-level interface than the above
+//               send_request(); it accepts a header and body string
+//               that have already been defined.
+////////////////////////////////////////////////////////////////////
+bool HTTPDocument::
+send_request(const string &header, const string &body) {
+  if (prepare_for_next()) {
+    issue_request(header, body);
+
+    if (get_status_code() == 407 && !_proxy.empty()) {
+      // 407: not authorized to proxy.  Try to get the authorization.
+      string authenticate_request = get_header_value("Proxy-Authenticate");
+      string authorization;
+      if (get_authorization(authorization, authenticate_request, _proxy, true)) {
+        string new_header = header;
+        new_header += "Proxy-Authorization: ";
+        new_header += authorization;
+        new_header += "\r\n";
+        if (prepare_for_next()) {
+          issue_request(new_header, body);
+        }
+      }
+    }
+
+    if (get_status_code() == 401) {
+      // 401: not authorized to remote server.  Try to get the authorization.
+      string authenticate_request = get_header_value("WWW-Authenticate");
+      string authorization;
+      if (get_authorization(authorization, authenticate_request, _url, false)) {
+        string new_header = header;
+        new_header += "Authorization: ";
+        new_header += authorization;
+        new_header += "\r\n";
+        if (prepare_for_next()) {
+          issue_request(new_header, body);
+        }
+      }
+    }
+  }
+
+  return is_valid();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -52,10 +168,7 @@ HTTPDocument(BIO *bio, bool owns_bio) {
 ////////////////////////////////////////////////////////////////////
 HTTPDocument::
 ~HTTPDocument() {
-  if (_source != (IBioStream *)NULL) {
-    delete _source;
-    _source = (IBioStream *)NULL;
-  }
+  free_bio();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -91,6 +204,31 @@ is_regular_file() const {
   return is_valid();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::will_close_connection
+//       Access: Public
+//  Description: Returns true if the server has indicated it will
+//               close the connection after this document has been
+//               read, or false if it will remain open (and future
+//               documents may be requested on the same connection).
+////////////////////////////////////////////////////////////////////
+bool HTTPDocument::
+will_close_connection() const {
+  if (get_http_version() == HTTPClient::HV_10) {
+    // HTTP 1.0 always closes.
+    return true;
+  }
+
+  string connection = get_header_value("Connection");
+  if (downcase(connection) == "close") {
+    // The server says it will close.
+    return true;
+  }
+
+  // Assume the serve will keep it open.
+  return false;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPDocument::open_read_file
 //       Access: Public, Virtual
@@ -102,24 +240,7 @@ is_regular_file() const {
 ////////////////////////////////////////////////////////////////////
 istream *HTTPDocument::
 open_read_file() const {
-  if (_source == (IBioStream *)NULL) {
-    return NULL;
-  }
-
-  string transfer_coding = get_header_value("Transfer-Encoding");
-  for (string::iterator si = transfer_coding.begin();
-       si != transfer_coding.end();
-       ++si) {
-    (*si) = tolower(*si);
-  }
-
-  istream *result = _source;
-  if (transfer_coding == "chunked") {
-    result = new IChunkedStream(_source, true, (HTTPDocument *)this);
-  }
-
-  ((HTTPDocument *)this)->_source = (IBioStream *)NULL;
-  return result;
+  return ((HTTPDocument *)this)->read_body(true);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -131,7 +252,7 @@ open_read_file() const {
 ////////////////////////////////////////////////////////////////////
 string HTTPDocument::
 get_header_value(const string &key) const {
-  Headers::const_iterator hi = _headers.find(key);
+  Headers::const_iterator hi = _headers.find(downcase(key));
   if (hi != _headers.end()) {
     return (*hi).second;
   }
@@ -153,15 +274,123 @@ write_headers(ostream &out) const {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPDocument::read_headers
+//     Function: HTTPDocument::make_header
+//       Access: Private
+//  Description: Formats the appropriate GET or POST (or whatever)
+//               request to send to the server.  Also saves the
+//               indicated url.
+////////////////////////////////////////////////////////////////////
+void HTTPDocument::
+make_header(string &header, const string &method, 
+            const URLSpec &url, const string &body) {
+  set_url(url);
+  _method = method;
+
+  string path;
+  if (_proxy.empty()) {
+    path = _url.get_path();
+  } else {
+    URLSpec url_no_username = _url;
+    url_no_username.set_username(string());
+    path = url_no_username.get_url();
+  }
+
+  ostringstream stream;
+
+  stream 
+    << method << " " << path << " " << get_http_version_string() << "\r\n";
+
+  if (_http_version > HTTPClient::HV_10) {
+    stream 
+      << "Host: " << _url.get_server() << "\r\n";
+    if (!get_persistent_connection()) {
+      stream
+        << "Connection: close\r\n";
+    }
+  }
+
+  if (!body.empty()) {
+    stream
+      << "Content-Type: application/x-www-form-urlencoded\r\n"
+      << "Content-Length: " << body.length() << "\r\n";
+  }
+
+  header = stream.str();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::set_url
+//       Access: Private
+//  Description: Specifies the document's URL before attempting a
+//               connection.  This controls the name of the server to
+//               be contacted, etc.
+////////////////////////////////////////////////////////////////////
+void HTTPDocument::
+set_url(const URLSpec &url) {
+  // If we change between http and https, we have to reset the
+  // connection regardless of proxy.  Otherwise, we have to drop the
+  // connection if the server or port changes, unless we're
+  // communicating through a proxy.
+
+  if (url.get_scheme() != _url.get_scheme() ||
+      (_proxy.empty() && (url.get_server() != _url.get_server() || 
+                          url.get_port() != _url.get_port()))) {
+    free_bio();
+  }
+  _url = url;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::issue_request
+//       Access: Private
+//  Description: Issues the request to the HTTP server and waits for a
+//               response.
+////////////////////////////////////////////////////////////////////
+void HTTPDocument::
+issue_request(const string &header, const string &body) {
+  if (_bio != (BIO *)NULL) {
+    string request = header;
+    request += "\r\n";
+    request += body;
+#ifndef NDEBUG
+    if (downloader_cat.is_spam()) {
+      show_send(request);
+    }
+#endif
+    BIO_puts(_bio, request.c_str());
+    read_http_response();
+
+    if (_source->eof() || _source->fail()) {
+      if (downloader_cat.is_debug()) {
+        downloader_cat.debug()
+          << "Whoops, socket closed.\n";
+        free_bio();
+        if (prepare_for_next()) {
+#ifndef NDEBUG
+          if (downloader_cat.is_spam()) {
+            show_send(request);
+          }
+#endif
+          BIO_puts(_bio, request.c_str());
+          read_http_response();
+        }
+      }
+    }
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::read_http_response
 //       Access: Private
 //  Description: Reads all of the responses from the server up until
 //               the first blank line, and stores the list of header
 //               key:value pairs so retrieved.
 ////////////////////////////////////////////////////////////////////
 void HTTPDocument::
-read_headers() {
+read_http_response() {
   nassertv(_source != (IBioStream *)NULL);
+  _headers.clear();
+  _realm = string();
 
   // The first line back should include the HTTP version and the
   // result code.
@@ -170,9 +399,10 @@ read_headers() {
   if (!line.empty() && line[line.length() - 1] == '\r') {
     line = line.substr(0, line.length() - 1);
   }
-  if (downloader_cat.is_debug()) {
-    downloader_cat.debug() << "recv: " << line << "\n";
+  if (downloader_cat.is_spam()) {
+    downloader_cat.spam() << "recv: " << line << "\n";
   }
+
   if (!(*_source) || line.length() < 5 || line.substr(0, 5) != "HTTP/") {
     // Not an HTTP response.
     _status_code = 0;
@@ -185,7 +415,8 @@ read_headers() {
   while (p < line.length() && !isspace(line[p])) {
     p++;
   }
-  _http_version = line.substr(0, p);
+  _http_version_string = line.substr(0, p);
+  _http_version = HTTPClient::parse_http_version_string(_http_version_string);
 
   while (p < line.length() && isspace(line[p])) {
     p++;
@@ -211,9 +442,10 @@ read_headers() {
   if (!line.empty() && line[line.length() - 1] == '\r') {
     line = line.substr(0, line.length() - 1);
   }
-  if (downloader_cat.is_debug()) {
-    downloader_cat.debug() << "recv: " << line << "\n";
+  if (downloader_cat.is_spam()) {
+    downloader_cat.spam() << "recv: " << line << "\n";
   }
+
   while (!_source->eof() && !_source->fail() && !line.empty()) {
     if (isspace(line[0])) {
       // If the line begins with a space, that continues the previous
@@ -228,13 +460,13 @@ read_headers() {
       // If the line does not begin with a space, that defines a new
       // field.
       if (!field_name.empty()) {
-        _headers[field_name] = field_value;
+        store_header_field(field_name, field_value);
         field_value = string();
       }
 
       size_t colon = line.find(':');
       if (colon != string::npos) {
-        field_name = line.substr(0, colon);
+        field_name = downcase(line.substr(0, colon));
         p = colon + 1;
         while (p < line.length() && isspace(line[p])) {
           p++;
@@ -247,31 +479,531 @@ read_headers() {
     if (!line.empty() && line[line.length() - 1] == '\r') {
       line = line.substr(0, line.length() - 1);
     }
-    if (downloader_cat.is_debug()) {
-      downloader_cat.debug() << "recv: " << line << "\n";
+    if (downloader_cat.is_spam()) {
+      downloader_cat.spam() << "recv: " << line << "\n";
     }
   }
   if (!field_name.empty()) {
-    _headers[field_name] = field_value;
+    store_header_field(field_name, field_value);
     field_value = string();
   }
 
   // A blank line terminates the headers.
+  _state = S_read_header;
+
+  if (get_status_code() / 100 == 1 ||
+      get_status_code() == 204 ||
+      get_status_code() == 304 || 
+      _method == "HEAD") {
+    // These status codes, or method HEAD, indicate we have no body.
+    // Therefore, we have already read the (nonexistent) body.
+    _state = S_read_trailer;
+  }
+
+  _file_size = 0;
+  string content_length = get_header_value("Content-Length");
+  if (!content_length.empty()) {
+    _file_size = atoi(content_length.c_str());
+  }
+  _redirect = get_header_value("Location");
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPDocument::determine_content_length
+//     Function: HTTPDocument::store_header_field
 //       Access: Private
-//  Description: Determines the file size based on the Content-Length
-//               field if it has been supplied.
+//  Description: Stores a single name: value pair in the header list,
+//               or appends the value to the end of the existing
+//               value, if the header has been repeated.
 ////////////////////////////////////////////////////////////////////
 void HTTPDocument::
-determine_content_length() {
+store_header_field(const string &field_name, const string &field_value) {
+  pair<Headers::iterator, bool> insert_result =
+    _headers.insert(Headers::value_type(field_name, field_value));
+
+  if (!insert_result.second) {
+    // It didn't insert; thus, the field already existed.  Append the
+    // new value.
+    Headers::iterator hi = insert_result.first;
+    (*hi).second += ", ";
+    (*hi).second += field_value;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::get_authorization
+//       Access: Private
+//  Description: Looks for a username:password to satisfy the given
+//               authenticate_request string from the server or proxy.
+//               If found, fills in authorization and returns true;
+//               otherwise, returns false.
+////////////////////////////////////////////////////////////////////
+bool HTTPDocument::
+get_authorization(string &authorization, const string &authenticate_request,
+                  const URLSpec &url, bool is_proxy) {
+  AuthenticationSchemes schemes;
+  parse_authentication_schemes(schemes, authenticate_request);
+
+  AuthenticationSchemes::iterator si;
+  si = schemes.find("basic");
+  if (si != schemes.end()) {
+    return get_basic_authorization(authorization, (*si).second, url, is_proxy);
+  }
+
+  downloader_cat.warning() 
+    << "Don't know how to use any of the server's available authorization schemes:\n";
+  for (si = schemes.begin(); si != schemes.end(); ++si) {
+    downloader_cat.warning() << (*si).first << "\n";
+  }
+
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::downcase
+//       Access: Private, Static
+//  Description: Returns the input string with all uppercase letters
+//               converted to lowercase.
+////////////////////////////////////////////////////////////////////
+string HTTPDocument::
+downcase(const string &s) {
+  string result;
+  result.reserve(s.size());
+  string::const_iterator p;
+  for (p = s.begin(); p != s.end(); ++p) {
+    result += tolower(*p);
+  }
+  return result;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::base64_encode
+//       Access: Private, Static
+//  Description: Returns the input string encoded using base64.  No
+//               respect is paid to maintaining a 76-char line length.
+////////////////////////////////////////////////////////////////////
+string HTTPDocument::
+base64_encode(const string &s) {
+  // Collect the string 3 bytes at a time into 24-bit words, then
+  // output each word using 4 bytes.
+  size_t num_words = (s.size() + 2) / 3;
+  string result;
+  result.reserve(num_words * 4);
+  size_t p;
+  for (p = 0; p + 2 < s.size(); p += 3) {
+    unsigned int word = 
+      ((unsigned)s[p] << 16) |
+      ((unsigned)s[p + 1] << 8) |
+      ((unsigned)s[p + 2]);
+    result += base64_table[(word >> 18) & 0x3f];
+    result += base64_table[(word >> 12) & 0x3f];
+    result += base64_table[(word >> 6) & 0x3f];
+    result += base64_table[(word) & 0x3f];
+  }
+  // What's left over?
+  if (p < s.size()) {
+    unsigned int word = ((unsigned)s[p] << 16);
+    p++;
+    if (p < s.size()) {
+      word |= ((unsigned)s[p] << 8);
+      p++;
+      nassertr(p == s.size(), result);
+
+      result += base64_table[(word >> 18) & 0x3f];
+      result += base64_table[(word >> 12) & 0x3f];
+      result += base64_table[(word >> 6) & 0x3f];
+      result += '=';
+    } else {
+      result += base64_table[(word >> 18) & 0x3f];
+      result += base64_table[(word >> 12) & 0x3f];
+      result += '=';
+      result += '=';
+    }
+  }
+
+  return result;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::scan_quoted_or_unquoted_string
+//       Access: Private, Static
+//  Description: Scans the string source beginning at character
+//               position start, to identify either the
+//               (spaced-delimited) unquoted string there, or the
+//               (quote-delimited) quoted string.  In either case,
+//               fills the string found into result, and returns the
+//               next character position after the string (or after
+//               its closing quote mark).
+////////////////////////////////////////////////////////////////////
+size_t HTTPDocument::
+scan_quoted_or_unquoted_string(string &result, const string &source, 
+                               size_t start) {
+  result = string();
+
+  if (start < source.length()) {
+    if (source[start] == '"') {
+      // Quoted string.
+      size_t p = start + 1;
+      while (p < source.length() && source[p] != '"') {
+        if (source[p] == '\\') {
+          // Backslash escapes.
+          ++p;
+          if (p < source.length()) {
+            result += source[p];
+            ++p;
+          }
+        } else {
+          result += source[p];
+          ++p;
+        }
+      }
+      if (p < source.length()) {
+        ++p;
+      }
+      return p;
+    }
+
+    // Unquoted string.
+    size_t p = start;
+    while (p < source.length() && source[p] != ',' && !isspace(source[p])) {
+      result += source[p];
+      ++p;
+    }
+
+    return p;
+  }
+
+  // Empty string.
+  return start;
+}
+
+#ifndef NDEBUG
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::show_send
+//       Access: Private, Static
+//  Description: Writes the outgoing message, one line at a time, to
+//               the debugging log.
+////////////////////////////////////////////////////////////////////
+void HTTPDocument::
+show_send(const string &message) {
+  size_t start = 0;
+  size_t newline = message.find('\n', start);
+  while (newline != string::npos) {
+    downloader_cat.spam()
+      << "send: " << message.substr(start, newline - start + 1);
+    start = newline + 1;
+    newline = message.find('\n', start);
+  }
+
+  if (start < message.length()) {
+    downloader_cat.spam()
+      << "send: " << message.substr(start) << " (no newline)\n";
+  }
+}
+#endif   // NDEBUG
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::parse_authentication_schemes
+//       Access: Private, Static
+//  Description: Decodes the text following a WWW-Authenticate: or
+//               Proxy-Authenticate: header field.
+////////////////////////////////////////////////////////////////////
+void HTTPDocument::
+parse_authentication_schemes(HTTPDocument::AuthenticationSchemes &schemes,
+                             const string &field_value) {
+  // This string will consist of one or more records of the form:
+  //
+  //  scheme token=value[,token=value[,...]]
+  //
+  // If there are multiple records, they will be comma-delimited,
+  // which makes parsing just a bit tricky.
+
+  // Start by skipping initial whitespace.
+  size_t p = 0;
+  while (p < field_value.length() && isspace(field_value[p])) {
+    ++p;
+  }
+
+  if (p < field_value.length()) {
+    size_t q = p;
+    while (q < field_value.length() && !isspace(field_value[q])) {
+      ++q;
+    }
+    // Here's our first scheme.
+    string scheme = downcase(field_value.substr(p, q - p));
+    Tokens *tokens = &(schemes[scheme]);
+    
+    // Now pull off the tokens, one at a time.
+    p = q + 1;
+    while (p < field_value.length()) {
+      q = p;
+      while (q < field_value.length() && field_value[q] != '=' && 
+             field_value[q] != ',' && !isspace(field_value[q])) {
+        ++q;
+      }
+      if (field_value[q] == '=') {
+        // This is a token.
+        string token = downcase(field_value.substr(p, q - p));
+        string value;
+        p = scan_quoted_or_unquoted_string(value, field_value, q + 1);
+        (*tokens)[token] = value;
+
+        // Skip trailing whitespace and extra commas.
+        while (p < field_value.length() && 
+               (field_value[p] == ',' || isspace(field_value[p]))) {
+          ++p;
+        }
+
+      } else {
+        // This is not a token; it must be the start of a new scheme.
+        scheme = downcase(field_value.substr(p, q - p));
+        tokens = &(schemes[scheme]);
+        p = q + 1;
+      }
+    }
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::read_body
+//       Access: Private
+//  Description: Returns a newly-allocated istream suitable for
+//               reading the body of the document.  If owns_source is
+//               true, the ownership of the _source pointer will be
+//               passed to the istream; otherwise, it will be
+//               retained.  (owns_source must be true in order to read
+//               "identity" encoded documents.)
+////////////////////////////////////////////////////////////////////
+istream *HTTPDocument::
+read_body(bool owns_source) {
+  if (_state != S_read_header || _source == (IBioStream *)NULL) {
+    return NULL;
+  }
+
+  string transfer_coding = downcase(get_header_value("Transfer-Encoding"));
   string content_length = get_header_value("Content-Length");
-  if (!content_length.empty()) {
-    _file_size = atoi(content_length.c_str());
+
+  istream *result;
+  if (transfer_coding == "chunked") {
+    // Chunked can be used directly.
+    _file_size = 0;
+    _state = S_started_body;
+    _read_index++;
+    result = new IChunkedStream(_source, owns_source, (HTTPDocument *)this);
+    if (owns_source) {
+      _source = (IBioStream *)NULL;
+    }
+
+  } else if (!content_length.empty()) {
+    // If we have a content length, we can use an IdentityStream.
+    _state = S_started_body;
+    _read_index++;
+    result = new IIdentityStream(_source, owns_source, (HTTPDocument *)this, _file_size);
+    if (owns_source) {
+      _source = (IBioStream *)NULL;
+    }
+
+  } else if (owns_source) {
+    // If we own the source, we can return it.
+    _state = S_started_body;
+    result = _source;
+    _source = (IBioStream *)NULL;
+
+  } else {
+    // Otherwise, we don't own the source; too bad.
+    result = NULL;
   }
+
+  return result;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::prepare_for_next
+//       Access: Private
+//  Description: Resets the state to prepare it for sending a new
+//               request to the server.  This might mean closing the
+//               connection and opening a new one, or it might mean
+//               skipping past the unread body in the persistent
+//               connection, or it might do nothing at all if the body
+//               has already been completely read.
+////////////////////////////////////////////////////////////////////
+bool HTTPDocument::
+prepare_for_next() {
+  if (get_persistent_connection() && !will_close_connection() &&
+      _proxy == _client->get_proxy()) {
+    // See if we can reuse the current connection.
+    if (_state == S_read_header) {
+      // We have read the header; now skip past the body.
+      istream *body = read_body(false); 
+      if (body != (istream *)NULL) {
+        string line;
+        getline(*body, line);
+        while (!body->fail() && !body->eof()) {
+          if (downloader_cat.is_spam()) {
+            downloader_cat.spam() << "skip: " << line << "\n";
+          }
+          getline(*body, line);
+        }
+        nassertr(body != _source, false);
+        delete body;
+      }
+    }
+
+    if (_source == (IBioStream *)NULL) {
+      _source = new IBioStream(_bio, false);
+    }
+
+    if (_state == S_read_body) {
+      // We have read the body, but there's a trailer to read.
+      string line;
+      getline(*_source, line);
+      if (!line.empty() && line[line.length() - 1] == '\r') {
+        line = line.substr(0, line.length() - 1);
+      }
+      if (downloader_cat.is_spam()) {
+        downloader_cat.spam() << "skip: " << line << "\n";
+      }
+      while (!_source->eof() && !_source->fail() && !line.empty()) {
+        getline(*_source, line);
+        if (!line.empty() && line[line.length() - 1] == '\r') {
+          line = line.substr(0, line.length() - 1);
+        }
+        if (downloader_cat.is_spam()) {
+          downloader_cat.spam() << "skip: " << line << "\n";
+        }
+      }
+      _state = S_read_trailer;
+    }
+
+    if (_state == S_read_trailer) {
+      // Great; this connection is ready to go!
+      return true;
+    }
+  }
+
+  if (_bio != (BIO *)NULL && _state == S_new) {
+    // If we have a BIO and the _state is S_new, then we haven't done
+    // anything with the BIO yet, so we can still use it.
+    if (_source == (IBioStream *)NULL) {
+      _source = new IBioStream(_bio, false);
+    }
+    return true;
+  }
+
+  // Either the client will close the connection after reading the
+  // body, or we were only partly through reading the body elsewhere;
+  // or possibly we don't have a connection yet at all.  In any case,
+  // we must now get a new connection.
+  if (_bio != (BIO *)NULL && !_owns_bio) {
+    // We have a connection, but we don't own it, so we can't close
+    // it.  Too bad.
+    return false;
+  }
+  
+  // Go ahead and close the old BIO.
+  free_bio();
+
+  _proxy = _client->get_proxy();
+  _http_version = _client->get_http_version();
+  _http_version_string = _client->get_http_version_string();
+  _bio = _client->establish_connection(_url);
+  _owns_bio = true;
+  if (_bio != (BIO *)NULL) {
+    _source = new IBioStream(_bio, false);
+    return true;
+  }
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::free_bio
+//       Access: Private
+//  Description: Frees the BIO and its IBioStream object, if
+//               allocated.  This will close the connection if it is
+//               open.
+////////////////////////////////////////////////////////////////////
+void HTTPDocument::
+free_bio() {
+  if (_source != (IBioStream *)NULL) {
+    delete _source;
+    _source = (IBioStream *)NULL;
+  }
+  if (_bio != (BIO *)NULL) {
+    if (_owns_bio) {
+      // TODO: We should be more careful here to manage reference
+      // counts so we don't free the bio out from under a BIOStreamBuf
+      // that's trying to read from it.
+      if (downloader_cat.is_debug()) {
+        const URLSpec &url = _proxy.empty() ? _url : _proxy;
+        downloader_cat.debug()
+          << "Dropping connection to " << url.get_server() << "\n";
+      }
+      BIO_free_all(_bio);
+    }
+    _bio = (BIO *)NULL;
+  }
+  _read_index++;
+  _state = S_new;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPDocument::get_basic_authorization
+//       Access: Private
+//  Description: Looks for a username:password to satisfy the "Basic"
+//               scheme authorization request from the server or
+//               proxy.
+////////////////////////////////////////////////////////////////////
+bool HTTPDocument::
+get_basic_authorization(string &authorization, const HTTPDocument::Tokens &tokens, const URLSpec &url, bool is_proxy) {
+  Tokens::const_iterator ti;
+  ti = tokens.find("realm");
+  if (ti != tokens.end()) {
+    _realm = (*ti).second;
+  }
+
+  string username;
+
+  // Look in several places in order to find the matching username.
+
+  // Fist, if there's a username on the URL, that always wins.
+  if (url.has_username()) {
+    username = url.get_username();
+  }
+
+  // Otherwise, start looking on the HTTPClient.  
+  if (is_proxy) {
+    if (username.empty()) {
+      // Try the *proxy/realm.
+      username = _client->get_username("*proxy", _realm);
+    }
+    if (username.empty()) {
+      // Then, try *proxy/any realm.
+      username = _client->get_username("*proxy", string());
+    }
+  }
+  if (username.empty()) {
+    // Try the specific server/realm.
+    username = _client->get_username(url.get_server(), _realm);
+  }
+  if (username.empty()) {
+    // Then, try the specific server/any realm.
+    username = _client->get_username(url.get_server(), string());
+  }
+  if (username.empty()) {
+    // Then, try any server with this realm.
+    username = _client->get_username(string(), _realm);
+  }
+  if (username.empty()) {
+    // Then, take the general password.
+    username = _client->get_username(string(), string());
+  }
+
+  if (username.empty()) {
+    // No username:password available.
+    return false;
+  }
+
+  authorization = "Basic " + base64_encode(username);
+  return true;
+}
 
 #endif  // HAVE_SSL

panda/src/downloader/httpDocument.h

@@ -28,20 +28,28 @@
 
 #ifdef HAVE_SSL
 
+#include "httpClient.h"
 #include "urlSpec.h"
 #include "virtualFile.h"
 #include "pmap.h"
 #include <openssl/ssl.h>
 
 class IBioStream;
+class HTTPClient;
 
 ////////////////////////////////////////////////////////////////////
 //       Class : HTTPDocument
 // Description : A single document retrieved from an HTTP server.
 ////////////////////////////////////////////////////////////////////
 class EXPCL_PANDAEXPRESS HTTPDocument : public VirtualFile {
+private:
+  HTTPDocument(HTTPClient *client, BIO *bio = NULL);
+
+  bool send_request(const string &method, const URLSpec &url, 
+                    const string &body);
+  bool send_request(const string &header, const string &body);
+
 public:
-  HTTPDocument(BIO *bio, bool owns_bio);
   virtual ~HTTPDocument();
 
   virtual VirtualFileSystem *get_file_system() const;
@@ -50,32 +58,90 @@ public:
   virtual bool is_regular_file() const;
   virtual istream *open_read_file() const;
 
+  bool will_close_connection() const;
+
 PUBLISHED:
   INLINE bool is_valid() const;
-  INLINE const string &get_http_version() const;
+  INLINE const URLSpec &get_url() const;
+  INLINE HTTPClient::HTTPVersion get_http_version() const;
+  INLINE const string &get_http_version_string() const;
   INLINE int get_status_code() const;
   INLINE const string &get_status_string() const;
+  INLINE const string &get_realm() const;
+  INLINE const URLSpec &get_redirect() const;
   string get_header_value(const string &key) const;
 
+  INLINE void set_persistent_connection(bool persistent_connection);
+  INLINE bool get_persistent_connection() const;
+
   INLINE size_t get_file_size() const;
 
   void write_headers(ostream &out) const;
 
+  INLINE bool get_document(const URLSpec &url, const string &body = string());
+  INLINE bool get_header(const URLSpec &url);
+
 private:
-  void read_headers();
-  void determine_content_length();
+  void make_header(string &header, const string &method, 
+                   const URLSpec &url, const string &body);
+  void set_url(const URLSpec &url);
+  void issue_request(const string &header, const string &body);
+  void read_http_response();
+  void store_header_field(const string &field_name, const string &field_value);
+  bool get_authorization(string &authorization,
+                         const string &authenticate_request, 
+                         const URLSpec &url, bool is_proxy);
+
+  static string downcase(const string &s);
+  static string base64_encode(const string &s);
+  static size_t scan_quoted_or_unquoted_string(string &result, const string &source, size_t start);
+
+#ifndef NDEBUG
+  static void show_send(const string &message);
+#endif
 
-  IBioStream *_source;
+  istream *read_body(bool owns_source);
+  bool prepare_for_next();
+  void free_bio();
 
-  string _http_version;
+  HTTPClient *_client;
+  URLSpec _proxy;
+  BIO *_bio;
+  bool _owns_bio;
+  IBioStream *_source;
+  bool _persistent_connection;
+
+  URLSpec _url;
+  string _method;
+
+  enum State {
+    S_new,
+    S_read_header,
+    S_started_body,
+    S_read_body,
+    S_read_trailer
+  };
+  State _state;
+  int _read_index;
+
+  HTTPClient::HTTPVersion _http_version;
+  string _http_version_string;
   int _status_code;
   string _status_string;
+  string _realm;
+  URLSpec _redirect;
 
   typedef pmap<string, string> Headers;
   Headers _headers;
 
   size_t _file_size;
 
+  typedef pmap<string, string> Tokens;
+  typedef pmap<string, Tokens> AuthenticationSchemes;
+  static void parse_authentication_schemes(AuthenticationSchemes &schemes,
+                                           const string &field_value);
+  bool get_basic_authorization(string &authorization, const Tokens &tokens,
+                               const URLSpec &url, bool is_proxy);
 
 public:
   virtual TypeHandle get_type() const {
@@ -94,6 +160,8 @@ public:
 private:
   static TypeHandle _type_handle;
   friend class ChunkedStreamBuf;
+  friend class IdentityStreamBuf;
+  friend class HTTPClient;
 };
 
 #include "httpDocument.I"

panda/src/downloader/identityStream.I

@@ -0,0 +1,63 @@
+// Filename: identityStream.I
+// Created by:  drose (09Oct02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: IIdentityStream::Constructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE IIdentityStream::
+IIdentityStream() : istream(&_buf) {
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IIdentityStream::Constructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE IIdentityStream::
+IIdentityStream(istream *source, bool owns_source, 
+               HTTPDocument *doc, size_t content_length) : istream(&_buf) {
+  open(source, owns_source, doc, content_length);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IIdentityStream::open
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE IIdentityStream &IIdentityStream::
+open(istream *source, bool owns_source, HTTPDocument *doc, 
+     size_t content_length) {
+  clear(0);
+  _buf.open_read(source, owns_source, doc, content_length);
+  return *this;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IIdentityStream::close
+//       Access: Public
+//  Description: Resets the IdentityStream to empty, but does not actually
+//               close the source IDENTITY unless owns_source was true.
+////////////////////////////////////////////////////////////////////
+INLINE IIdentityStream &IIdentityStream::
+close() {
+  _buf.close_read();
+  return *this;
+}

panda/src/downloader/identityStream.cxx

@@ -0,0 +1,19 @@
+// Filename: identityStream.cxx
+// Created by:  drose (09Oct02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#include "identityStream.h"

panda/src/downloader/identityStream.h

@@ -0,0 +1,60 @@
+// Filename: identityStream.h
+// Created by:  drose (09Oct02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef IDENTITYSTREAM_H
+#define IDENTITYSTREAM_H
+
+#include "pandabase.h"
+
+#include "identityStreamBuf.h"
+
+class HTTPDocument;
+
+////////////////////////////////////////////////////////////////////
+//       Class : IIdentityStream
+// Description : An input stream object that reads data from a source
+//               istream, but automatically decodes the "identity"
+//               transfer-coding specified by an HTTP server.
+//
+//               In practice, this just means it reads from the sub
+//               stream (like a SubStreamBuf) up to but not past the
+//               specified content-length.  (If the content-length was
+//               unspecified, this class cannot be used.)  It also
+//               updates the HTTPDocument when the stream is
+//               completely read.
+////////////////////////////////////////////////////////////////////
+// No need to export from DLL.
+class IIdentityStream : public istream {
+public:
+  INLINE IIdentityStream();
+  INLINE IIdentityStream(istream *source, bool owns_source, 
+                         HTTPDocument *doc, size_t content_length);
+
+  INLINE IIdentityStream &open(istream *source, bool owns_source, 
+                              HTTPDocument *doc, size_t content_length);
+  INLINE IIdentityStream &close();
+
+private:
+  IdentityStreamBuf _buf;
+};
+
+#include "identityStream.I"
+
+#endif
+
+

panda/src/downloader/identityStreamBuf.cxx

@@ -0,0 +1,160 @@
+// Filename: identityStreamBuf.cxx
+// Created by:  drose (09Oct02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#include "identityStreamBuf.h"
+
+#ifndef HAVE_STREAMSIZE
+// Some compilers (notably SGI) don't define this for us
+typedef int streamsize;
+#endif /* HAVE_STREAMSIZE */
+
+////////////////////////////////////////////////////////////////////
+//     Function: IdentityStreamBuf::Constructor
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+IdentityStreamBuf::
+IdentityStreamBuf() {
+  _source = (istream *)NULL;
+  _owns_source = false;
+  _bytes_remaining = 0;
+
+#ifdef WIN32_VC
+  // In spite of the claims of the MSDN Library to the contrary,
+  // Windows doesn't seem to provide an allocate() function, so we'll
+  // do it by hand.
+  char *buf = new char[4096];
+  char *ebuf = buf + 4096;
+  setg(buf, ebuf, ebuf);
+  setp(buf, ebuf);
+
+#else
+  allocate();
+  setg(base(), ebuf(), ebuf());
+  setp(base(), ebuf());
+#endif
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IdentityStreamBuf::Destructor
+//       Access: Public, Virtual
+//  Description:
+////////////////////////////////////////////////////////////////////
+IdentityStreamBuf::
+~IdentityStreamBuf() {
+  close_read();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IdentityStreamBuf::open_read
+//       Access: Public
+//  Description: If the document pointer is non-NULL, it will be
+//               updated with the length of the file as it is derived
+//               from the identity encoding.
+////////////////////////////////////////////////////////////////////
+void IdentityStreamBuf::
+open_read(istream *source, bool owns_source, HTTPDocument *doc, 
+          size_t content_length) {
+  _source = source;
+  _owns_source = owns_source;
+  _doc = doc;
+  _bytes_remaining = content_length;
+
+  if (_doc != (HTTPDocument *)NULL) {
+    _read_index = doc->_read_index;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IdentityStreamBuf::close_read
+//       Access: Public
+//  Description:
+////////////////////////////////////////////////////////////////////
+void IdentityStreamBuf::
+close_read() {
+  if (_source != (istream *)NULL) {
+    if (_owns_source) {
+      delete _source;
+      _owns_source = false;
+    }
+    _source = (istream *)NULL;
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: IdentityStreamBuf::underflow
+//       Access: Protected, Virtual
+//  Description: Called by the system istream implementation when its
+//               internal buffer needs more characters.
+////////////////////////////////////////////////////////////////////
+int IdentityStreamBuf::
+underflow() {
+  // Sometimes underflow() is called even if the buffer is not empty.
+  if (gptr() >= egptr()) {
+    size_t buffer_size = egptr() - eback();
+    gbump(-(int)buffer_size);
+
+    size_t num_bytes = buffer_size;
+    size_t read_count = read_chars(gptr(), buffer_size);
+
+    if (read_count != num_bytes) {
+      // Oops, we didn't read what we thought we would.
+      if (read_count == 0) {
+        return EOF;
+      }
+
+      // Slide what we did read to the top of the buffer.
+      nassertr(read_count < num_bytes, EOF);
+      size_t delta = num_bytes - read_count;
+      memmove(gptr() + delta, gptr(), read_count);
+      gbump(delta);
+    }
+  }
+
+  return (unsigned char)*gptr();
+}
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: IdentityStreamBuf::read_chars
+//       Access: Private
+//  Description: Gets some characters from the source stream.
+////////////////////////////////////////////////////////////////////
+size_t IdentityStreamBuf::
+read_chars(char *start, size_t length) {
+  if (_bytes_remaining == 0) {
+    return 0;
+  }
+
+  // Extract some of the bytes remaining in the chunk.
+  length = min(length, _bytes_remaining);
+  _source->read(start, length);
+  length = _source->gcount();
+  _bytes_remaining -= length;
+
+  if (_bytes_remaining == 0) {
+    // We're done.
+    if (_doc != (HTTPDocument *)NULL && _read_index == _doc->_read_index) {
+      // An IdentityStreamBuf doesn't have a trailer, so we've already
+      // "read" it.
+      _doc->_state = HTTPDocument::S_read_trailer;
+    }
+  }
+
+  return length;
+}

panda/src/downloader/identityStreamBuf.h

@@ -0,0 +1,54 @@
+// Filename: identityStreamBuf.h
+// Created by:  drose (09Oct02)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
+//
+// All use of this software is subject to the terms of the Panda 3d
+// Software license.  You should have received a copy of this license
+// along with this source code; you will also find a current copy of
+// the license at http://www.panda3d.org/license.txt .
+//
+// To contact the maintainers of this program write to
+// [email protected] .
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef IDENTITYSTREAMBUF_H
+#define IDENTITYSTREAMBUF_H
+
+#include "pandabase.h"
+#include "httpDocument.h"
+#include "pointerTo.h"
+
+////////////////////////////////////////////////////////////////////
+//       Class : IdentityStreamBuf
+// Description : The streambuf object that implements
+//               IIdentityStream.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDAEXPRESS IdentityStreamBuf : public streambuf {
+public:
+  IdentityStreamBuf();
+  virtual ~IdentityStreamBuf();
+
+  void open_read(istream *source, bool owns_source, HTTPDocument *doc,
+                 size_t content_length);
+  void close_read();
+
+protected:
+  virtual int underflow(void);
+
+private:
+  size_t read_chars(char *start, size_t length);
+
+  istream *_source;
+  bool _owns_source;
+  size_t _bytes_remaining;
+
+  PT(HTTPDocument) _doc;
+  int _read_index;
+};
+
+#endif

panda/src/downloader/urlSpec.I

@@ -47,6 +47,36 @@ operator = (const string &url) {
   set_url(url);
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: URLSpec::Operator ==
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE bool URLSpec::
+operator == (const URLSpec &other) const {
+  return _url == other._url;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: URLSpec::Operator !=
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE bool URLSpec::
+operator != (const URLSpec &other) const {
+  return !operator == (other);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: URLSpec::Operator <
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+INLINE bool URLSpec::
+operator < (const URLSpec &other) const {
+  return _url < other._url;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: URLSpec::has_scheme
 //       Access: Published

panda/src/downloader/urlSpec.h

@@ -38,6 +38,10 @@ PUBLISHED:
   INLINE void operator = (const string &url);
   void operator = (const URLSpec &copy);
 
+  INLINE bool operator == (const URLSpec &other) const;
+  INLINE bool operator != (const URLSpec &other) const;
+  INLINE bool operator < (const URLSpec &other) const;
+
   INLINE bool has_scheme() const;
   INLINE bool has_authority() const;
   INLINE bool has_username() const;